From defd4f07f92d6fa5e7d344357d12e15390f95a5e Mon Sep 17 00:00:00 2001 From: ibby Date: Mon, 15 Jun 2026 11:09:21 -0400 Subject: [PATCH 001/101] add error info to workerd/io --- src/workerd/io/trace.c++ | 46 ++++++++++++++++++++++++--- src/workerd/io/trace.h | 29 ++++++++++++++++- src/workerd/io/worker-interface.capnp | 15 +++++++++ 3 files changed, 85 insertions(+), 5 deletions(-) diff --git a/src/workerd/io/trace.c++ b/src/workerd/io/trace.c++ index a786c5d38be..8b7a7e4be67 100644 --- a/src/workerd/io/trace.c++ +++ b/src/workerd/io/trace.c++ @@ -733,19 +733,53 @@ FetchResponseInfo FetchResponseInfo::clone() const { return FetchResponseInfo(statusCode); } -Log::Log(kj::Date timestamp, LogLevel logLevel, kj::String message) +ErrorInfo::ErrorInfo(kj::String name, kj::String message, kj::Maybe stack) + : name(kj::mv(name)), + message(kj::mv(message)), + stack(kj::mv(stack)) {} + +ErrorInfo::ErrorInfo(rpc::Trace::ErrorInfo::Reader reader) + : name(kj::str(reader.getName())), + message(kj::str(reader.getMessage())) { + if (reader.hasStack()) { + stack = kj::str(reader.getStack()); + } +} + +void ErrorInfo::copyTo(rpc::Trace::ErrorInfo::Builder builder) const { + builder.setName(name); + builder.setMessage(message); + KJ_IF_SOME(s, stack) { + builder.setStack(s); + } +} + +ErrorInfo ErrorInfo::clone() const { + return ErrorInfo(kj::str(name), kj::str(message), + stack.map([](const kj::String& s) { return kj::str(s); })); +} + +Log::Log(kj::Date timestamp, + LogLevel logLevel, + kj::String message, + kj::Maybe errorInfo) : timestamp(timestamp), logLevel(logLevel), - message(kj::mv(message)) {} + message(kj::mv(message)), + errorInfo(kj::mv(errorInfo)) {} void Log::copyTo(rpc::Trace::Log::Builder builder) const { builder.setTimestampNs((timestamp - kj::UNIX_EPOCH) / kj::NANOSECONDS); builder.setLogLevel(logLevel); builder.setMessage(message); + KJ_IF_SOME(info, errorInfo) { + info.copyTo(builder.initErrorInfo()); + } } Log Log::clone() const { - return Log(timestamp, logLevel, kj::str(message)); + return Log(timestamp, logLevel, kj::str(message), + errorInfo.map([](const ErrorInfo& info) { return info.clone(); })); } Exception::Exception( @@ -758,7 +792,11 @@ Exception::Exception( Log::Log(rpc::Trace::Log::Reader reader) : timestamp(kj::UNIX_EPOCH + reader.getTimestampNs() * kj::NANOSECONDS), logLevel(reader.getLogLevel()), - message(kj::str(reader.getMessage())) {} + message(kj::str(reader.getMessage())) { + if (reader.hasErrorInfo()) { + errorInfo = ErrorInfo(reader.getErrorInfo()); + } +} Exception::Exception(rpc::Trace::Exception::Reader reader) : timestamp(kj::UNIX_EPOCH + reader.getTimestampNs() * kj::NANOSECONDS), diff --git a/src/workerd/io/trace.h b/src/workerd/io/trace.h index 0bb75c5334a..dd7c08b8973 100644 --- a/src/workerd/io/trace.h +++ b/src/workerd/io/trace.h @@ -607,9 +607,31 @@ struct StreamDiagnosticsEvent final { StreamDiagnosticsEvent clone() const; }; +// Describes structured Error fields extracted from an Error argument passed to a +// console.* call. Attached to a Log entry via Log::errorInfo. Mirrors the shape of +// tracing::Exception (which represents uncaught exceptions) but has no timestamp of +// its own — the enclosing Log already carries one. +struct ErrorInfo final { + explicit ErrorInfo(kj::String name, kj::String message, kj::Maybe stack); + ErrorInfo(rpc::Trace::ErrorInfo::Reader reader); + ErrorInfo(ErrorInfo&&) noexcept = default; + KJ_DISALLOW_COPY(ErrorInfo); + ~ErrorInfo() noexcept(false) = default; + + kj::String name; + kj::String message; + kj::Maybe stack; + + void copyTo(rpc::Trace::ErrorInfo::Builder builder) const; + ErrorInfo clone() const; +}; + // Describes a log event struct Log final { - explicit Log(kj::Date timestamp, LogLevel logLevel, kj::String message); + explicit Log(kj::Date timestamp, + LogLevel logLevel, + kj::String message, + kj::Maybe errorInfo = kj::none); Log(rpc::Trace::Log::Reader reader); Log(Log&&) noexcept = default; KJ_DISALLOW_COPY(Log); @@ -622,6 +644,11 @@ struct Log final { // TODO(soon): Just string for now. Eventually, capture serialized JS objects. kj::String message; + // Populated when at least one argument to the originating console call was a + // native Error. Carries the structured {name, message, stack} fields so that tail + // workers can surface them without depending on lossy stringification of `message`. + kj::Maybe errorInfo; + void copyTo(rpc::Trace::Log::Builder builder) const; Log clone() const; }; diff --git a/src/workerd/io/worker-interface.capnp b/src/workerd/io/worker-interface.capnp index 0029fb00766..cea897f1244 100644 --- a/src/workerd/io/worker-interface.capnp +++ b/src/workerd/io/worker-interface.capnp @@ -81,6 +81,21 @@ struct Trace @0x8e8d911203762d34 { } message @2 :Text; + + errorInfo @3 :ErrorInfo; + # Structured Error fields extracted from any Error argument passed to the console + # call that produced this log (e.g. `console.error(new Error("x"))`). Absent when + # none of the arguments was a native Error. The stringified form of the arguments + # remains in `message` unchanged for backwards compatibility. + } + + struct ErrorInfo { + # Mirrors the shape of `Exception` below, but is attached to a Log entry rather + # than being a standalone trace item. No timestamp: the containing Log already + # carries `timestampNs`. + name @0 :Text; + message @1 :Text; + stack @2 :Text; } obsolete26 @26 :List(UserSpanData); From 30e75963c1a5ba657e06ee1bd4d4e1dc77a898d7 Mon Sep 17 00:00:00 2001 From: ibby Date: Mon, 15 Jun 2026 15:21:37 -0400 Subject: [PATCH 002/101] populate error info --- src/workerd/io/tracer.c++ | 13 ++++++--- src/workerd/io/tracer.h | 10 +++++-- src/workerd/io/worker.c++ | 55 ++++++++++++++++++++++++++++++++++++++- 3 files changed, 71 insertions(+), 7 deletions(-) diff --git a/src/workerd/io/tracer.c++ b/src/workerd/io/tracer.c++ index aea8ecec04a..a4a7bf145ac 100644 --- a/src/workerd/io/tracer.c++ +++ b/src/workerd/io/tracer.c++ @@ -122,7 +122,8 @@ constexpr kj::LiteralStringConst logSizeExceeded = void WorkerTracer::addLog(const tracing::InvocationSpanContext& context, kj::Date timestamp, LogLevel logLevel, - kj::String message) { + kj::String message, + kj::Maybe errorInfo) { if (pipelineLogLevel == PipelineLogLevel::NONE) { return; } @@ -133,9 +134,13 @@ void WorkerTracer::addLog(const tracing::InvocationSpanContext& context, KJ_IF_SOME(writer, maybeTailStreamWriter) { // If message is too big on its own, truncate it. size_t messageSize = kj::min(message.size(), MAX_TRACE_BYTES); + // Clone errorInfo for the STW path because the batched-tail path below also needs it. + auto streamErrorInfo = errorInfo.map( + [](const tracing::ErrorInfo& info) { return info.clone(); }); writer->report(context, - {tracing::Log(timestamp, logLevel, kj::str(message.first(messageSize)))}, timestamp, - messageSize); + {tracing::Log(timestamp, logLevel, kj::str(message.first(messageSize)), + kj::mv(streamErrorInfo))}, + timestamp, messageSize); } if (trace->exceededLogLimit) { @@ -150,7 +155,7 @@ void WorkerTracer::addLog(const tracing::InvocationSpanContext& context, trace->truncated = true; } else { trace->bytesUsed += messageSize; - trace->logs.add(timestamp, logLevel, kj::mv(message)); + trace->logs.add(timestamp, logLevel, kj::mv(message), kj::mv(errorInfo)); } } diff --git a/src/workerd/io/tracer.h b/src/workerd/io/tracer.h index 012a9b92603..4fc3b411011 100644 --- a/src/workerd/io/tracer.h +++ b/src/workerd/io/tracer.h @@ -37,10 +37,15 @@ class BaseTracer: public kj::Refcounted { } // Adds log line to trace. For Spectre, timestamp should only be as accurate as JS Date.now(). + // + // `errorInfo` carries structured Error fields extracted from any Error argument passed to + // the originating console.* call (e.g. `console.error(new Error("x"))`). Pass `kj::none` + // for non-console internal warning paths, or when no argument was a native Error. virtual void addLog(const tracing::InvocationSpanContext& context, kj::Date timestamp, LogLevel logLevel, - kj::String message) = 0; + kj::String message, + kj::Maybe errorInfo = kj::none) = 0; // Add a span open event. virtual void addSpanOpen(tracing::SpanId spanId, tracing::SpanId parentSpanId, @@ -146,7 +151,8 @@ class WorkerTracer final: public BaseTracer { void addLog(const tracing::InvocationSpanContext& context, kj::Date timestamp, LogLevel logLevel, - kj::String message) override; + kj::String message, + kj::Maybe errorInfo = kj::none) override; void addSpanOpen(tracing::SpanId spanId, tracing::SpanId parentSpanId, kj::ConstString operationName, diff --git a/src/workerd/io/worker.c++ b/src/workerd/io/worker.c++ index 8f9227a0a2e..b31edb28b09 100644 --- a/src/workerd/io/worker.c++ +++ b/src/workerd/io/worker.c++ @@ -2127,6 +2127,41 @@ void Worker::processEntrypointClass(jsg::Lock& js, }); } +namespace { + +// Reads a string-valued property off a v8 Object. Returns kj::none if the property is missing, +// not a string, or its accessor throws (some user code defines `stack` as a throwing getter). +// Caller must already hold a v8::TryCatch in scope. +kj::Maybe tryGetStringProperty(jsg::Lock& js, + v8::Local obj, + kj::StringPtr propName) { + auto context = js.v8Context(); + auto key = jsg::v8StrIntern(js.v8Isolate, propName); + v8::Local val; + if (!obj->Get(context, key).ToLocal(&val)) { + return kj::none; + } + if (!val->IsString()) { + return kj::none; + } + return kj::str(val.As()); +} + +// Extracts {name, message, stack} from a v8 Error. Returns kj::none if name or message are +// missing (defensive: a stripped-down Error-like is not useful enough to surface as ErrorInfo). +// Caller must hold a v8::TryCatch covering this call. +kj::Maybe extractErrorInfo(jsg::Lock& js, v8::Local errorObj) { + KJ_IF_SOME(name, tryGetStringProperty(js, errorObj, "name"_kj)) { + KJ_IF_SOME(message, tryGetStringProperty(js, errorObj, "message"_kj)) { + auto stack = tryGetStringProperty(js, errorObj, "stack"_kj); + return tracing::ErrorInfo(kj::mv(name), kj::mv(message), kj::mv(stack)); + } + } + return kj::none; +} + +} // namespace + void Worker::handleLog(jsg::Lock& js, const LoggingOptions& loggingOptions, LogLevel level, @@ -2146,6 +2181,23 @@ void Worker::handleLog(jsg::Lock& js, // terminating, usually as a result of an infinite loop. We need to perform the initialization // here because `message` is called multiple times. v8::TryCatch tryCatch(js.v8Isolate); + + // Scan arguments for the first native Error and capture its {name, message, stack} as + // structured ErrorInfo. We do this independently of the stringification below so that + // (a) calling `message()` multiple times doesn't re-extract, and (b) the existing + // stringification behavior is unchanged for backwards compatibility — the structured + // ErrorInfo is purely additive. + kj::Maybe capturedError; + for (auto i: kj::zeroTo(length)) { + if (!tryCatch.CanContinue()) break; + auto arg = info[i]; + if (!arg->IsNativeError()) continue; + js.withinHandleScope([&] { + capturedError = extractErrorInfo(js, arg.As()); + }); + if (capturedError != kj::none) break; + } + auto message = [&]() { int length = info.Length(); kj::Vector stringified(length); @@ -2221,7 +2273,8 @@ void Worker::handleLog(jsg::Lock& js, KJ_IF_SOME(ioContext, IoContext::tryCurrent()) { KJ_IF_SOME(tracer, ioContext.getWorkerTracer()) { auto timestamp = ioContext.now(); - tracer.addLog(ioContext.getInvocationSpanContext(), timestamp, level, message()); + tracer.addLog(ioContext.getInvocationSpanContext(), timestamp, level, message(), + kj::mv(capturedError)); } } From 29b518208f5fa0bb55ef5fe89394e3b0096cf387 Mon Sep 17 00:00:00 2001 From: Ibrahim Khajanchi Date: Mon, 15 Jun 2026 16:15:59 -0400 Subject: [PATCH 003/101] js API surface, ts types --- src/workerd/api/trace.c++ | 21 ++++++++++++- src/workerd/api/trace.h | 30 +++++++++++++++++-- src/workerd/io/trace-stream.c++ | 10 +++++++ types/defines/trace.d.ts | 11 +++++++ .../experimental/index.d.ts | 18 +++++++++++ .../generated-snapshot/experimental/index.ts | 18 +++++++++++ types/generated-snapshot/latest/index.d.ts | 18 +++++++++++ types/generated-snapshot/latest/index.ts | 18 +++++++++++ 8 files changed, 141 insertions(+), 3 deletions(-) diff --git a/src/workerd/api/trace.c++ b/src/workerd/api/trace.c++ index 14fc0fc0e34..13f13d109b4 100644 --- a/src/workerd/api/trace.c++ +++ b/src/workerd/api/trace.c++ @@ -601,10 +601,22 @@ bool TraceItem::HibernatableWebSocketEventInfo::Close::getWasClean() { return eventInfo.wasClean; } +TraceLogErrorInfo::TraceLogErrorInfo(const tracing::ErrorInfo& info) + : name(kj::str(info.name)), + message(kj::str(info.message)), + stack(mapCopyString(info.stack)) {} + +TraceLogErrorInfo::TraceLogErrorInfo(const TraceLogErrorInfo& other) + : name(kj::str(other.name)), + message(kj::str(other.message)), + stack(other.stack.map([](const kj::String& s) { return kj::str(s); })) {} + TraceLog::TraceLog(jsg::Lock& js, const Trace& trace, const tracing::Log& log) : timestamp(getTraceLogTimestamp(log)), level(getTraceLogLevel(log)), - message(getTraceLogMessage(js, log)) {} + message(getTraceLogMessage(js, log)), + errorInfo(log.errorInfo.map( + [](const tracing::ErrorInfo& info) { return TraceLogErrorInfo(info); })) {} double TraceLog::getTimestamp() { return timestamp; @@ -618,6 +630,13 @@ jsg::V8Ref TraceLog::getMessage(jsg::Lock& js) { return message.addRef(js); } +jsg::Optional TraceLog::getErrorInfo() { + KJ_IF_SOME(info, errorInfo) { + return TraceLogErrorInfo(info); + } + return kj::none; +} + TraceException::TraceException(const Trace& trace, const tracing::Exception& exception) : timestamp(getTraceExceptionTimestamp(exception)), name(kj::str(exception.name)), diff --git a/src/workerd/api/trace.h b/src/workerd/api/trace.h index 97f4594f300..5833e1b22b9 100644 --- a/src/workerd/api/trace.h +++ b/src/workerd/api/trace.h @@ -621,6 +621,27 @@ class TraceDiagnosticChannelEvent final: public jsg::Object { kj::Array message; }; +// Structured Error fields surfaced on a TraceLog whose originating console.* call had +// a native Error among its arguments. Mirrors the shape of TraceException, but is +// attached to a log entry instead of being a standalone trace item. Absent on logs +// whose arguments did not include any native Error. +struct TraceLogErrorInfo { + explicit TraceLogErrorInfo(const tracing::ErrorInfo& info); + TraceLogErrorInfo(const TraceLogErrorInfo&); + + kj::String name; + kj::String message; + jsg::Optional stack; + + JSG_STRUCT(name, message, stack); + + JSG_MEMORY_INFO(TraceLogErrorInfo) { + tracker.trackField("name", name); + tracker.trackField("message", message); + tracker.trackField("stack", stack); + } +}; + class TraceLog final: public jsg::Object { public: TraceLog(jsg::Lock& js, const Trace& trace, const tracing::Log& log); @@ -628,21 +649,25 @@ class TraceLog final: public jsg::Object { double getTimestamp(); kj::StringPtr getLevel(); jsg::V8Ref getMessage(jsg::Lock& js); + jsg::Optional getErrorInfo(); JSG_RESOURCE_TYPE(TraceLog) { JSG_LAZY_READONLY_INSTANCE_PROPERTY(timestamp, getTimestamp); JSG_LAZY_READONLY_INSTANCE_PROPERTY(level, getLevel); JSG_LAZY_READONLY_INSTANCE_PROPERTY(message, getMessage); + JSG_LAZY_READONLY_INSTANCE_PROPERTY(errorInfo, getErrorInfo); } void visitForMemoryInfo(jsg::MemoryTracker& tracker) const { tracker.trackField("message", message); + tracker.trackField("errorInfo", errorInfo); } private: double timestamp; kj::LiteralStringConst level; jsg::V8Ref message; + kj::Maybe errorInfo; void visitForGc(jsg::GcVisitor& visitor) { visitor.visit(message); @@ -756,8 +781,9 @@ class TraceCustomEvent final: public WorkerInterface::CustomEvent { api::TraceItem::HibernatableWebSocketEventInfo, \ api::TraceItem::HibernatableWebSocketEventInfo::Message, \ api::TraceItem::HibernatableWebSocketEventInfo::Close, \ - api::TraceItem::HibernatableWebSocketEventInfo::Error, api::TraceLog, api::TraceException, \ - api::TraceDiagnosticChannelEvent, api::TraceMetrics, api::UnsafeTraceMetrics + api::TraceItem::HibernatableWebSocketEventInfo::Error, api::TraceLog, \ + api::TraceLogErrorInfo, api::TraceException, api::TraceDiagnosticChannelEvent, \ + api::TraceMetrics, api::UnsafeTraceMetrics // The list of trace.h types that are added to worker.c++'s JSG_DECLARE_ISOLATE_TYPE } // namespace workerd::api diff --git a/src/workerd/io/trace-stream.c++ b/src/workerd/io/trace-stream.c++ index e0ba5c56e7e..94d4eee4243 100644 --- a/src/workerd/io/trace-stream.c++ +++ b/src/workerd/io/trace-stream.c++ @@ -39,6 +39,7 @@ namespace { V(EMAIL, "email") \ V(ENTRYPOINT, "entrypoint") \ V(ERROR, "error") \ + V(ERRORINFO, "errorInfo") \ V(EVENT, "event") \ V(EXCEEDEDCPU, "exceededCpu") \ V(EXCEEDEDMEMORY, "exceededMemory") \ @@ -518,6 +519,15 @@ jsg::JsValue ToJs(jsg::Lock& js, const Log& log, StringCache& cache) { obj.set(js, LEVEL_STR, ToJs(js, log.logLevel, cache)); // TODO(o11y): Check that we are always returning an object here obj.set(js, MESSAGE_STR, jsg::JsValue(js.parseJson(log.message).getHandle(js))); + KJ_IF_SOME(info, log.errorInfo) { + auto errObj = js.obj(); + errObj.set(js, NAME_STR, js.str(info.name)); + errObj.set(js, MESSAGE_STR, js.str(info.message)); + KJ_IF_SOME(stack, info.stack) { + errObj.set(js, STACK_STR, js.str(stack)); + } + obj.set(js, ERRORINFO_STR, errObj); + } return obj; } diff --git a/types/defines/trace.d.ts b/types/defines/trace.d.ts index ebdda871503..3cfc21572f8 100644 --- a/types/defines/trace.d.ts +++ b/types/defines/trace.d.ts @@ -150,6 +150,17 @@ interface Log { readonly type: "log"; readonly level: "debug" | "error" | "info" | "log" | "warn"; readonly message: object; + /** + * Structured Error fields surfaced when a `console.*` argument was a native Error. + * Absent for log entries whose arguments did not include any native Error. + */ + readonly errorInfo?: TailStreamErrorInfo; +} + +interface TailStreamErrorInfo { + readonly name: string; + readonly message: string; + readonly stack?: string; } interface DroppedEventsDiagnostic { diff --git a/types/generated-snapshot/experimental/index.d.ts b/types/generated-snapshot/experimental/index.d.ts index 19fdbc03bb5..54fd0d8d93e 100755 --- a/types/generated-snapshot/experimental/index.d.ts +++ b/types/generated-snapshot/experimental/index.d.ts @@ -678,6 +678,8 @@ type DurableObjectLocationHint = | "weur" | "eeur" | "apac" + | "apac-ne" + | "apac-se" | "oc" | "afr" | "me"; @@ -3410,6 +3412,12 @@ interface TraceLog { readonly timestamp: number; readonly level: string; readonly message: any; + readonly errorInfo?: TraceLogErrorInfo; +} +interface TraceLogErrorInfo { + name: string; + message: string; + stack?: string; } interface TraceException { readonly timestamp: number; @@ -15490,6 +15498,16 @@ declare namespace TailStream { readonly type: "log"; readonly level: "debug" | "error" | "info" | "log" | "warn"; readonly message: object; + /** + * Structured Error fields surfaced when a `console.*` argument was a native Error. + * Absent for log entries whose arguments did not include any native Error. + */ + readonly errorInfo?: TailStreamErrorInfo; + } + interface TailStreamErrorInfo { + readonly name: string; + readonly message: string; + readonly stack?: string; } interface DroppedEventsDiagnostic { readonly diagnosticsType: "droppedEvents"; diff --git a/types/generated-snapshot/experimental/index.ts b/types/generated-snapshot/experimental/index.ts index 3b99f046bf2..169a5b50444 100755 --- a/types/generated-snapshot/experimental/index.ts +++ b/types/generated-snapshot/experimental/index.ts @@ -680,6 +680,8 @@ export type DurableObjectLocationHint = | "weur" | "eeur" | "apac" + | "apac-ne" + | "apac-se" | "oc" | "afr" | "me"; @@ -3416,6 +3418,12 @@ export interface TraceLog { readonly timestamp: number; readonly level: string; readonly message: any; + readonly errorInfo?: TraceLogErrorInfo; +} +export interface TraceLogErrorInfo { + name: string; + message: string; + stack?: string; } export interface TraceException { readonly timestamp: number; @@ -15451,6 +15459,16 @@ export declare namespace TailStream { readonly type: "log"; readonly level: "debug" | "error" | "info" | "log" | "warn"; readonly message: object; + /** + * Structured Error fields surfaced when a `console.*` argument was a native Error. + * Absent for log entries whose arguments did not include any native Error. + */ + readonly errorInfo?: TailStreamErrorInfo; + } + interface TailStreamErrorInfo { + readonly name: string; + readonly message: string; + readonly stack?: string; } interface DroppedEventsDiagnostic { readonly diagnosticsType: "droppedEvents"; diff --git a/types/generated-snapshot/latest/index.d.ts b/types/generated-snapshot/latest/index.d.ts index 9450123248b..1d21976f9f9 100755 --- a/types/generated-snapshot/latest/index.d.ts +++ b/types/generated-snapshot/latest/index.d.ts @@ -651,6 +651,8 @@ type DurableObjectLocationHint = | "weur" | "eeur" | "apac" + | "apac-ne" + | "apac-se" | "oc" | "afr" | "me"; @@ -3307,6 +3309,12 @@ interface TraceLog { readonly timestamp: number; readonly level: string; readonly message: any; + readonly errorInfo?: TraceLogErrorInfo; +} +interface TraceLogErrorInfo { + name: string; + message: string; + stack?: string; } interface TraceException { readonly timestamp: number; @@ -14845,6 +14853,16 @@ declare namespace TailStream { readonly type: "log"; readonly level: "debug" | "error" | "info" | "log" | "warn"; readonly message: object; + /** + * Structured Error fields surfaced when a `console.*` argument was a native Error. + * Absent for log entries whose arguments did not include any native Error. + */ + readonly errorInfo?: TailStreamErrorInfo; + } + interface TailStreamErrorInfo { + readonly name: string; + readonly message: string; + readonly stack?: string; } interface DroppedEventsDiagnostic { readonly diagnosticsType: "droppedEvents"; diff --git a/types/generated-snapshot/latest/index.ts b/types/generated-snapshot/latest/index.ts index 8495f88fddd..1ff71f702d1 100755 --- a/types/generated-snapshot/latest/index.ts +++ b/types/generated-snapshot/latest/index.ts @@ -653,6 +653,8 @@ export type DurableObjectLocationHint = | "weur" | "eeur" | "apac" + | "apac-ne" + | "apac-se" | "oc" | "afr" | "me"; @@ -3313,6 +3315,12 @@ export interface TraceLog { readonly timestamp: number; readonly level: string; readonly message: any; + readonly errorInfo?: TraceLogErrorInfo; +} +export interface TraceLogErrorInfo { + name: string; + message: string; + stack?: string; } export interface TraceException { readonly timestamp: number; @@ -14806,6 +14814,16 @@ export declare namespace TailStream { readonly type: "log"; readonly level: "debug" | "error" | "info" | "log" | "warn"; readonly message: object; + /** + * Structured Error fields surfaced when a `console.*` argument was a native Error. + * Absent for log entries whose arguments did not include any native Error. + */ + readonly errorInfo?: TailStreamErrorInfo; + } + interface TailStreamErrorInfo { + readonly name: string; + readonly message: string; + readonly stack?: string; } interface DroppedEventsDiagnostic { readonly diagnosticsType: "droppedEvents"; From e2ad1774658b029c581818dc8738a57929eab5f6 Mon Sep 17 00:00:00 2001 From: Ibrahim Khajanchi Date: Mon, 15 Jun 2026 19:28:18 -0400 Subject: [PATCH 004/101] console.* can have n number of arguments --- src/workerd/api/tests/BUILD.bazel | 13 ++ src/workerd/api/tests/errorinfo-stw-test.js | 85 ++++++++++++ .../api/tests/errorinfo-stw-test.wd-test | 20 +++ src/workerd/api/tests/errorinfo-tail-test.js | 125 ++++++++++++++++++ .../api/tests/errorinfo-tail-test.wd-test | 21 +++ src/workerd/api/trace.c++ | 35 ++++- src/workerd/api/trace.h | 9 +- src/workerd/io/trace-stream.c++ | 25 ++-- src/workerd/io/trace.c++ | 46 +++++-- src/workerd/io/trace.h | 19 ++- src/workerd/io/tracer.c++ | 5 +- src/workerd/io/tracer.h | 12 +- src/workerd/io/worker-interface.capnp | 23 +++- src/workerd/io/worker.c++ | 42 +++--- types/defines/trace.d.ts | 8 +- .../experimental/index.d.ts | 10 +- .../generated-snapshot/experimental/index.ts | 10 +- types/generated-snapshot/latest/index.d.ts | 10 +- types/generated-snapshot/latest/index.ts | 10 +- 19 files changed, 451 insertions(+), 77 deletions(-) create mode 100644 src/workerd/api/tests/errorinfo-stw-test.js create mode 100644 src/workerd/api/tests/errorinfo-stw-test.wd-test create mode 100644 src/workerd/api/tests/errorinfo-tail-test.js create mode 100644 src/workerd/api/tests/errorinfo-tail-test.wd-test diff --git a/src/workerd/api/tests/BUILD.bazel b/src/workerd/api/tests/BUILD.bazel index e6eb69a6aa2..5acaa0a3852 100644 --- a/src/workerd/api/tests/BUILD.bazel +++ b/src/workerd/api/tests/BUILD.bazel @@ -190,6 +190,19 @@ wd_test( ], ) +# Tests for WO-1390: structured errorInfo on TraceLog entries from console.*(Error, ...). +# Covers both batched tail (`tail(events)`) and streaming tail (`tailStream`) surfaces. +wd_test( + src = "errorinfo-tail-test.wd-test", + data = ["errorinfo-tail-test.js"], +) + +wd_test( + src = "errorinfo-stw-test.wd-test", + args = ["--experimental"], + data = ["errorinfo-stw-test.js"], +) + wd_test( src = "analytics-engine-test.wd-test", args = ["--experimental"], diff --git a/src/workerd/api/tests/errorinfo-stw-test.js b/src/workerd/api/tests/errorinfo-stw-test.js new file mode 100644 index 00000000000..f2ce77e193e --- /dev/null +++ b/src/workerd/api/tests/errorinfo-stw-test.js @@ -0,0 +1,85 @@ +// Copyright (c) 2026 Cloudflare, Inc. +// Licensed under the Apache 2.0 license found in the LICENSE file or at: +// https://opensource.org/licenses/Apache-2.0 +// +// Tests that streaming tail workers (`tailStream`) receive a positional +// `errorInfo` array on log events whose originating console.* call had at +// least one native Error among its arguments. See WO-1390. +import * as assert from 'node:assert'; + +const capturedLogEvents = []; + +export default { + async fetch(request, env) { + // Same matrix as the batched tail test for parity. + console.log('plain string'); + console.error(new TypeError('boom')); + console.warn('context:', new RangeError('out of range')); + console.error({ name: 'FakeError', message: 'duck', stack: 'fake stack' }); + console.error( + 'before:', + new Error('first'), + { plain: 'obj' }, + new TypeError('second') + ); + return new Response('ok'); + }, + + tailStream(onsetEvent, env) { + return (event) => { + if (event.event.type === 'log') { + capturedLogEvents.push(event.event); + } + }; + }, +}; + +export const test = { + async test(ctrl, env) { + capturedLogEvents.length = 0; + + const response = await env.SERVICE.fetch('http://example.com/'); + assert.strictEqual(await response.text(), 'ok'); + await scheduler.wait(100); + + assert.strictEqual( + capturedLogEvents.length, + 5, + `expected 5 streamed log events, got ${capturedLogEvents.length}` + ); + + const [plain, typeErr, ctxRange, fakeErr, mixed] = capturedLogEvents; + + assert.strictEqual(plain.errorInfo, undefined); + + assert.ok(Array.isArray(typeErr.errorInfo)); + assert.strictEqual(typeErr.errorInfo.length, 1); + assert.strictEqual(typeErr.errorInfo[0].name, 'TypeError'); + assert.strictEqual(typeErr.errorInfo[0].message, 'boom'); + assert.ok( + typeof typeErr.errorInfo[0].stack === 'string' && + typeErr.errorInfo[0].stack.includes('TypeError'), + `stack should be a string containing "TypeError", got: ${typeErr.errorInfo[0].stack}` + ); + + assert.ok(Array.isArray(ctxRange.errorInfo)); + assert.strictEqual(ctxRange.errorInfo.length, 2); + assert.strictEqual(ctxRange.errorInfo[0], null); + assert.ok(ctxRange.errorInfo[1]); + assert.strictEqual(ctxRange.errorInfo[1].name, 'RangeError'); + assert.strictEqual(ctxRange.errorInfo[1].message, 'out of range'); + + assert.strictEqual(fakeErr.errorInfo, undefined); + + assert.ok(Array.isArray(mixed.errorInfo)); + assert.strictEqual(mixed.errorInfo.length, 4); + assert.strictEqual(mixed.errorInfo[0], null); + assert.ok(mixed.errorInfo[1]); + assert.strictEqual(mixed.errorInfo[1].name, 'Error'); + assert.strictEqual(mixed.errorInfo[1].message, 'first'); + assert.strictEqual(mixed.errorInfo[2], null); + assert.ok(mixed.errorInfo[3]); + assert.strictEqual(mixed.errorInfo[3].name, 'TypeError'); + assert.strictEqual(mixed.errorInfo[3].message, 'second'); + }, +}; diff --git a/src/workerd/api/tests/errorinfo-stw-test.wd-test b/src/workerd/api/tests/errorinfo-stw-test.wd-test new file mode 100644 index 00000000000..8542fe12f1e --- /dev/null +++ b/src/workerd/api/tests/errorinfo-stw-test.wd-test @@ -0,0 +1,20 @@ +using Workerd = import "/workerd/workerd.capnp"; + +const unitTests :Workerd.Config = ( + services = [ + (name = "errorinfo-stw", worker = .errorinfoStwWorker), + ], +); + +const errorinfoStwWorker :Workerd.Worker = ( + modules = [ + (name = "worker", esModule = embed "errorinfo-stw-test.js") + ], + bindings = [ + (name = "SERVICE", service = "errorinfo-stw"), + ], + compatibilityFlags = ["nodejs_compat", "experimental"], + # Self-streaming-tail: the worker streams its own events into its own + # `tailStream` handler, so we can drive producer + consumer from one module. + streamingTails = ["errorinfo-stw"], +); diff --git a/src/workerd/api/tests/errorinfo-tail-test.js b/src/workerd/api/tests/errorinfo-tail-test.js new file mode 100644 index 00000000000..170c71a9d9c --- /dev/null +++ b/src/workerd/api/tests/errorinfo-tail-test.js @@ -0,0 +1,125 @@ +// Copyright (c) 2026 Cloudflare, Inc. +// Licensed under the Apache 2.0 license found in the LICENSE file or at: +// https://opensource.org/licenses/Apache-2.0 +// +// Tests that batched tail workers (`tail(events)`) receive a positional +// `errorInfo` array on TraceLog entries whose originating console.* call had at +// least one native Error among its arguments. See WO-1390. +// +// Shape: `log.errorInfo` is either undefined (no Errors in the args) or an +// array whose length matches the argument count, where each slot is either +// `{name, message, stack?}` (for native Error args) or `null` (for everything +// else). +import * as assert from 'node:assert'; + +// Captured by the tail handler so the `test` block can assert on it after the +// fetch invocation completes. +let capturedLogs = null; + +export default { + async fetch(request, env) { + // 1. Plain string, no Errors anywhere. + console.log('plain string'); + + // 2. Single native Error as the only arg. + console.error(new TypeError('boom')); + + // 3. Error in second position, string in first. + console.warn('context:', new RangeError('out of range')); + + // 4. Duck-typed object that LOOKS like an Error but isn't. + console.error({ name: 'FakeError', message: 'duck', stack: 'fake stack' }); + + // 5. Multiple Errors interleaved with strings/objects: both Errors must + // appear in their respective positions; non-Error slots must be null. + console.error( + 'before:', + new Error('first'), + { plain: 'obj' }, + new TypeError('second') + ); + + return new Response('ok'); + }, + + tail(events) { + const logs = []; + for (const event of events) { + if (event.logs) { + for (const log of event.logs) { + logs.push(log); + } + } + } + capturedLogs = logs; + }, +}; + +export const test = { + async test(ctrl, env) { + const response = await env.SERVICE.fetch('http://example.com/'); + assert.strictEqual(await response.text(), 'ok'); + + // The tail handler runs after the fetch invocation completes. + await scheduler.wait(100); + + assert.ok(capturedLogs, 'tail handler was never called'); + assert.strictEqual( + capturedLogs.length, + 5, + `expected 5 log entries, got ${capturedLogs.length}` + ); + + const [plain, typeErr, ctxRange, fakeErr, mixed] = capturedLogs; + + // 1. No Errors anywhere: field absent entirely. + assert.strictEqual( + plain.errorInfo, + undefined, + 'plain string log should have no errorInfo' + ); + + // 2. Single Error in slot 0: array of length 1. + assert.ok(Array.isArray(typeErr.errorInfo)); + assert.strictEqual(typeErr.errorInfo.length, 1); + assert.strictEqual(typeErr.errorInfo[0].name, 'TypeError'); + assert.strictEqual(typeErr.errorInfo[0].message, 'boom'); + assert.ok( + typeof typeErr.errorInfo[0].stack === 'string' && + typeErr.errorInfo[0].stack.includes('TypeError'), + `stack should be a string containing "TypeError", got: ${typeErr.errorInfo[0].stack}` + ); + + // 3. Two-arg call with Error at index 1: positional shape preserved. + assert.ok(Array.isArray(ctxRange.errorInfo)); + assert.strictEqual(ctxRange.errorInfo.length, 2); + assert.strictEqual(ctxRange.errorInfo[0], null, 'string slot must be null'); + assert.ok(ctxRange.errorInfo[1]); + assert.strictEqual(ctxRange.errorInfo[1].name, 'RangeError'); + assert.strictEqual(ctxRange.errorInfo[1].message, 'out of range'); + + // 4. Duck-typed plain object: NOT treated as Error. Since the duck object + // is the only arg AND not an Error, errorInfo should be absent. + assert.strictEqual( + fakeErr.errorInfo, + undefined, + 'plain object with Error-like fields should NOT produce errorInfo' + ); + + // 5. Four args: ['before:', Error('first'), {plain:'obj'}, TypeError('second')] + // → errorInfo = [null, {first}, null, {second}] + assert.ok(Array.isArray(mixed.errorInfo)); + assert.strictEqual(mixed.errorInfo.length, 4); + assert.strictEqual(mixed.errorInfo[0], null); + assert.ok(mixed.errorInfo[1]); + assert.strictEqual(mixed.errorInfo[1].name, 'Error'); + assert.strictEqual(mixed.errorInfo[1].message, 'first'); + assert.strictEqual(mixed.errorInfo[2], null); + assert.ok(mixed.errorInfo[3]); + assert.strictEqual(mixed.errorInfo[3].name, 'TypeError'); + assert.strictEqual(mixed.errorInfo[3].message, 'second'); + + // Backwards compat: existing message field still present and an object. + assert.strictEqual(typeof typeErr.message, 'object'); + }, +}; diff --git a/src/workerd/api/tests/errorinfo-tail-test.wd-test b/src/workerd/api/tests/errorinfo-tail-test.wd-test new file mode 100644 index 00000000000..2891d55504d --- /dev/null +++ b/src/workerd/api/tests/errorinfo-tail-test.wd-test @@ -0,0 +1,21 @@ +using Workerd = import "/workerd/workerd.capnp"; + +const unitTests :Workerd.Config = ( + services = [ + (name = "errorinfo-tail", worker = .errorinfoTailWorker), + ], +); + +const errorinfoTailWorker :Workerd.Worker = ( + modules = [ + (name = "worker", esModule = embed "errorinfo-tail-test.js") + ], + bindings = [ + (name = "SERVICE", service = "errorinfo-tail"), + ], + compatibilityFlags = ["nodejs_compat"], + # Self-tail: the worker is registered as its own batched tail worker so the + # `tail` handler in the same module receives the trace events emitted by its + # own `fetch` handler. This avoids needing a separate tail worker module. + tails = ["errorinfo-tail"], +); diff --git a/src/workerd/api/trace.c++ b/src/workerd/api/trace.c++ index 13f13d109b4..bfd73e1bdf2 100644 --- a/src/workerd/api/trace.c++ +++ b/src/workerd/api/trace.c++ @@ -611,12 +611,29 @@ TraceLogErrorInfo::TraceLogErrorInfo(const TraceLogErrorInfo& other) message(kj::str(other.message)), stack(other.stack.map([](const kj::String& s) { return kj::str(s); })) {} +namespace { +kj::Maybe>> convertLogErrorInfo( + const tracing::LogErrorInfo& src) { + KJ_IF_SOME(slots, src) { + // Each slot starts out kj::none (default-constructed by heapArray); we only + // populate slots that hold an ErrorInfo in the source. + auto out = kj::heapArray>(slots.size()); + for (auto i: kj::zeroTo(slots.size())) { + KJ_IF_SOME(info, slots[i]) { + out[i] = TraceLogErrorInfo(info); + } + } + return kj::mv(out); + } + return kj::none; +} +} // namespace + TraceLog::TraceLog(jsg::Lock& js, const Trace& trace, const tracing::Log& log) : timestamp(getTraceLogTimestamp(log)), level(getTraceLogLevel(log)), message(getTraceLogMessage(js, log)), - errorInfo(log.errorInfo.map( - [](const tracing::ErrorInfo& info) { return TraceLogErrorInfo(info); })) {} + errorInfo(convertLogErrorInfo(log.errorInfo)) {} double TraceLog::getTimestamp() { return timestamp; @@ -630,9 +647,17 @@ jsg::V8Ref TraceLog::getMessage(jsg::Lock& js) { return message.addRef(js); } -jsg::Optional TraceLog::getErrorInfo() { - KJ_IF_SOME(info, errorInfo) { - return TraceLogErrorInfo(info); +jsg::Optional>> TraceLog::getErrorInfo() { + KJ_IF_SOME(slots, errorInfo) { + // Each slot starts out kj::none (default-constructed by heapArray); we only + // populate slots that hold a TraceLogErrorInfo in the source. + auto out = kj::heapArray>(slots.size()); + for (auto i: kj::zeroTo(slots.size())) { + KJ_IF_SOME(info, slots[i]) { + out[i] = TraceLogErrorInfo(info); + } + } + return kj::mv(out); } return kj::none; } diff --git a/src/workerd/api/trace.h b/src/workerd/api/trace.h index 5833e1b22b9..75bdffc0137 100644 --- a/src/workerd/api/trace.h +++ b/src/workerd/api/trace.h @@ -649,7 +649,11 @@ class TraceLog final: public jsg::Object { double getTimestamp(); kj::StringPtr getLevel(); jsg::V8Ref getMessage(jsg::Lock& js); - jsg::Optional getErrorInfo(); + // Returns the per-argument errorInfo list, or kj::none / undefined when none of + // the originating console call's arguments was a native Error. When present, + // the array length matches the argument count; slots whose argument was not an + // Error are JS `null`. + jsg::Optional>> getErrorInfo(); JSG_RESOURCE_TYPE(TraceLog) { JSG_LAZY_READONLY_INSTANCE_PROPERTY(timestamp, getTimestamp); @@ -660,14 +664,13 @@ class TraceLog final: public jsg::Object { void visitForMemoryInfo(jsg::MemoryTracker& tracker) const { tracker.trackField("message", message); - tracker.trackField("errorInfo", errorInfo); } private: double timestamp; kj::LiteralStringConst level; jsg::V8Ref message; - kj::Maybe errorInfo; + kj::Maybe>> errorInfo; void visitForGc(jsg::GcVisitor& visitor) { visitor.visit(message); diff --git a/src/workerd/io/trace-stream.c++ b/src/workerd/io/trace-stream.c++ index 94d4eee4243..7c7c4cec52d 100644 --- a/src/workerd/io/trace-stream.c++ +++ b/src/workerd/io/trace-stream.c++ @@ -519,14 +519,23 @@ jsg::JsValue ToJs(jsg::Lock& js, const Log& log, StringCache& cache) { obj.set(js, LEVEL_STR, ToJs(js, log.logLevel, cache)); // TODO(o11y): Check that we are always returning an object here obj.set(js, MESSAGE_STR, jsg::JsValue(js.parseJson(log.message).getHandle(js))); - KJ_IF_SOME(info, log.errorInfo) { - auto errObj = js.obj(); - errObj.set(js, NAME_STR, js.str(info.name)); - errObj.set(js, MESSAGE_STR, js.str(info.message)); - KJ_IF_SOME(stack, info.stack) { - errObj.set(js, STACK_STR, js.str(stack)); - } - obj.set(js, ERRORINFO_STR, errObj); + KJ_IF_SOME(slots, log.errorInfo) { + // Emit as a positional JS array: each slot is either { name, message, stack? } + // for arguments that were native Errors, or `null` for non-Error arguments. + auto arr = js.arr(slots.asPtr(), + [&cache](jsg::Lock& js, const kj::Maybe& slot) -> jsg::JsValue { + KJ_IF_SOME(info, slot) { + auto errObj = js.obj(); + errObj.set(js, NAME_STR, cache.get(js, info.name)); + errObj.set(js, MESSAGE_STR, js.str(info.message)); + KJ_IF_SOME(stack, info.stack) { + errObj.set(js, STACK_STR, js.str(stack)); + } + return errObj; + } + return js.null(); + }); + obj.set(js, ERRORINFO_STR, arr); } return obj; } diff --git a/src/workerd/io/trace.c++ b/src/workerd/io/trace.c++ index 8b7a7e4be67..4942ddd1968 100644 --- a/src/workerd/io/trace.c++ +++ b/src/workerd/io/trace.c++ @@ -759,10 +759,22 @@ ErrorInfo ErrorInfo::clone() const { stack.map([](const kj::String& s) { return kj::str(s); })); } -Log::Log(kj::Date timestamp, - LogLevel logLevel, - kj::String message, - kj::Maybe errorInfo) +LogErrorInfo cloneLogErrorInfo(const LogErrorInfo& src) { + KJ_IF_SOME(slots, src) { + // Each slot starts out kj::none (default-constructed by heapArray); we only + // populate slots that hold an Error in the source. + auto out = kj::heapArray>(slots.size()); + for (auto i: kj::zeroTo(slots.size())) { + KJ_IF_SOME(info, slots[i]) { + out[i] = info.clone(); + } + } + return out; + } + return kj::none; +} + +Log::Log(kj::Date timestamp, LogLevel logLevel, kj::String message, LogErrorInfo errorInfo) : timestamp(timestamp), logLevel(logLevel), message(kj::mv(message)), @@ -772,14 +784,21 @@ void Log::copyTo(rpc::Trace::Log::Builder builder) const { builder.setTimestampNs((timestamp - kj::UNIX_EPOCH) / kj::NANOSECONDS); builder.setLogLevel(logLevel); builder.setMessage(message); - KJ_IF_SOME(info, errorInfo) { - info.copyTo(builder.initErrorInfo()); + KJ_IF_SOME(slots, errorInfo) { + auto listBuilder = builder.initErrorInfo(slots.size()); + for (auto i: kj::zeroTo(slots.size())) { + auto slotBuilder = listBuilder[i]; + KJ_IF_SOME(info, slots[i]) { + info.copyTo(slotBuilder.initInfo()); + } else { + slotBuilder.setNone(); + } + } } } Log Log::clone() const { - return Log(timestamp, logLevel, kj::str(message), - errorInfo.map([](const ErrorInfo& info) { return info.clone(); })); + return Log(timestamp, logLevel, kj::str(message), cloneLogErrorInfo(errorInfo)); } Exception::Exception( @@ -794,7 +813,16 @@ Log::Log(rpc::Trace::Log::Reader reader) logLevel(reader.getLogLevel()), message(kj::str(reader.getMessage())) { if (reader.hasErrorInfo()) { - errorInfo = ErrorInfo(reader.getErrorInfo()); + auto listReader = reader.getErrorInfo(); + auto slots = kj::heapArray>(listReader.size()); + for (auto i: kj::zeroTo(listReader.size())) { + auto slotReader = listReader[i]; + if (slotReader.which() == rpc::Trace::ErrorInfoSlot::INFO) { + slots[i] = ErrorInfo(slotReader.getInfo()); + } + // else: stays kj::none + } + errorInfo = kj::mv(slots); } } diff --git a/src/workerd/io/trace.h b/src/workerd/io/trace.h index dd7c08b8973..626b15a4ecd 100644 --- a/src/workerd/io/trace.h +++ b/src/workerd/io/trace.h @@ -626,12 +626,23 @@ struct ErrorInfo final { ErrorInfo clone() const; }; +// Per-argument errorInfo for a Log. The outer kj::Maybe is `kj::none` when none +// of the originating console call's arguments was a native Error (the dominant +// case, kept cheap on the wire). When present, the inner array's length matches +// the argument count; slot `i` is `kj::some(ErrorInfo)` if `info[i]` was a native +// Error, `kj::none` otherwise. +using LogErrorInfo = kj::Maybe>>; + +// Helper: deep-copy a LogErrorInfo. Needed because kj::Array is move-only and +// ErrorInfo contains non-copyable strings. +LogErrorInfo cloneLogErrorInfo(const LogErrorInfo& src); + // Describes a log event struct Log final { explicit Log(kj::Date timestamp, LogLevel logLevel, kj::String message, - kj::Maybe errorInfo = kj::none); + LogErrorInfo errorInfo = kj::none); Log(rpc::Trace::Log::Reader reader); Log(Log&&) noexcept = default; KJ_DISALLOW_COPY(Log); @@ -644,10 +655,8 @@ struct Log final { // TODO(soon): Just string for now. Eventually, capture serialized JS objects. kj::String message; - // Populated when at least one argument to the originating console call was a - // native Error. Carries the structured {name, message, stack} fields so that tail - // workers can surface them without depending on lossy stringification of `message`. - kj::Maybe errorInfo; + // Per-argument structured Error fields. See LogErrorInfo above for semantics. + LogErrorInfo errorInfo; void copyTo(rpc::Trace::Log::Builder builder) const; Log clone() const; diff --git a/src/workerd/io/tracer.c++ b/src/workerd/io/tracer.c++ index a4a7bf145ac..b9ba766c96c 100644 --- a/src/workerd/io/tracer.c++ +++ b/src/workerd/io/tracer.c++ @@ -123,7 +123,7 @@ void WorkerTracer::addLog(const tracing::InvocationSpanContext& context, kj::Date timestamp, LogLevel logLevel, kj::String message, - kj::Maybe errorInfo) { + tracing::LogErrorInfo errorInfo) { if (pipelineLogLevel == PipelineLogLevel::NONE) { return; } @@ -135,8 +135,7 @@ void WorkerTracer::addLog(const tracing::InvocationSpanContext& context, // If message is too big on its own, truncate it. size_t messageSize = kj::min(message.size(), MAX_TRACE_BYTES); // Clone errorInfo for the STW path because the batched-tail path below also needs it. - auto streamErrorInfo = errorInfo.map( - [](const tracing::ErrorInfo& info) { return info.clone(); }); + auto streamErrorInfo = tracing::cloneLogErrorInfo(errorInfo); writer->report(context, {tracing::Log(timestamp, logLevel, kj::str(message.first(messageSize)), kj::mv(streamErrorInfo))}, diff --git a/src/workerd/io/tracer.h b/src/workerd/io/tracer.h index 4fc3b411011..6070deae816 100644 --- a/src/workerd/io/tracer.h +++ b/src/workerd/io/tracer.h @@ -38,14 +38,16 @@ class BaseTracer: public kj::Refcounted { // Adds log line to trace. For Spectre, timestamp should only be as accurate as JS Date.now(). // - // `errorInfo` carries structured Error fields extracted from any Error argument passed to - // the originating console.* call (e.g. `console.error(new Error("x"))`). Pass `kj::none` - // for non-console internal warning paths, or when no argument was a native Error. + // `errorInfo` carries per-argument structured Error fields for the originating console.* + // call. See `tracing::LogErrorInfo` for the semantics: the outer kj::Maybe is none when + // none of the arguments was a native Error; otherwise the inner array's length matches + // the argument count, with kj::none in slots whose argument was not an Error. Pass + // `kj::none` for non-console internal warning paths. virtual void addLog(const tracing::InvocationSpanContext& context, kj::Date timestamp, LogLevel logLevel, kj::String message, - kj::Maybe errorInfo = kj::none) = 0; + tracing::LogErrorInfo errorInfo = kj::none) = 0; // Add a span open event. virtual void addSpanOpen(tracing::SpanId spanId, tracing::SpanId parentSpanId, @@ -152,7 +154,7 @@ class WorkerTracer final: public BaseTracer { kj::Date timestamp, LogLevel logLevel, kj::String message, - kj::Maybe errorInfo = kj::none) override; + tracing::LogErrorInfo errorInfo = kj::none) override; void addSpanOpen(tracing::SpanId spanId, tracing::SpanId parentSpanId, kj::ConstString operationName, diff --git a/src/workerd/io/worker-interface.capnp b/src/workerd/io/worker-interface.capnp index cea897f1244..129a7dcec4b 100644 --- a/src/workerd/io/worker-interface.capnp +++ b/src/workerd/io/worker-interface.capnp @@ -82,11 +82,15 @@ struct Trace @0x8e8d911203762d34 { message @2 :Text; - errorInfo @3 :ErrorInfo; - # Structured Error fields extracted from any Error argument passed to the console - # call that produced this log (e.g. `console.error(new Error("x"))`). Absent when - # none of the arguments was a native Error. The stringified form of the arguments - # remains in `message` unchanged for backwards compatibility. + errorInfo @3 :List(ErrorInfoSlot); + # Per-argument structured Error fields for the console call that produced this + # log. The list is positional: slot `i` corresponds to the i-th argument of the + # originating `console.*` call. Slots whose argument was not a native Error are + # encoded as `none`. The field is absent entirely (zero-length list on the wire) + # when none of the arguments was a native Error. + # + # The stringified form of the arguments remains in `message` unchanged for + # backwards compatibility. } struct ErrorInfo { @@ -98,6 +102,15 @@ struct Trace @0x8e8d911203762d34 { stack @2 :Text; } + struct ErrorInfoSlot { + # One slot in the per-argument errorInfo list. `info` is set when the + # corresponding console argument was a native Error; `none` otherwise. + union { + none @0 :Void; + info @1 :ErrorInfo; + } + } + obsolete26 @26 :List(UserSpanData); # spans are unavailable in full trace objects. diff --git a/src/workerd/io/worker.c++ b/src/workerd/io/worker.c++ index b31edb28b09..9b0951aa10b 100644 --- a/src/workerd/io/worker.c++ +++ b/src/workerd/io/worker.c++ @@ -2182,20 +2182,32 @@ void Worker::handleLog(jsg::Lock& js, // here because `message` is called multiple times. v8::TryCatch tryCatch(js.v8Isolate); - // Scan arguments for the first native Error and capture its {name, message, stack} as - // structured ErrorInfo. We do this independently of the stringification below so that - // (a) calling `message()` multiple times doesn't re-extract, and (b) the existing - // stringification behavior is unchanged for backwards compatibility — the structured - // ErrorInfo is purely additive. - kj::Maybe capturedError; - for (auto i: kj::zeroTo(length)) { - if (!tryCatch.CanContinue()) break; - auto arg = info[i]; - if (!arg->IsNativeError()) continue; - js.withinHandleScope([&] { - capturedError = extractErrorInfo(js, arg.As()); - }); - if (capturedError != kj::none) break; + // Scan all arguments for native Errors and build a positional ErrorInfo array: + // slot `i` holds the extracted {name, message, stack} for arg `i` if it was a native + // Error, otherwise kj::none. If no argument was a native Error, the whole array is + // left absent (kj::none) to keep the common case cheap on the wire. + // + // We do this independently of the stringification below so that calling `message()` + // multiple times doesn't re-extract, and so the existing stringification behavior is + // unchanged for backwards compatibility — the structured ErrorInfo is purely additive. + tracing::LogErrorInfo capturedErrors; + { + auto slots = kj::heapArray>(length); + bool anyError = false; + for (auto i: kj::zeroTo(length)) { + if (!tryCatch.CanContinue()) break; + auto arg = info[i]; + if (!arg->IsNativeError()) continue; + js.withinHandleScope([&] { + KJ_IF_SOME(extracted, extractErrorInfo(js, arg.As())) { + slots[i] = kj::mv(extracted); + anyError = true; + } + }); + } + if (anyError) { + capturedErrors = kj::mv(slots); + } } auto message = [&]() { @@ -2274,7 +2286,7 @@ void Worker::handleLog(jsg::Lock& js, KJ_IF_SOME(tracer, ioContext.getWorkerTracer()) { auto timestamp = ioContext.now(); tracer.addLog(ioContext.getInvocationSpanContext(), timestamp, level, message(), - kj::mv(capturedError)); + kj::mv(capturedErrors)); } } diff --git a/types/defines/trace.d.ts b/types/defines/trace.d.ts index 3cfc21572f8..fee4f024b20 100644 --- a/types/defines/trace.d.ts +++ b/types/defines/trace.d.ts @@ -151,10 +151,12 @@ interface Log { readonly level: "debug" | "error" | "info" | "log" | "warn"; readonly message: object; /** - * Structured Error fields surfaced when a `console.*` argument was a native Error. - * Absent for log entries whose arguments did not include any native Error. + * Per-argument structured Error fields for the originating `console.*` call. + * The array is positional: index `i` corresponds to the i-th argument. Indices + * whose argument was not a native Error are `null`. The whole property is + * absent (undefined) when none of the arguments was a native Error. */ - readonly errorInfo?: TailStreamErrorInfo; + readonly errorInfo?: readonly (TailStreamErrorInfo | null)[]; } interface TailStreamErrorInfo { diff --git a/types/generated-snapshot/experimental/index.d.ts b/types/generated-snapshot/experimental/index.d.ts index 54fd0d8d93e..30fbd31c772 100755 --- a/types/generated-snapshot/experimental/index.d.ts +++ b/types/generated-snapshot/experimental/index.d.ts @@ -3412,7 +3412,7 @@ interface TraceLog { readonly timestamp: number; readonly level: string; readonly message: any; - readonly errorInfo?: TraceLogErrorInfo; + readonly errorInfo?: (TraceLogErrorInfo | null)[]; } interface TraceLogErrorInfo { name: string; @@ -15499,10 +15499,12 @@ declare namespace TailStream { readonly level: "debug" | "error" | "info" | "log" | "warn"; readonly message: object; /** - * Structured Error fields surfaced when a `console.*` argument was a native Error. - * Absent for log entries whose arguments did not include any native Error. + * Per-argument structured Error fields for the originating `console.*` call. + * The array is positional: index `i` corresponds to the i-th argument. Indices + * whose argument was not a native Error are `null`. The whole property is + * absent (undefined) when none of the arguments was a native Error. */ - readonly errorInfo?: TailStreamErrorInfo; + readonly errorInfo?: readonly (TailStreamErrorInfo | null)[]; } interface TailStreamErrorInfo { readonly name: string; diff --git a/types/generated-snapshot/experimental/index.ts b/types/generated-snapshot/experimental/index.ts index 169a5b50444..1f4e6f0fc05 100755 --- a/types/generated-snapshot/experimental/index.ts +++ b/types/generated-snapshot/experimental/index.ts @@ -3418,7 +3418,7 @@ export interface TraceLog { readonly timestamp: number; readonly level: string; readonly message: any; - readonly errorInfo?: TraceLogErrorInfo; + readonly errorInfo?: (TraceLogErrorInfo | null)[]; } export interface TraceLogErrorInfo { name: string; @@ -15460,10 +15460,12 @@ export declare namespace TailStream { readonly level: "debug" | "error" | "info" | "log" | "warn"; readonly message: object; /** - * Structured Error fields surfaced when a `console.*` argument was a native Error. - * Absent for log entries whose arguments did not include any native Error. + * Per-argument structured Error fields for the originating `console.*` call. + * The array is positional: index `i` corresponds to the i-th argument. Indices + * whose argument was not a native Error are `null`. The whole property is + * absent (undefined) when none of the arguments was a native Error. */ - readonly errorInfo?: TailStreamErrorInfo; + readonly errorInfo?: readonly (TailStreamErrorInfo | null)[]; } interface TailStreamErrorInfo { readonly name: string; diff --git a/types/generated-snapshot/latest/index.d.ts b/types/generated-snapshot/latest/index.d.ts index 1d21976f9f9..c69208012be 100755 --- a/types/generated-snapshot/latest/index.d.ts +++ b/types/generated-snapshot/latest/index.d.ts @@ -3309,7 +3309,7 @@ interface TraceLog { readonly timestamp: number; readonly level: string; readonly message: any; - readonly errorInfo?: TraceLogErrorInfo; + readonly errorInfo?: (TraceLogErrorInfo | null)[]; } interface TraceLogErrorInfo { name: string; @@ -14854,10 +14854,12 @@ declare namespace TailStream { readonly level: "debug" | "error" | "info" | "log" | "warn"; readonly message: object; /** - * Structured Error fields surfaced when a `console.*` argument was a native Error. - * Absent for log entries whose arguments did not include any native Error. + * Per-argument structured Error fields for the originating `console.*` call. + * The array is positional: index `i` corresponds to the i-th argument. Indices + * whose argument was not a native Error are `null`. The whole property is + * absent (undefined) when none of the arguments was a native Error. */ - readonly errorInfo?: TailStreamErrorInfo; + readonly errorInfo?: readonly (TailStreamErrorInfo | null)[]; } interface TailStreamErrorInfo { readonly name: string; diff --git a/types/generated-snapshot/latest/index.ts b/types/generated-snapshot/latest/index.ts index 1ff71f702d1..4dfce3f6404 100755 --- a/types/generated-snapshot/latest/index.ts +++ b/types/generated-snapshot/latest/index.ts @@ -3315,7 +3315,7 @@ export interface TraceLog { readonly timestamp: number; readonly level: string; readonly message: any; - readonly errorInfo?: TraceLogErrorInfo; + readonly errorInfo?: (TraceLogErrorInfo | null)[]; } export interface TraceLogErrorInfo { name: string; @@ -14815,10 +14815,12 @@ export declare namespace TailStream { readonly level: "debug" | "error" | "info" | "log" | "warn"; readonly message: object; /** - * Structured Error fields surfaced when a `console.*` argument was a native Error. - * Absent for log entries whose arguments did not include any native Error. + * Per-argument structured Error fields for the originating `console.*` call. + * The array is positional: index `i` corresponds to the i-th argument. Indices + * whose argument was not a native Error are `null`. The whole property is + * absent (undefined) when none of the arguments was a native Error. */ - readonly errorInfo?: TailStreamErrorInfo; + readonly errorInfo?: readonly (TailStreamErrorInfo | null)[]; } interface TailStreamErrorInfo { readonly name: string; From aca56f8aac28ea23d2ed6264174d387586c444a4 Mon Sep 17 00:00:00 2001 From: Ibrahim Khajanchi Date: Tue, 16 Jun 2026 13:33:02 -0400 Subject: [PATCH 005/101] include errorInfoSize in trace buget & only allocate slots if an error is found --- src/workerd/io/tracer.c++ | 17 +++++++++++++++-- src/workerd/io/worker.c++ | 6 +++++- 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/src/workerd/io/tracer.c++ b/src/workerd/io/tracer.c++ index b9ba766c96c..e82c648a0c1 100644 --- a/src/workerd/io/tracer.c++ +++ b/src/workerd/io/tracer.c++ @@ -128,6 +128,19 @@ void WorkerTracer::addLog(const tracing::InvocationSpanContext& context, return; } + // Compute the heap size of errorInfo once for both the streaming and buffered paths. + size_t errorInfoSize = 0; + KJ_IF_SOME(infos, errorInfo) { + for (const auto& entry: infos) { + KJ_IF_SOME(info, entry) { + errorInfoSize += info.name.size() + info.message.size(); + KJ_IF_SOME(s, info.stack) { + errorInfoSize += s.size(); + } + } + } + } + // TODO(streaming-tail): Here we add the log to the trace object and the tail stream writer, if // available. If the given worker stage is only tailed by a streaming tail worker, adding the log // to the buffered trace object is not needed; this will be addressed in a future refactor. @@ -139,14 +152,14 @@ void WorkerTracer::addLog(const tracing::InvocationSpanContext& context, writer->report(context, {tracing::Log(timestamp, logLevel, kj::str(message.first(messageSize)), kj::mv(streamErrorInfo))}, - timestamp, messageSize); + timestamp, messageSize + errorInfoSize); } if (trace->exceededLogLimit) { return; } - size_t messageSize = sizeof(tracing::Log) + message.size(); + size_t messageSize = sizeof(tracing::Log) + message.size() + errorInfoSize; if (trace->bytesUsed + messageSize > MAX_TRACE_BYTES) { // We use a JSON encoded array/string to match other console.log() recordings: trace->logs.add(timestamp, LogLevel::WARN, kj::str(logSizeExceeded)); diff --git a/src/workerd/io/worker.c++ b/src/workerd/io/worker.c++ index 9b0951aa10b..85f5308cc83 100644 --- a/src/workerd/io/worker.c++ +++ b/src/workerd/io/worker.c++ @@ -2192,12 +2192,16 @@ void Worker::handleLog(jsg::Lock& js, // unchanged for backwards compatibility — the structured ErrorInfo is purely additive. tracing::LogErrorInfo capturedErrors; { - auto slots = kj::heapArray>(length); + kj::Array> slots; bool anyError = false; for (auto i: kj::zeroTo(length)) { if (!tryCatch.CanContinue()) break; auto arg = info[i]; if (!arg->IsNativeError()) continue; + if (!anyError) { + slots = kj::heapArray>(length); + anyError = true; + } js.withinHandleScope([&] { KJ_IF_SOME(extracted, extractErrorInfo(js, arg.As())) { slots[i] = kj::mv(extracted); From 21b8231cf826ea8f4c9be85118013a46a2cc7c8e Mon Sep 17 00:00:00 2001 From: Ibrahim Khajanchi Date: Wed, 17 Jun 2026 21:04:58 +0000 Subject: [PATCH 006/101] coalesce addExceptionToTrace and console.* errorInfo handling --- src/workerd/api/tests/errorinfo-stw-test.js | 12 +- src/workerd/api/tests/errorinfo-tail-test.js | 12 +- src/workerd/io/worker.c++ | 166 +++++++++---------- 3 files changed, 100 insertions(+), 90 deletions(-) diff --git a/src/workerd/api/tests/errorinfo-stw-test.js b/src/workerd/api/tests/errorinfo-stw-test.js index f2ce77e193e..929690af80f 100644 --- a/src/workerd/api/tests/errorinfo-stw-test.js +++ b/src/workerd/api/tests/errorinfo-stw-test.js @@ -58,8 +58,16 @@ export const test = { assert.strictEqual(typeErr.errorInfo[0].message, 'boom'); assert.ok( typeof typeErr.errorInfo[0].stack === 'string' && - typeErr.errorInfo[0].stack.includes('TypeError'), - `stack should be a string containing "TypeError", got: ${typeErr.errorInfo[0].stack}` + typeErr.errorInfo[0].stack.length > 0, + `stack should be a non-empty string, got: ${typeErr.errorInfo[0].stack}` + ); + assert.ok( + typeErr.errorInfo[0].stack.includes('at '), + `stack should contain call frames, got: ${typeErr.errorInfo[0].stack}` + ); + assert.ok( + !typeErr.errorInfo[0].stack.includes('TypeError'), + `stack should have the "Name: message" prefix stripped, got: ${typeErr.errorInfo[0].stack}` ); assert.ok(Array.isArray(ctxRange.errorInfo)); diff --git a/src/workerd/api/tests/errorinfo-tail-test.js b/src/workerd/api/tests/errorinfo-tail-test.js index 170c71a9d9c..0274112a2f8 100644 --- a/src/workerd/api/tests/errorinfo-tail-test.js +++ b/src/workerd/api/tests/errorinfo-tail-test.js @@ -86,8 +86,16 @@ export const test = { assert.strictEqual(typeErr.errorInfo[0].message, 'boom'); assert.ok( typeof typeErr.errorInfo[0].stack === 'string' && - typeErr.errorInfo[0].stack.includes('TypeError'), - `stack should be a string containing "TypeError", got: ${typeErr.errorInfo[0].stack}` + typeErr.errorInfo[0].stack.length > 0, + `stack should be a non-empty string, got: ${typeErr.errorInfo[0].stack}` + ); + assert.ok( + typeErr.errorInfo[0].stack.includes('at '), + `stack should contain call frames, got: ${typeErr.errorInfo[0].stack}` + ); + assert.ok( + !typeErr.errorInfo[0].stack.includes('TypeError'), + `stack should have the "Name: message" prefix stripped, got: ${typeErr.errorInfo[0].stack}` ); // 3. Two-arg call with Error at index 1: positional shape preserved. diff --git a/src/workerd/io/worker.c++ b/src/workerd/io/worker.c++ index 85f5308cc83..157ca24ace3 100644 --- a/src/workerd/io/worker.c++ +++ b/src/workerd/io/worker.c++ @@ -166,25 +166,17 @@ void sendExceptionToInspector(jsg::Lock& js, jsg::sendExceptionToInspector(js, inspector, kj::str(source), exception, message); } -void addExceptionToTrace(jsg::Lock& js, - IoContext& ioContext, - BaseTracer& tracer, - UncaughtExceptionSource source, +// Extracts {name, message, stack} from a JS value into a tracing::ErrorInfo. This is the single +// source of truth for how an error is represented in traces, shared by the uncaught-exception +// path (addExceptionToTrace) and the console.* logging path (Worker::handleLog), so that an error +// reported either way produces an identical representation. +// +// `name` defaults to "Error" when absent; `message` falls back to stringifying the whole value; +// and the redundant leading "Name: message" prefix is stripped off `stack` (since name and message +// are stored as separate fields). +tracing::ErrorInfo getErrorInfoForTrace(jsg::Lock& js, const jsg::JsValue& exception, const jsg::TypeHandler& errorTypeHandler) { - if (source == UncaughtExceptionSource::INTERNAL || - source == UncaughtExceptionSource::INTERNAL_ASYNC) { - // Skip redundant intermediate JS->C++ exception reporting. See: IoContext::runImpl(), - // PromiseWrapper::tryUnwrap() - // - // TODO(someday): Arguably it could make sense to store these exceptions off to the side and - // report them only if they don't end up being duplicates of a later exception that has a more - // specific context. This would cover cases where the C++ code that eventually received the - // exception never ended up reporting it. - return; - } - - auto timestamp = ioContext.now(); Worker::Api::ErrorInterface error; if (exception.isObject()) { @@ -242,8 +234,32 @@ void addExceptionToTrace(jsg::Lock& js, } } - tracer.addException(ioContext.getInvocationSpanContext(), timestamp, kj::mv(name), - kj::mv(message), kj::mv(stack)); + return tracing::ErrorInfo(kj::mv(name), kj::mv(message), kj::mv(stack)); +} + +void addExceptionToTrace(jsg::Lock& js, + IoContext& ioContext, + BaseTracer& tracer, + UncaughtExceptionSource source, + const jsg::JsValue& exception, + const jsg::TypeHandler& errorTypeHandler) { + if (source == UncaughtExceptionSource::INTERNAL || + source == UncaughtExceptionSource::INTERNAL_ASYNC) { + // Skip redundant intermediate JS->C++ exception reporting. See: IoContext::runImpl(), + // PromiseWrapper::tryUnwrap() + // + // TODO(someday): Arguably it could make sense to store these exceptions off to the side and + // report them only if they don't end up being duplicates of a later exception that has a more + // specific context. This would cover cases where the C++ code that eventually received the + // exception never ended up reporting it. + return; + } + + auto timestamp = ioContext.now(); + auto errorInfo = getErrorInfoForTrace(js, exception, errorTypeHandler); + + tracer.addException(ioContext.getInvocationSpanContext(), timestamp, kj::mv(errorInfo.name), + kj::mv(errorInfo.message), kj::mv(errorInfo.stack)); } void reportStartupError(kj::StringPtr id, @@ -2127,41 +2143,6 @@ void Worker::processEntrypointClass(jsg::Lock& js, }); } -namespace { - -// Reads a string-valued property off a v8 Object. Returns kj::none if the property is missing, -// not a string, or its accessor throws (some user code defines `stack` as a throwing getter). -// Caller must already hold a v8::TryCatch in scope. -kj::Maybe tryGetStringProperty(jsg::Lock& js, - v8::Local obj, - kj::StringPtr propName) { - auto context = js.v8Context(); - auto key = jsg::v8StrIntern(js.v8Isolate, propName); - v8::Local val; - if (!obj->Get(context, key).ToLocal(&val)) { - return kj::none; - } - if (!val->IsString()) { - return kj::none; - } - return kj::str(val.As()); -} - -// Extracts {name, message, stack} from a v8 Error. Returns kj::none if name or message are -// missing (defensive: a stripped-down Error-like is not useful enough to surface as ErrorInfo). -// Caller must hold a v8::TryCatch covering this call. -kj::Maybe extractErrorInfo(jsg::Lock& js, v8::Local errorObj) { - KJ_IF_SOME(name, tryGetStringProperty(js, errorObj, "name"_kj)) { - KJ_IF_SOME(message, tryGetStringProperty(js, errorObj, "message"_kj)) { - auto stack = tryGetStringProperty(js, errorObj, "stack"_kj); - return tracing::ErrorInfo(kj::mv(name), kj::mv(message), kj::mv(stack)); - } - } - return kj::none; -} - -} // namespace - void Worker::handleLog(jsg::Lock& js, const LoggingOptions& loggingOptions, LogLevel level, @@ -2182,38 +2163,6 @@ void Worker::handleLog(jsg::Lock& js, // here because `message` is called multiple times. v8::TryCatch tryCatch(js.v8Isolate); - // Scan all arguments for native Errors and build a positional ErrorInfo array: - // slot `i` holds the extracted {name, message, stack} for arg `i` if it was a native - // Error, otherwise kj::none. If no argument was a native Error, the whole array is - // left absent (kj::none) to keep the common case cheap on the wire. - // - // We do this independently of the stringification below so that calling `message()` - // multiple times doesn't re-extract, and so the existing stringification behavior is - // unchanged for backwards compatibility — the structured ErrorInfo is purely additive. - tracing::LogErrorInfo capturedErrors; - { - kj::Array> slots; - bool anyError = false; - for (auto i: kj::zeroTo(length)) { - if (!tryCatch.CanContinue()) break; - auto arg = info[i]; - if (!arg->IsNativeError()) continue; - if (!anyError) { - slots = kj::heapArray>(length); - anyError = true; - } - js.withinHandleScope([&] { - KJ_IF_SOME(extracted, extractErrorInfo(js, arg.As())) { - slots[i] = kj::mv(extracted); - anyError = true; - } - }); - } - if (anyError) { - capturedErrors = kj::mv(slots); - } - } - auto message = [&]() { int length = info.Length(); kj::Vector stringified(length); @@ -2289,6 +2238,51 @@ void Worker::handleLog(jsg::Lock& js, KJ_IF_SOME(ioContext, IoContext::tryCurrent()) { KJ_IF_SOME(tracer, ioContext.getWorkerTracer()) { auto timestamp = ioContext.now(); + + // Scan all arguments for native Errors and build a positional ErrorInfo array: slot `i` + // holds the extracted {name, message, stack} for arg `i` if it was a native Error, otherwise + // kj::none. If no argument was a native Error, the whole array is left absent (kj::none) to + // keep the common case cheap on the wire. We extract via the same helper used for uncaught + // exceptions (getErrorInfoForTrace) so that `console.error(err)` and a thrown `err` produce + // an identical representation in traces. + tracing::LogErrorInfo capturedErrors; + { + auto& errorTypeHandler = + ioContext.getWorker().getIsolate().getApi().getErrorInterfaceTypeHandler(js); + kj::Array> slots; + bool anyError = false; + for (auto i: kj::zeroTo(length)) { + if (!tryCatch.CanContinue()) break; + auto arg = info[i]; + if (!arg->IsNativeError()) continue; + + // A property accessor on the error may throw (e.g. user code defines `stack` as a + // throwing getter). console.* must not throw, so catch that and skip this argument, + // rethrowing only if the isolate is terminating (not a recoverable error). + kj::Maybe extracted; + v8::TryCatch argTryCatch(js.v8Isolate); + try { + js.withinHandleScope( + [&] { extracted = getErrorInfoForTrace(js, jsg::JsValue(arg), errorTypeHandler); }); + } catch (jsg::JsExceptionThrown&) { + if (!argTryCatch.CanContinue()) { + throw; + } + } + + KJ_IF_SOME(e, extracted) { + if (!anyError) { + slots = kj::heapArray>(length); + anyError = true; + } + slots[i] = kj::mv(e); + } + } + if (anyError) { + capturedErrors = kj::mv(slots); + } + } + tracer.addLog(ioContext.getInvocationSpanContext(), timestamp, level, message(), kj::mv(capturedErrors)); } From 8df424533d13d21ca7facde741e4f04624d589d3 Mon Sep 17 00:00:00 2001 From: Ibrahim Khajanchi Date: Tue, 23 Jun 2026 04:18:19 +0000 Subject: [PATCH 007/101] feedback --- src/workerd/api/tests/errorinfo-stw-test.js | 26 +++++++++++++++--- src/workerd/api/tests/errorinfo-tail-test.js | 26 +++++++++++++++--- src/workerd/io/worker.c++ | 28 ++++++++++++-------- 3 files changed, 63 insertions(+), 17 deletions(-) diff --git a/src/workerd/api/tests/errorinfo-stw-test.js b/src/workerd/api/tests/errorinfo-stw-test.js index 929690af80f..031d403ad39 100644 --- a/src/workerd/api/tests/errorinfo-stw-test.js +++ b/src/workerd/api/tests/errorinfo-stw-test.js @@ -22,6 +22,15 @@ export default { { plain: 'obj' }, new TypeError('second') ); + // Native Error whose `stack` getter throws. Extraction must be skipped for + // that arg while a normal Error in the same call is still captured. + const throwingStack = new Error('explode'); + Object.defineProperty(throwingStack, 'stack', { + get() { + throw new Error('stack getter blew up'); + }, + }); + console.error(throwingStack, new TypeError('survivor')); return new Response('ok'); }, @@ -44,11 +53,12 @@ export const test = { assert.strictEqual( capturedLogEvents.length, - 5, - `expected 5 streamed log events, got ${capturedLogEvents.length}` + 6, + `expected 6 streamed log events, got ${capturedLogEvents.length}` ); - const [plain, typeErr, ctxRange, fakeErr, mixed] = capturedLogEvents; + const [plain, typeErr, ctxRange, fakeErr, mixed, throwy] = + capturedLogEvents; assert.strictEqual(plain.errorInfo, undefined); @@ -89,5 +99,15 @@ export const test = { assert.ok(mixed.errorInfo[3]); assert.strictEqual(mixed.errorInfo[3].name, 'TypeError'); assert.strictEqual(mixed.errorInfo[3].message, 'second'); + + // Error with a throwing `stack` getter: extraction throws and is swallowed, + // so slot 0 is null, but the request still completed and the + // sibling Error in slot 1 is captured normally. + assert.ok(Array.isArray(throwy.errorInfo)); + assert.strictEqual(throwy.errorInfo.length, 2); + assert.strictEqual(throwy.errorInfo[0], null); + assert.ok(throwy.errorInfo[1]); + assert.strictEqual(throwy.errorInfo[1].name, 'TypeError'); + assert.strictEqual(throwy.errorInfo[1].message, 'survivor'); }, }; diff --git a/src/workerd/api/tests/errorinfo-tail-test.js b/src/workerd/api/tests/errorinfo-tail-test.js index 0274112a2f8..a96fcc07688 100644 --- a/src/workerd/api/tests/errorinfo-tail-test.js +++ b/src/workerd/api/tests/errorinfo-tail-test.js @@ -39,6 +39,16 @@ export default { new TypeError('second') ); + // 6. Native Error whose `stack` getter throws. Extraction must be skipped + // for that arg while a normal Error in the same call is still captured. + const throwingStack = new Error('explode'); + Object.defineProperty(throwingStack, 'stack', { + get() { + throw new Error('stack getter blew up'); + }, + }); + console.error(throwingStack, new TypeError('survivor')); + return new Response('ok'); }, @@ -66,11 +76,11 @@ export const test = { assert.ok(capturedLogs, 'tail handler was never called'); assert.strictEqual( capturedLogs.length, - 5, - `expected 5 log entries, got ${capturedLogs.length}` + 6, + `expected 6 log entries, got ${capturedLogs.length}` ); - const [plain, typeErr, ctxRange, fakeErr, mixed] = capturedLogs; + const [plain, typeErr, ctxRange, fakeErr, mixed, throwy] = capturedLogs; // 1. No Errors anywhere: field absent entirely. assert.strictEqual( @@ -127,6 +137,16 @@ export const test = { assert.strictEqual(mixed.errorInfo[3].name, 'TypeError'); assert.strictEqual(mixed.errorInfo[3].message, 'second'); + // 6. Error with a throwing `stack` getter: extraction throws and is + // swallowed, so slot 0 is null, but the request still completed and + // the sibling Error in slot 1 is captured normally. + assert.ok(Array.isArray(throwy.errorInfo)); + assert.strictEqual(throwy.errorInfo.length, 2); + assert.strictEqual(throwy.errorInfo[0], null); + assert.ok(throwy.errorInfo[1]); + assert.strictEqual(throwy.errorInfo[1].name, 'TypeError'); + assert.strictEqual(throwy.errorInfo[1].message, 'survivor'); + // Backwards compat: existing message field still present and an object. assert.strictEqual(typeof typeErr.message, 'object'); }, diff --git a/src/workerd/io/worker.c++ b/src/workerd/io/worker.c++ index 157ca24ace3..7d6b2c3f2e3 100644 --- a/src/workerd/io/worker.c++ +++ b/src/workerd/io/worker.c++ @@ -2256,18 +2256,24 @@ void Worker::handleLog(jsg::Lock& js, auto arg = info[i]; if (!arg->IsNativeError()) continue; - // A property accessor on the error may throw (e.g. user code defines `stack` as a - // throwing getter). console.* must not throw, so catch that and skip this argument, - // rethrowing only if the isolate is terminating (not a recoverable error). + // Reading the error's properties may throw an application-level JS exception (e.g. user + // code defines `stack`/`message` as a throwing getter). JSG_CATCH still rethrows + // automatically if the isolate is terminating, so only recoverable JS exceptions land in + // the handler below. kj::Maybe extracted; - v8::TryCatch argTryCatch(js.v8Isolate); - try { - js.withinHandleScope( - [&] { extracted = getErrorInfoForTrace(js, jsg::JsValue(arg), errorTypeHandler); }); - } catch (jsg::JsExceptionThrown&) { - if (!argTryCatch.CanContinue()) { - throw; - } + JSG_TRY(js) { + // Fresh handle scope so the temporary handles created while reading the error's + // properties are released each iteration rather than accumulating across the loop. + v8::HandleScope handleScope(js.v8Isolate); + extracted = getErrorInfoForTrace(js, jsg::JsValue(arg), errorTypeHandler); + } + JSG_CATCH(_) { + // Deliberately swallow the exception because: + // - console.* must not throw + // - errorInfo is purely additive/best-effort. If it fails to parse, + // the argument is still stringified + // - We don't log the failure to mirror the silent fallback + // in getErrorInfoForTrace's stringify path. } KJ_IF_SOME(e, extracted) { From 11f75080e0025084d6f5e9270ca6bfc82d89b1f1 Mon Sep 17 00:00:00 2001 From: James M Snell Date: Wed, 1 Jul 2026 16:41:56 -0700 Subject: [PATCH 008/101] Remove this capture in basics.c++ --- src/workerd/api/basics.c++ | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/workerd/api/basics.c++ b/src/workerd/api/basics.c++ index 53b64e33e9f..4bb81d8208b 100644 --- a/src/workerd/api/basics.c++ +++ b/src/workerd/api/basics.c++ @@ -925,9 +925,9 @@ void AbortSignal::subscribeToRpcAbort(jsg::Lock& js) { // though, we don't want to awaitIo() since it blocks hibernation in actors. KJ_IF_SOME(promise, rpcAbortPromise) { - IoContext::current().awaitIo(js, kj::mv(*promise), [this, self = JSG_THIS](jsg::Lock& js) { - KJ_IF_SOME(r, deserializePendingReason(js)) { - triggerAbort(js, r); + IoContext::current().awaitIo(js, kj::mv(*promise), [self = JSG_THIS](jsg::Lock& js) mutable { + KJ_IF_SOME(r, self->deserializePendingReason(js)) { + self->triggerAbort(js, r); } }); From 3b96f764e63b15b2d3074d5cae7f47aa99a7b987 Mon Sep 17 00:00:00 2001 From: James M Snell Date: Wed, 1 Jul 2026 16:44:33 -0700 Subject: [PATCH 009/101] Remove this capture in analytics-engine.c++ --- src/workerd/api/analytics-engine.c++ | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/workerd/api/analytics-engine.c++ b/src/workerd/api/analytics-engine.c++ index 70e2c3aa56a..f4a1dbd18ab 100644 --- a/src/workerd/api/analytics-engine.c++ +++ b/src/workerd/api/analytics-engine.c++ @@ -20,8 +20,8 @@ void AnalyticsEngine::writeDataPoint( // awaitIo() and such by not going back to the event loop at all. KJ_IF_SOME(promise, context.waitForOutputLocksIfNecessary()) { context.awaitIo( - js, kj::mv(promise), [this, self = JSG_THIS, event = kj::mv(event)](jsg::Lock& js) mutable { - writeDataPointNoOutputLock(js, kj::mv(event)); + js, kj::mv(promise), [self = JSG_THIS, event = kj::mv(event)](jsg::Lock& js) mutable { + self->writeDataPointNoOutputLock(js, kj::mv(event)); }); } else { writeDataPointNoOutputLock(js, kj::mv(event)); From 1d57f4cf7620c975fdf2fa54db56f195ef3cff51 Mon Sep 17 00:00:00 2001 From: James M Snell Date: Wed, 1 Jul 2026 16:46:11 -0700 Subject: [PATCH 010/101] Remove this capture in container.c++ --- src/workerd/api/container.c++ | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/workerd/api/container.c++ b/src/workerd/api/container.c++ index b7e11af4277..0073b08799d 100644 --- a/src/workerd/api/container.c++ +++ b/src/workerd/api/container.c++ @@ -609,13 +609,13 @@ jsg::Promise Container::monitor(jsg::Lock& js) { // Note: `self` (jsg::Ref) is captured to prevent GC from collecting this object while // the promise continuation is pending. Without it, the bare `this` pointer dangles. .then(js, - [this, self = JSG_THIS]( - jsg::Lock& js, capnp::Response results) { - running = false; + [self = JSG_THIS]( + jsg::Lock& js, capnp::Response results) mutable { + self->running = false; auto exitCode = results.getExitCode(); - KJ_IF_SOME(d, destroyReason) { + KJ_IF_SOME(d, self->destroyReason) { jsg::Value error = kj::mv(d); - destroyReason = kj::none; + self->destroyReason = kj::none; js.throwException(kj::mv(error)); } @@ -625,9 +625,9 @@ jsg::Promise Container::monitor(jsg::Lock& js) { js.throwException(err); } }, - [this, self = JSG_THIS](jsg::Lock& js, jsg::Value&& error) { - running = false; - destroyReason = kj::none; + [self = JSG_THIS](jsg::Lock& js, jsg::Value&& error) mutable { + self->running = false; + self->destroyReason = kj::none; js.throwException(kj::mv(error)); }); } From 0acd8c821eb91edf8d1fcfb3dd920d9c6ac6cbb4 Mon Sep 17 00:00:00 2001 From: James M Snell Date: Wed, 1 Jul 2026 16:48:44 -0700 Subject: [PATCH 011/101] Remove this capture in http.c++ --- src/workerd/api/http.c++ | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/workerd/api/http.c++ b/src/workerd/api/http.c++ index 164829304f3..8f723dc9c88 100644 --- a/src/workerd/api/http.c++ +++ b/src/workerd/api/http.c++ @@ -337,8 +337,8 @@ jsg::Promise> Body::blob(jsg::Lock& js) { // Note: `self` (jsg::Ref) is captured to prevent GC from collecting this object while // the promise continuation is pending. Without it, the bare `this` pointer dangles. return arrayBuffer(js).then( - js, [this, self = JSG_THIS](jsg::Lock& js, jsg::JsRef buffer) { - kj::String contentType = headersRef.getCommon(js, capnp::CommonHeaderName::CONTENT_TYPE) + js, [self = JSG_THIS](jsg::Lock& js, jsg::JsRef buffer) mutable { + kj::String contentType = self->headersRef.getCommon(js, capnp::CommonHeaderName::CONTENT_TYPE) .map([](auto&& b) -> kj::String { return kj::mv(b); }).orDefault(nullptr); From e1dbd21862ffd1a8941e8d17a28e41d2c2b89fc7 Mon Sep 17 00:00:00 2001 From: James M Snell Date: Wed, 1 Jul 2026 16:50:15 -0700 Subject: [PATCH 012/101] Remove this capture from r2-bucket.c++ --- src/workerd/api/r2-bucket.c++ | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/workerd/api/r2-bucket.c++ b/src/workerd/api/r2-bucket.c++ index 4c4fb17179f..467ac66e55d 100644 --- a/src/workerd/api/r2-bucket.c++ +++ b/src/workerd/api/r2-bucket.c++ @@ -1479,12 +1479,12 @@ jsg::Promise R2Bucket::GetResult::json(jsg::Lock& js) { jsg::Promise> R2Bucket::GetResult::blob(jsg::Lock& js) { // Copy-pasted from http.c++ return arrayBuffer(js).then( - js, [this, self = JSG_THIS](jsg::Lock& js, jsg::JsRef buffer) { + js, [self = JSG_THIS](jsg::Lock& js, jsg::JsRef buffer) mutable { // httpMetadata can't be null because GetResult always populates it. // Note: `self` (jsg::Ref) is captured to prevent GC from collecting this object while // the promise continuation is pending. Without it, the bare `this` pointer dangles. kj::String contentType = - mapCopyString(KJ_REQUIRE_NONNULL(httpMetadata).contentType).orDefault(nullptr); + mapCopyString(KJ_REQUIRE_NONNULL(self->httpMetadata).contentType).orDefault(nullptr); return js.alloc(js, jsg::JsBufferSource(buffer.getHandle(js)), kj::mv(contentType)); }); } From 562ef4e69658a3e15cabdd30191daaa137bd2ec9 Mon Sep 17 00:00:00 2001 From: James M Snell Date: Wed, 1 Jul 2026 16:57:18 -0700 Subject: [PATCH 013/101] Remove this captures from sockets.c++ --- src/workerd/api/sockets.c++ | 55 +++++++++++++++++++------------------ 1 file changed, 28 insertions(+), 27 deletions(-) diff --git a/src/workerd/api/sockets.c++ b/src/workerd/api/sockets.c++ index d6ccd2d166f..38b4fc41c71 100644 --- a/src/workerd/api/sockets.c++ +++ b/src/workerd/api/sockets.c++ @@ -295,33 +295,34 @@ jsg::Promise Socket::close(jsg::Lock& js) { // this object while the promise chain is pending. Without it, the bare `this` pointer dangles. return openedPromiseCopy.whenResolved(js) .then(js, - [this, self = JSG_THIS](jsg::Lock& js) { - if (!writable->getController().isClosedOrClosing()) { - return writable->getController().flush(js); + [self = JSG_THIS](jsg::Lock& js) mutable { + auto& controller = self->writable->getController(); + if (!controller.isClosedOrClosing()) { + return controller.flush(js); } else { return js.resolvedPromise(); } }) .then(js, - [this, self = JSG_THIS](jsg::Lock& js) { + [self = JSG_THIS](jsg::Lock& js) mutable { // Forcibly abort the readable/writable streams. - auto cancelPromise = readable->getController().cancel(js, kj::none); - auto abortPromise = writable->getController().abort(js, kj::none); + auto cancelPromise = self->readable->getController().cancel(js, kj::none); + auto abortPromise = self->writable->getController().abort(js, kj::none); // The below is effectively `Promise.all(cancelPromise, abortPromise)` return cancelPromise.then(js, [abortPromise = kj::mv(abortPromise)](jsg::Lock& js) mutable { return kj::mv(abortPromise); }); }) - .then(js, [this, self = JSG_THIS](jsg::Lock& js) { + .then(js, [self = JSG_THIS](jsg::Lock& js) mutable { // Destroy the connection stream to close the connection. - { auto _ = kj::mv(connectionData); } - connectionData = kj::none; + { auto _ = kj::mv(self->connectionData); } + self->connectionData = kj::none; - resolveFulfiller(js, kj::none); + self->resolveFulfiller(js, kj::none); return js.resolvedPromise(); - }).catch_(js, [this, self = JSG_THIS](jsg::Lock& js, jsg::Value err) { - errorHandler(js, kj::mv(err)); + }).catch_(js, [self = JSG_THIS](jsg::Lock& js, jsg::Value err) mutable { + self->errorHandler(js, kj::mv(err)); }); } @@ -450,32 +451,32 @@ void Socket::handleProxyStatus( } return kj::HttpClient::ConnectRequest::Status(500, nullptr, kj::Own()); }; - auto func = [this, self = JSG_THIS]( - jsg::Lock& js, kj::HttpClient::ConnectRequest::Status&& status) -> void { + auto func = [self = JSG_THIS]( + jsg::Lock& js, kj::HttpClient::ConnectRequest::Status&& status) mutable -> void { if (status.statusCode < 200 || status.statusCode >= 300) { // If the status indicates an unsuccessful connection we need to reject the `closeFulfiller` // with an exception. This will reject the socket's `closed` promise. auto msg = kj::str("proxy request failed, cannot connect to the specified address"); - if (isDefaultFetchPort) { + if (self->isDefaultFetchPort) { msg = kj::str(msg, ". It looks like you might be trying to connect to a HTTP-based service", " — consider using fetch instead"); - } else if (remoteAddress.orDefault(kj::String()).contains(".hyperdrive.local"_kj)) { + } else if (self->remoteAddress.orDefault(kj::String()).contains(".hyperdrive.local"_kj)) { // No attempts to connect to Hyperdrive should end up here, since they go through the other // version of handleProxyStatus. If they end up here somehow, log about it to get some // context that can aid in debugging. LOG_WARNING_PERIODICALLY( - "attempt to connect to Hyperdrive failed to trigger connectOverride", remoteAddress, - status.statusCode, status.statusText); + "attempt to connect to Hyperdrive failed to trigger connectOverride", + self->remoteAddress, status.statusCode, status.statusText); } - handleProxyError(js, JSG_KJ_EXCEPTION(FAILED, Error, msg)); + self->handleProxyError(js, JSG_KJ_EXCEPTION(FAILED, Error, msg)); } else { // For outbound sockets we have no useful local address to expose. Inbound sockets (produced // by the `connect()` handler dispatch path) populate `localAddress` with the CONNECT // authority that the peer targeted. - openedResolver.resolve(js, + self->openedResolver.resolve(js, SocketInfo{ - .remoteAddress = mapCopyString(remoteAddress), - .localAddress = mapCopyString(localAddress), + .remoteAddress = mapCopyString(self->remoteAddress), + .localAddress = mapCopyString(self->localAddress), }); } }; @@ -497,17 +498,17 @@ void Socket::handleProxyStatus(jsg::Lock& js, kj::Promise result) -> void { + auto func = [self = JSG_THIS](jsg::Lock& js, kj::Maybe result) mutable -> void { if (result != kj::none) { - handleProxyError(js, JSG_KJ_EXCEPTION(FAILED, Error, "connection attempt failed")); + self->handleProxyError(js, JSG_KJ_EXCEPTION(FAILED, Error, "connection attempt failed")); } else { // For outbound sockets we have no useful local address to expose. Inbound sockets (produced // by the `connect()` handler dispatch path) populate `localAddress` with the CONNECT // authority that the peer targeted. - openedResolver.resolve(js, + self->openedResolver.resolve(js, SocketInfo{ - .remoteAddress = mapCopyString(remoteAddress), - .localAddress = mapCopyString(localAddress), + .remoteAddress = mapCopyString(self->remoteAddress), + .localAddress = mapCopyString(self->localAddress), }); } }; From 90e7126553538ec744b2097ba1c39d18240b4e33 Mon Sep 17 00:00:00 2001 From: Matt Provost Date: Thu, 11 Jun 2026 20:55:38 +0000 Subject: [PATCH 014/101] AUTH-8288: update getIdentity to rpc Signed-off-by: Matt Provost --- src/workerd/api/global-scope.c++ | 34 +++++++++++++++++++++++--------- src/workerd/api/global-scope.h | 15 +++++++++++++- src/workerd/io/access-info.h | 18 ++++++++++++----- 3 files changed, 52 insertions(+), 15 deletions(-) diff --git a/src/workerd/api/global-scope.c++ b/src/workerd/api/global-scope.c++ index 434ea9c3236..46b1c3abca2 100644 --- a/src/workerd/api/global-scope.c++ +++ b/src/workerd/api/global-scope.c++ @@ -14,6 +14,7 @@ #include #endif #include +#include #include #include #include @@ -106,15 +107,30 @@ kj::StringPtr AccessContext::getAud() { return info->getAudience(); } -jsg::Promise> AccessContext::getIdentity(jsg::Lock& js) { - auto& ioctx = IoContext::current(); - return ioctx.awaitIo(js, info->getIdentity(), - [](jsg::Lock& js, kj::Maybe json) -> jsg::Optional { - KJ_IF_SOME(j, json) { - return jsg::JsValue(js.parseJson(j).getHandle(js)); - } - return kj::none; - }); +jsg::Promise AccessContext::getIdentity(jsg::Lock& js, + const jsg::TypeHandler>& rpcPropHandler, + const jsg::TypeHandler>& getIdentityFnHandler) { + // Invoke the `getIdentity` JS-RPC method on the Access binding worker via a Fetcher bound to the + // embedder-supplied subrequest channel. If no identity service channel is available for this + // request (e.g. service-token auth), there is no identity to fetch, so resolve to `undefined`. + // Otherwise the binding worker's result (or error) propagates to the caller unchanged. + KJ_IF_SOME(channel, info->getIdentityServiceChannel()) { + auto fetcher = + js.alloc(channel, Fetcher::RequiresHostAndProtocol::NO, true /* isInHouse */); + auto rpcProp = JSG_REQUIRE_NONNULL(fetcher->getRpcMethodInternal(js, kj::str("getIdentity")), + Error, "Access binding worker is missing the getIdentity method"); + + auto getIdentityFn = JSG_REQUIRE_NONNULL( + getIdentityFnHandler.tryUnwrap(js, rpcPropHandler.wrap(js, kj::mv(rpcProp))), Error, + "Access binding worker getIdentity is not callable"); + + // The RPC method returns a `JsRpcPromise` (custom thenable). Normalize it into a real promise + // via a resolver so we're independent of the `unwrapCustomThenables` compat flag. + auto paf = js.newPromiseAndResolver(); + paf.resolver.resolve(js, getIdentityFn(js)); + return kj::mv(paf.promise); + } + return js.resolvedPromise(jsg::Value(js.v8Isolate, v8::Undefined(js.v8Isolate))); } jsg::Optional> ExecutionContext::getAccess(jsg::Lock& js) { diff --git a/src/workerd/api/global-scope.h b/src/workerd/api/global-scope.h index 4b1c28cd2ed..d33331ec621 100644 --- a/src/workerd/api/global-scope.h +++ b/src/workerd/api/global-scope.h @@ -261,7 +261,20 @@ class AccessContext: public jsg::Object { // Fetches the full identity information for the authenticated user. Resolves to `undefined` // if no identity is associated with the request (e.g. service-token authentication). - jsg::Promise> getIdentity(jsg::Lock& js); + // + // Returns `jsg::Promise` (a persistent V8 ref) rather than `jsg::JsValue`: the + // resolved value must survive across microtask boundaries until the awaiting code runs, and a + // transient `jsg::JsValue` (a `v8::Local`) would dangle. The `undefined` case is represented as a + // JS `undefined` value. The TS type is pinned to `CloudflareAccessIdentity | undefined` via the + // JSG_TS_OVERRIDE below. + // + // `rpcPropHandler` wraps the `JsRpcProperty` returned by `Fetcher::getRpcMethodInternal` into a + // JS value; `getIdentityFnHandler` then adapts it into a `jsg::Function` so we can invoke the + // RPC method as a C++ functor without hand-rolling raw `v8::Function` casts. Both are injected + // automatically by JSG. + jsg::Promise getIdentity(jsg::Lock& js, + const jsg::TypeHandler>& rpcPropHandler, + const jsg::TypeHandler>& getIdentityFnHandler); JSG_RESOURCE_TYPE(AccessContext) { JSG_READONLY_INSTANCE_PROPERTY(aud, getAud); diff --git a/src/workerd/io/access-info.h b/src/workerd/io/access-info.h index 57d1223dc83..880becd41a9 100644 --- a/src/workerd/io/access-info.h +++ b/src/workerd/io/access-info.h @@ -4,7 +4,7 @@ #pragma once -#include +#include #include #include @@ -31,10 +31,18 @@ class AccessInfo: public kj::Refcounted { // The audience claim from the Access JWT. Stable for the lifetime of the request. virtual kj::StringPtr getAudience() = 0; - // Fetches the full identity information for the authenticated user, equivalent to calling - // /cdn-cgi/access/get-identity. The returned string is a JSON document; `kj::none` indicates - // no identity is available (e.g. service-token authentication). - virtual kj::Promise> getIdentity() = 0; + // Returns the subrequest channel index for the Access "binding worker", on which workerd invokes + // the `getIdentity` JS-RPC method (via a `Fetcher`) to fetch the authenticated user's full + // identity (equivalent to calling /cdn-cgi/access/get-identity). + // + // Returns `kj::none` when no identity service is available for this request (e.g. service-token + // authentication, or the embedder has no channel configured), in which case + // `ctx.access.getIdentity()` resolves to `undefined`. + // + // This boundary is deliberately narrow: the embedder is responsible only for *routing* a channel + // to the Access binding worker (e.g. via a per-request channel token), while workerd owns the + // JS-RPC dispatch and result handling. + virtual kj::Maybe getIdentityServiceChannel() = 0; }; } // namespace workerd From 163a8fcf0e89f42c2c0e443bf611a076d2a817a9 Mon Sep 17 00:00:00 2001 From: Mike Aizatsky Date: Thu, 2 Jul 2026 08:43:33 -0700 Subject: [PATCH 015/101] Weak WritableStream owner --- build/deps/gen/deps.MODULE.bazel | 6 +++--- src/workerd/api/streams/common.h | 2 +- src/workerd/api/streams/internal.c++ | 2 +- src/workerd/api/streams/internal.h | 6 +++--- src/workerd/api/streams/standard.c++ | 28 ++++++++++++---------------- src/workerd/api/streams/standard.h | 6 +++--- src/workerd/api/streams/writable.c++ | 2 +- src/workerd/api/streams/writable.h | 12 ++---------- 8 files changed, 26 insertions(+), 38 deletions(-) diff --git a/build/deps/gen/deps.MODULE.bazel b/build/deps/gen/deps.MODULE.bazel index ec427dd2096..9213a5cb63c 100644 --- a/build/deps/gen/deps.MODULE.bazel +++ b/build/deps/gen/deps.MODULE.bazel @@ -27,10 +27,10 @@ bazel_dep(name = "brotli", version = "1.2.0.bcr.1") # capnp-cpp http.archive( name = "capnp-cpp", - sha256 = "94e5bcd03d14b8f1a1713e2477941e00b130b63b5a1aad97a10ff5edf05b6765", - strip_prefix = "capnproto-capnproto-e9fa5c7/c++", + sha256 = "6dae6cdfc776cadc94fe2a802186f94840329520cb53a9d02306175c06101f92", + strip_prefix = "capnproto-capnproto-6ef408c/c++", type = "tgz", - url = "https://github.com/capnproto/capnproto/tarball/e9fa5c7dc98192fc0dc0098ec770db68f997a938", + url = "https://github.com/capnproto/capnproto/tarball/6ef408c214210e3d19a1da6f695b23e782c8ffd8", ) use_repo(http, "capnp-cpp") diff --git a/src/workerd/api/streams/common.h b/src/workerd/api/streams/common.h index 758e10df2e6..3ab6c828ee5 100644 --- a/src/workerd/api/streams/common.h +++ b/src/workerd/api/streams/common.h @@ -716,7 +716,7 @@ class WritableStreamController { virtual ~WritableStreamController() noexcept(false) {} - virtual void setOwnerRef(WritableStream& stream) = 0; + virtual void setOwnerRef(kj::Weak stream) = 0; virtual jsg::Ref addRef() = 0; diff --git a/src/workerd/api/streams/internal.c++ b/src/workerd/api/streams/internal.c++ index 7e3a6f8945e..2ca378883df 100644 --- a/src/workerd/api/streams/internal.c++ +++ b/src/workerd/api/streams/internal.c++ @@ -1017,7 +1017,7 @@ WritableStreamInternalController::~WritableStreamInternalController() noexcept(f } jsg::Ref WritableStreamInternalController::addRef() { - return KJ_ASSERT_NONNULL(owner).addRef(); + return owner.assertLive().addRef(); } jsg::Promise WritableStreamInternalController::write( diff --git a/src/workerd/api/streams/internal.h b/src/workerd/api/streams/internal.h index 787967dc4d8..bba33ae19a2 100644 --- a/src/workerd/api/streams/internal.h +++ b/src/workerd/api/streams/internal.h @@ -216,8 +216,8 @@ class WritableStreamInternalController: public WritableStreamController { ~WritableStreamInternalController() noexcept(false) override; - void setOwnerRef(WritableStream& stream) override { - owner = stream; + void setOwnerRef(kj::Weak stream) override { + owner = kj::mv(stream); } jsg::Ref addRef() override; @@ -298,7 +298,7 @@ class WritableStreamInternalController: public WritableStreamController { ReadableStream& ref; }; - kj::Maybe owner; + kj::Weak owner; // State machine for WritableStreamInternalController: // Closed is terminal, Errored is implicitly terminal via ErrorState. diff --git a/src/workerd/api/streams/standard.c++ b/src/workerd/api/streams/standard.c++ index ae704ab056c..452cc2372bf 100644 --- a/src/workerd/api/streams/standard.c++ +++ b/src/workerd/api/streams/standard.c++ @@ -932,7 +932,7 @@ class WritableStreamJsController final: public WritableStreamController { kj::Maybe> removeSink(jsg::Lock& js) override; void detach(jsg::Lock& js) override; - void setOwnerRef(WritableStream& stream) override; + void setOwnerRef(kj::Weak stream) override; void setup(jsg::Lock& js, jsg::Optional maybeUnderlyingSink, @@ -966,7 +966,7 @@ class WritableStreamJsController final: public WritableStreamController { jsg::Promise pipeLoop(jsg::Lock& js); kj::Maybe ioContext; - kj::Maybe owner; + kj::Weak owner; // Initial state before setup() is called. struct Initial { @@ -1267,8 +1267,8 @@ kj::Own::Consumer> ReadableImpl::getConsumer( template WritableImpl::WritableImpl( - jsg::Lock& js, WritableStream& owner, jsg::Ref abortSignal) - : owner(owner.addWeakRef()), + jsg::Lock& js, kj::Weak owner, jsg::Ref abortSignal) + : owner(kj::mv(owner)), signal(kj::mv(abortSignal)) { flags.pedanticWpt = FeatureFlags::get(js).getPedanticWpt(); } @@ -1314,9 +1314,7 @@ jsg::Promise WritableImpl::abort( template kj::Maybe WritableImpl::tryGetOwner() { KJ_IF_SOME(o, owner) { - return o->tryGet().map([](WritableStream& owner) -> WritableStreamJsController& { - return static_cast(owner.getController()); - }); + return static_cast(o->getController()); } return kj::none; } @@ -3587,9 +3585,9 @@ kj::Promise> ReadableStreamJsController::pumpTo( // ====================================================================================== WritableStreamDefaultController::WritableStreamDefaultController( - jsg::Lock& js, WritableStream& owner, jsg::Ref abortSignal) + jsg::Lock& js, kj::Weak owner, jsg::Ref abortSignal) : ioContext(tryGetIoContext()), - impl(js, owner, kj::mv(abortSignal)) {} + impl(js, kj::mv(owner), kj::mv(abortSignal)) {} jsg::Promise WritableStreamDefaultController::abort(jsg::Lock& js, jsg::JsValue reason) { return impl.abort(js, JSG_THIS, reason); @@ -3659,8 +3657,6 @@ WritableStreamJsController::~WritableStreamJsController() noexcept(false) { // Clear the state to break the circular reference to the controller. // During destruction, we force the transition since the current state doesn't matter. state.forceTransitionTo(); - // Clear owner reference - owner = kj::none; // Clear any pending abort promise maybeAbortPromise = kj::none; } @@ -3707,7 +3703,7 @@ jsg::Promise WritableStreamJsController::abort( } jsg::Ref WritableStreamJsController::addRef() { - return KJ_ASSERT_NONNULL(owner).addRef(); + return owner.assertLive().addRef(); } bool WritableStreamJsController::isClosedOrClosing() { @@ -3899,8 +3895,8 @@ void WritableStreamJsController::detach(jsg::Lock& js) { KJ_UNIMPLEMENTED("WritableStreamJsController::detach is not implemented"); } -void WritableStreamJsController::setOwnerRef(WritableStream& stream) { - owner = stream; +void WritableStreamJsController::setOwnerRef(kj::Weak stream) { + owner = kj::mv(stream); } void WritableStreamJsController::setup(jsg::Lock& js, @@ -3919,7 +3915,7 @@ void WritableStreamJsController::setup(jsg::Lock& js, // We account for the memory usage of the WritableStreamDefaultController and AbortSignal together // because their lifetimes are identical and memory accounting itself has a memory overhead. auto controller = js.allocAccounted( - sizeof(WritableStreamDefaultController) + sizeof(AbortSignal), js, KJ_ASSERT_NONNULL(owner), + sizeof(WritableStreamDefaultController) + sizeof(AbortSignal), js, owner, js.alloc()); auto& controllerRef = *controller; state.transitionTo(kj::mv(controller)); @@ -3940,7 +3936,7 @@ kj::Maybe> WritableStreamJsController::tryPipeFrom( // completes, or is rejected if the pipe operation is aborted or errored. // Let's also acquire the destination pipe lock. - lock.pipeLock(KJ_ASSERT_NONNULL(owner), kj::mv(source), options); + lock.pipeLock(owner.assertLive(), kj::mv(source), options); return pipeLoop(js).then(js, [ref = addRef()](auto& js) {}); } diff --git a/src/workerd/api/streams/standard.h b/src/workerd/api/streams/standard.h index 5be2c6a7ff2..7c869bc3b1d 100644 --- a/src/workerd/api/streams/standard.h +++ b/src/workerd/api/streams/standard.h @@ -290,7 +290,7 @@ class WritableImpl { } }; - WritableImpl(jsg::Lock& js, WritableStream& owner, jsg::Ref abortSignal); + WritableImpl(jsg::Lock& js, kj::Weak owner, jsg::Ref abortSignal); jsg::Promise abort(jsg::Lock& js, jsg::Ref self, jsg::JsValue reason); @@ -405,7 +405,7 @@ class WritableImpl { // uses this WritableImpl. This creates a strong circular reference between jsg::Refs // that isn't allowed. GcTracing ends up with a stack overflow as the two jsg::Refs // try tracing each other. - kj::Maybe>> owner; + kj::Weak owner; jsg::Ref signal; State state = State::template create(); Algorithms algorithms; @@ -648,7 +648,7 @@ class WritableStreamDefaultController: public jsg::Object { using WritableImpl = WritableImpl; explicit WritableStreamDefaultController( - jsg::Lock& js, WritableStream& owner, jsg::Ref abortSignal); + jsg::Lock& js, kj::Weak owner, jsg::Ref abortSignal); ~WritableStreamDefaultController() noexcept(false); diff --git a/src/workerd/api/streams/writable.c++ b/src/workerd/api/streams/writable.c++ index 94991479558..ac04b9399cb 100644 --- a/src/workerd/api/streams/writable.c++ +++ b/src/workerd/api/streams/writable.c++ @@ -187,7 +187,7 @@ WritableStream::WritableStream(IoContext& ioContext, WritableStream::WritableStream(kj::Own controller) : ioContext(tryGetIoContext()), controller(kj::mv(controller)) { - getController().setOwnerRef(*this); + getController().setOwnerRef(addWeakToThis()); } jsg::Ref WritableStream::addRef() { diff --git a/src/workerd/api/streams/writable.h b/src/workerd/api/streams/writable.h index d3fba654fd4..0448121e7b5 100644 --- a/src/workerd/api/streams/writable.h +++ b/src/workerd/api/streams/writable.h @@ -133,7 +133,7 @@ class WritableStreamDefaultWriter: public jsg::Object, public WritableStreamCont void visitForGc(jsg::GcVisitor& visitor); }; -class WritableStream: public jsg::Object { +class WritableStream: public jsg::Object, public kj::PtrTarget { public: explicit WritableStream(IoContext& ioContext, kj::Own sink, @@ -142,9 +142,7 @@ class WritableStream: public jsg::Object { kj::Maybe> maybeClosureWaitable = kj::none); explicit WritableStream(kj::Own controller); - ~WritableStream() noexcept(false) { - weakRef->invalidate(); - } + ~WritableStream() noexcept(false) = default; WritableStreamController& getController(); @@ -211,12 +209,6 @@ class WritableStream: public jsg::Object { private: kj::Maybe ioContext; kj::Own controller; - kj::Own> weakRef = - kj::refcounted>(kj::Badge(), *this); - - kj::Own> addWeakRef() { - return weakRef->addRef(); - } void visitForGc(jsg::GcVisitor& visitor); From c0c7450c723c4a08993b62f27943c17bb7cfec4c Mon Sep 17 00:00:00 2001 From: Mike Aizatsky Date: Thu, 2 Jul 2026 08:56:24 -0700 Subject: [PATCH 016/101] Weak ReadableStream owner --- build/deps/gen/deps.MODULE.bazel | 6 +++--- src/workerd/api/streams/README.md | 23 +++++++++++++++++++++++ src/workerd/api/streams/common.h | 2 +- src/workerd/api/streams/internal.c++ | 14 +++++++++----- src/workerd/api/streams/internal.h | 6 ++---- src/workerd/api/streams/readable.c++ | 2 +- src/workerd/api/streams/readable.h | 2 +- src/workerd/api/streams/standard.c++ | 12 ++++++------ 8 files changed, 46 insertions(+), 21 deletions(-) diff --git a/build/deps/gen/deps.MODULE.bazel b/build/deps/gen/deps.MODULE.bazel index ec427dd2096..9213a5cb63c 100644 --- a/build/deps/gen/deps.MODULE.bazel +++ b/build/deps/gen/deps.MODULE.bazel @@ -27,10 +27,10 @@ bazel_dep(name = "brotli", version = "1.2.0.bcr.1") # capnp-cpp http.archive( name = "capnp-cpp", - sha256 = "94e5bcd03d14b8f1a1713e2477941e00b130b63b5a1aad97a10ff5edf05b6765", - strip_prefix = "capnproto-capnproto-e9fa5c7/c++", + sha256 = "6dae6cdfc776cadc94fe2a802186f94840329520cb53a9d02306175c06101f92", + strip_prefix = "capnproto-capnproto-6ef408c/c++", type = "tgz", - url = "https://github.com/capnproto/capnproto/tarball/e9fa5c7dc98192fc0dc0098ec770db68f997a938", + url = "https://github.com/capnproto/capnproto/tarball/6ef408c214210e3d19a1da6f695b23e782c8ffd8", ) use_repo(http, "capnp-cpp") diff --git a/src/workerd/api/streams/README.md b/src/workerd/api/streams/README.md index f01007f4393..bec3da97357 100644 --- a/src/workerd/api/streams/README.md +++ b/src/workerd/api/streams/README.md @@ -327,6 +327,29 @@ impl.controller->runIfAlive( [](ReadableByteStreamController& controller) { controller.maybeByobRequest = kj::none; }); ``` +### Pattern: Weak Owner Link + +- **When**: A controller or controller implementation needs to reach back to its owning + stream (`ReadableStreamController::setOwnerRef()`, `WritableImpl::tryGetOwner()`) +- **Why**: The owner holds the controller, so a strong back-reference can create a GC + tracing cycle or keep the stream alive only through its own controller. A raw + back-reference can also outlive the owner when async cleanup crosses ownership + boundaries. +- **How**: Store a weak owner reference during stream construction, then re-acquire + the owner only at the point of use. This is an internal owner back-link pattern, + distinct from weak refs used for handles exposed to user code. + +```cpp +ReadableStream::ReadableStream(kj::Own controller) + : controller(kj::mv(controller)) { + getController().setOwnerRef(addWeakToThis()); +} + +jsg::Ref ReadableStreamJsController::addRef() { + return owner.assertLive().addRef(); +} +``` + ### Pattern: `Rc` for Shared Queue Data - **When**: Queue entries shared across teed stream consumers diff --git a/src/workerd/api/streams/common.h b/src/workerd/api/streams/common.h index 758e10df2e6..ca45a854084 100644 --- a/src/workerd/api/streams/common.h +++ b/src/workerd/api/streams/common.h @@ -497,7 +497,7 @@ class ReadableStreamController { virtual ~ReadableStreamController() noexcept(false) {} - virtual void setOwnerRef(ReadableStream& stream) = 0; + virtual void setOwnerRef(kj::Weak stream) = 0; virtual jsg::Ref addRef() = 0; diff --git a/src/workerd/api/streams/internal.c++ b/src/workerd/api/streams/internal.c++ index 7e3a6f8945e..25691ca2683 100644 --- a/src/workerd/api/streams/internal.c++ +++ b/src/workerd/api/streams/internal.c++ @@ -437,7 +437,7 @@ ReadableStreamInternalController::~ReadableStreamInternalController() noexcept(f } jsg::Ref ReadableStreamInternalController::addRef() { - return KJ_ASSERT_NONNULL(owner).addRef(); + return owner.assertLive().addRef(); } kj::Maybe> ReadableStreamInternalController::read( @@ -604,7 +604,7 @@ kj::Maybe> ReadableStreamInternalController::read( controller.doClose(js); } KJ_IF_SOME(o, controller.owner) { - o.signalEof(js); + o->signalEof(js); } if (isByob && FeatureFlags::get(js).getInternalStreamByobReturn()) { // When using the BYOB reader, we must return a sized-0 Uint8Array that is backed @@ -766,7 +766,7 @@ kj::Maybe> ReadableStreamInternalController::dr controller.doClose(js); } KJ_IF_SOME(o, controller.owner) { - o.signalEof(js); + o->signalEof(js); } return js.resolvedPromise(DrainingReadResult{.done = true}); } @@ -803,8 +803,7 @@ jsg::Promise ReadableStreamInternalController::pipeTo( } disturbed = true; - KJ_IF_SOME(promise, - destination.tryPipeFrom(js, KJ_ASSERT_NONNULL(owner).addRef(), kj::mv(options))) { + KJ_IF_SOME(promise, destination.tryPipeFrom(js, owner.assertLive().addRef(), kj::mv(options))) { return kj::mv(promise); } @@ -2532,6 +2531,11 @@ kj::Maybe ReadableStreamInternalController::tryGetLength(StreamEncodin KJ_UNREACHABLE; } +void ReadableStreamInternalController::setOwnerRef(kj::Weak stream) { + KJ_ASSERT(owner == nullptr); + owner = kj::mv(stream); +} + kj::Own ReadableStreamInternalController::detach( jsg::Lock& js, bool ignoreDetached) { return newReadableStreamInternalController( diff --git a/src/workerd/api/streams/internal.h b/src/workerd/api/streams/internal.h index 787967dc4d8..99bc0ecfb6a 100644 --- a/src/workerd/api/streams/internal.h +++ b/src/workerd/api/streams/internal.h @@ -52,9 +52,7 @@ class ReadableStreamInternalController: public ReadableStreamController { ~ReadableStreamInternalController() noexcept(false) override; - void setOwnerRef(ReadableStream& stream) override { - owner = stream; - } + void setOwnerRef(kj::Weak stream) override; jsg::Ref addRef() override; @@ -153,7 +151,7 @@ class ReadableStreamInternalController: public ReadableStreamController { ReadableStreamInternalController& inner; }; - kj::Maybe owner; + kj::Weak owner; // State machine for ReadableStreamInternalController: // Closed is terminal, Errored is implicitly terminal via ErrorState. diff --git a/src/workerd/api/streams/readable.c++ b/src/workerd/api/streams/readable.c++ index 9490f02b9d1..2d9c7efe8d0 100644 --- a/src/workerd/api/streams/readable.c++ +++ b/src/workerd/api/streams/readable.c++ @@ -390,7 +390,7 @@ ReadableStream::ReadableStream(IoContext& ioContext, kj::Own controller) : ioContext(tryGetIoContext()), controller(kj::mv(controller)) { - getController().setOwnerRef(*this); + getController().setOwnerRef(addWeakToThis()); } void ReadableStream::visitForGc(jsg::GcVisitor& visitor) { diff --git a/src/workerd/api/streams/readable.h b/src/workerd/api/streams/readable.h index 61fd2a69b5f..f0a30992725 100644 --- a/src/workerd/api/streams/readable.h +++ b/src/workerd/api/streams/readable.h @@ -263,7 +263,7 @@ class DrainingReader: public ReadableStreamController::Reader { kj::Maybe>> closedPromise; }; -class ReadableStream: public jsg::Object { +class ReadableStream: public kj::PtrTarget, public jsg::Object { private: struct AsyncIteratorState { diff --git a/src/workerd/api/streams/standard.c++ b/src/workerd/api/streams/standard.c++ index ae704ab056c..45a47a3bbb6 100644 --- a/src/workerd/api/streams/standard.c++ +++ b/src/workerd/api/streams/standard.c++ @@ -785,7 +785,7 @@ class ReadableStreamJsController final: public ReadableStreamController { // See the comment for releaseReader in common.h for details on the use of maybeJs void releaseReader(Reader& reader, kj::Maybe maybeJs) override; - void setOwnerRef(ReadableStream& stream) override; + void setOwnerRef(kj::Weak stream) override; Tee tee(jsg::Lock& js) override; @@ -814,7 +814,7 @@ class ReadableStreamJsController final: public ReadableStreamController { // If the stream was created within the scope of a request, we want to treat it as I/O // and make sure it is not advanced from the scope of a different request. kj::Maybe ioContext; - kj::Maybe owner; + kj::Weak owner; // Initial state before setup() is called. struct Initial { @@ -2643,7 +2643,7 @@ ReadableStreamJsController::ReadableStreamJsController(jsg::Lock& js, ByteReadab } jsg::Ref ReadableStreamJsController::addRef() { - return KJ_REQUIRE_NONNULL(owner).addRef(); + return owner.assertLive().addRef(); } jsg::Promise ReadableStreamJsController::cancel( @@ -3024,9 +3024,9 @@ ReadableStreamController::Tee ReadableStreamJsController::tee(jsg::Lock& js) { KJ_UNREACHABLE; } -void ReadableStreamJsController::setOwnerRef(ReadableStream& stream) { - KJ_ASSERT(owner == kj::none); - owner = &stream; +void ReadableStreamJsController::setOwnerRef(kj::Weak stream) { + KJ_ASSERT(owner == nullptr); + owner = kj::mv(stream); } void ReadableStreamJsController::setup(jsg::Lock& js, From 98033f66f75e473308d5713522ceeb5269bf872f Mon Sep 17 00:00:00 2001 From: Mike Aizatsky Date: Thu, 2 Jul 2026 10:30:33 -0700 Subject: [PATCH 017/101] allocate Pipe on the heap we maintain weak reference to it, allow it to have its identity. --- src/workerd/api/streams/internal.c++ | 98 ++++++++++++++++------------ src/workerd/api/streams/internal.h | 46 ++++--------- 2 files changed, 69 insertions(+), 75 deletions(-) diff --git a/src/workerd/api/streams/internal.c++ b/src/workerd/api/streams/internal.c++ index 7e3a6f8945e..d30e10d952d 100644 --- a/src/workerd/api/streams/internal.c++ +++ b/src/workerd/api/streams/internal.c++ @@ -1402,7 +1402,7 @@ kj::Maybe> WritableStreamInternalController::tryPipeFrom( } queue.push_back(WriteEvent{ .outputLock = IoContext::current().waitForOutputLocksIfNecessaryIoOwn(), - .event = Pipe(*this, sourceLock, kj::mv(prp.resolver), preventAbort, preventClose, + .event = kj::heap(*this, sourceLock, kj::mv(prp.resolver), preventAbort, preventClose, preventCancel, kj::mv(options.signal)), }); ensureWriting(js); @@ -1634,7 +1634,7 @@ jsg::Promise WritableStreamInternalController::writeLoopAfterFrontOutputLo KJ_CASE_ONEOF(close, Close) { events.add(kj::str("Close")); } - KJ_CASE_ONEOF(pipe, Pipe) { + KJ_CASE_ONEOF(pipe, kj::Own) { events.add(kj::str("Pipe")); } } @@ -1663,7 +1663,7 @@ jsg::Promise WritableStreamInternalController::writeLoopAfterFrontOutputLo if constexpr (kj::isSameType()) { // Pipe and Close requests are always the last one in the queue. KJ_ASSERT(queue.size() == 1, queue.size(), inspectQueue(queue)); - } else if constexpr (kj::isSameType()) { + } else if constexpr (kj::isSameType>()) { // Pipe and Close requests are always the last one in the queue. KJ_ASSERT(queue.size() == 1, queue.size(), inspectQueue(queue)); } @@ -1761,8 +1761,8 @@ jsg::Promise WritableStreamInternalController::writeLoopAfterFrontOutputLo return js.resolvedPromise(); })); } - KJ_CASE_ONEOF(request, Pipe) { - if (request.checkSignal(js)) { + KJ_CASE_ONEOF(request, kj::Own) { + if (request->checkSignal(js)) { // If the signal is triggered, checkSignal will handle erroring the source and destination. return js.resolvedPromise(); } @@ -1770,15 +1770,15 @@ jsg::Promise WritableStreamInternalController::writeLoopAfterFrontOutputLo // The readable side should *should* still be readable here but let's double check, just // to be safe, both for closed state and errored states. We just constructed the Pipe // and haven't yet entered pipeLoop, so source is guaranteed non-null. - auto& sourceRef = KJ_ASSERT_NONNULL(request.source); - auto preventClose = request.flags.preventClose; - auto preventAbort = request.flags.preventAbort; + auto& sourceRef = KJ_ASSERT_NONNULL(request->source); + auto preventClose = request->flags.preventClose; + auto preventAbort = request->flags.preventAbort; if (sourceRef.isClosed()) { // Resolve the pipe promise before pop_front destroys the Pipe event. - auto promise = request.takePromise(); + auto promise = request->takePromise(); maybeResolvePromise(js, promise); - request.releaseSource(js); + request->releaseSource(js); // Pop the Pipe from the queue before calling close() — isPiping() // checks the queue, and close() rejects if isPiping() is true. queue.pop_front(); @@ -1796,9 +1796,9 @@ jsg::Promise WritableStreamInternalController::writeLoopAfterFrontOutputLo KJ_IF_SOME(errored, sourceRef.tryGetErrored(js)) { // Reject the pipe promise before pop_front destroys the Pipe event. - auto promise = request.takePromise(); + auto promise = request->takePromise(); maybeRejectPromise(js, promise, errored); - request.releaseSource(js); + request->releaseSource(js); // Pop the Pipe from the queue before further processing — the source // has been released, so the Pipe entry is stale. queue.pop_front(); @@ -1833,14 +1833,14 @@ jsg::Promise WritableStreamInternalController::writeLoopAfterFrontOutputLo // Under some conditions, the clean up has already happened. if (controller.queue.empty()) return js.resolvedPromise(); - auto& request = check.template operator()(controller); + auto& request = check.template operator()>(controller); // KJ_IF_SOME on request.source(): if pipeLoop already released the // source (via Pipe::State::releaseSource()), source is now // kj::none and we MUST NOT attempt a deref. Use the stashed // capturedSourceError in that case. - KJ_IF_SOME(sourceRef, request.source) { - auto fulfiller = request.takePromise(); + KJ_IF_SOME(sourceRef, request->source) { + auto fulfiller = request->takePromise(); KJ_IF_SOME(errored, sourceRef.tryGetErrored(js)) { if (preventAbort) preventClose = true; // Even through we're not going to close the destination, we still want the @@ -1863,12 +1863,12 @@ jsg::Promise WritableStreamInternalController::writeLoopAfterFrontOutputLo // path, the loop detects isClosed() and releases on its next iteration. // But the KJ tryPumpTo path has no loop — handlePromise is the terminal // handler — so we must release explicitly here. - request.releaseSource(js); + request->releaseSource(js); } else { // pipeLoop already released the source; consult the stashed // error value (if any) rather than dereferencing source. - auto promise = request.takePromise(); - KJ_IF_SOME(err, request.capturedSourceError) { + auto promise = request->takePromise(); + KJ_IF_SOME(err, request->capturedSourceError) { if (preventAbort) preventClose = true; maybeRejectPromise(js, promise, err.getHandle(js)); } else KJ_IF_SOME(errored, controller.state.tryGetUnsafe()) { @@ -1902,13 +1902,13 @@ jsg::Promise WritableStreamInternalController::writeLoopAfterFrontOutputLo auto handle = jsg::JsValue(reason.getHandle(js)); - auto& request = check.template operator()(controller); + auto& request = check.template operator()>(controller); - auto fulfiller = request.takePromise(); + auto fulfiller = request->takePromise(); maybeRejectPromise(js, fulfiller, handle); // KJ_IF_SOME on request.source(): if pipeLoop already released the // source, skip — the underlying PipeController is gone. - KJ_IF_SOME(sourceRef, request.source) { + KJ_IF_SOME(sourceRef, request->source) { // TODO(conform): Remember all those checks we performed in ReadableStream::pipeTo()? // We're supposed to perform the same checks continually, e.g., errored writes should // cancel the readable side unless preventCancel is truthy... This would require @@ -1920,7 +1920,7 @@ jsg::Promise WritableStreamInternalController::writeLoopAfterFrontOutputLo // Release the readable's pipe lock — same rationale as the success // path: the KJ tryPumpTo path has no loop iteration to detect the // error and release. - request.releaseSource(js); + request->releaseSource(js); } controller.queue.pop_front(); if (!preventAbort) { @@ -1941,14 +1941,14 @@ jsg::Promise WritableStreamInternalController::writeLoopAfterFrontOutputLo return handlePromise(js, ioContext.awaitIo(js, writable->canceler.wrap( - AbortSignal::maybeCancelWrap(js, request.maybeSignal, kj::mv(promise))))); + AbortSignal::maybeCancelWrap(js, request->maybeSignal, kj::mv(promise))))); } else { // The ReadableStream is JavaScript-backed. We can still pipe the data but it's going to be // a bit slower because we will be relying on JavaScript promises when reading the data // from the ReadableStream, then waiting on kj::Promises to write the data. We will keep // reading until either the source or destination errors or until the source signals that // it is done. - return handlePromise(js, request.pipeLoop(js)); + return handlePromise(js, request->pipeLoop(js)); } })); } @@ -2003,7 +2003,9 @@ jsg::Promise WritableStreamInternalController::writeLoopAfterFrontOutputLo bool WritableStreamInternalController::Pipe::State::checkSignal(jsg::Lock& js) { // If the weakRef is not alive, we'll return true to indicate aborted. bool answer = true; - weakRef->runIfAlive([&](Pipe& ref) { answer = ref.checkSignal(js); }); + KJ_IF_SOME(ref, weakRef) { + answer = ref->checkSignal(js); + } return answer; } @@ -2020,7 +2022,6 @@ bool WritableStreamInternalController::Pipe::checkSignal(jsg::Lock& js) { auto preventCancel = flags.preventCancel; auto preventAbort = flags.preventAbort; auto promiseCopy = kj::mv(promise); - auto weakRef = kj::mv(selfRef); // Before the drain, keep the readable alive so sourceRef stays valid auto readableRef = [&]() -> kj::Maybe> { @@ -2052,7 +2053,6 @@ bool WritableStreamInternalController::Pipe::checkSignal(jsg::Lock& js) { } maybeRejectPromise(js, promiseCopy, reason); - KJ_ASSERT_NONNULL(weakRef)->invalidate(); return true; } } @@ -2062,7 +2062,9 @@ bool WritableStreamInternalController::Pipe::checkSignal(jsg::Lock& js) { jsg::Promise WritableStreamInternalController::Pipe::State::write( jsg::Lock& js, jsg::JsValue handle) { kj::Maybe> promise; - weakRef->runIfAlive([&](auto& ref) { promise = ref.write(js, handle); }); + KJ_IF_SOME(ref, weakRef) { + promise = ref->write(js, handle); + } KJ_IF_SOME(p, promise) { return kj::mv(p); } @@ -2101,13 +2103,17 @@ jsg::Promise WritableStreamInternalController::Pipe::write( bool WritableStreamInternalController::Pipe::State::isSourceReleased() { bool answer = true; - weakRef->runIfAlive([&](auto& ref) { answer = ref.isSourceReleased(); }); + KJ_IF_SOME(ref, weakRef) { + answer = ref->isSourceReleased(); + } return answer; } void WritableStreamInternalController::Pipe::State::tryErrorParent( jsg::Lock& js, jsg::JsValue reason) { - weakRef->runIfAlive([&](auto& ref) { ref.errorParent(js, reason); }); + KJ_IF_SOME(ref, weakRef) { + ref->errorParent(js, reason); + } } void WritableStreamInternalController::Pipe::errorParent(jsg::Lock& js, jsg::JsValue reason) { @@ -2115,12 +2121,16 @@ void WritableStreamInternalController::Pipe::errorParent(jsg::Lock& js, jsg::JsV } void WritableStreamInternalController::Pipe::State::tryFinishCloseParent(jsg::Lock& js) { - weakRef->runIfAlive([&](auto& ref) { ref.finishCloseParent(js); }); + KJ_IF_SOME(ref, weakRef) { + ref->finishCloseParent(js); + } } void WritableStreamInternalController::Pipe::State::tryFinishErrorParent( jsg::Lock& js, jsg::JsValue reason) { - weakRef->runIfAlive([&](auto& ref) { ref.finishErrorParent(js, reason); }); + KJ_IF_SOME(ref, weakRef) { + ref->finishErrorParent(js, reason); + } } void WritableStreamInternalController::Pipe::finishCloseParent(jsg::Lock& js) { @@ -2132,7 +2142,9 @@ void WritableStreamInternalController::Pipe::finishErrorParent(jsg::Lock& js, js } void WritableStreamInternalController::Pipe::State::tryNoBytesError(jsg::Lock& js) { - weakRef->runIfAlive([&](auto& ref) { ref.noBytesError(js); }); + KJ_IF_SOME(ref, weakRef) { + ref->noBytesError(js); + } } void WritableStreamInternalController::Pipe::noBytesError(jsg::Lock& js) { @@ -2144,7 +2156,9 @@ void WritableStreamInternalController::Pipe::noBytesError(jsg::Lock& js) { jsg::Promise WritableStreamInternalController::Pipe::State::pipeLoop(jsg::Lock& js) { kj::Maybe> promise; - weakRef->runIfAlive([&](auto& ref) { promise = ref.pipeLoop(js); }); + KJ_IF_SOME(ref, weakRef) { + promise = ref->pipeLoop(js); + } KJ_IF_SOME(p, promise) { return kj::mv(p); } @@ -2320,7 +2334,9 @@ jsg::Promise WritableStreamInternalController::Pipe::pipeLoop(jsg::Lock& j void WritableStreamInternalController::Pipe::State::releaseSource( jsg::Lock& js, kj::Maybe maybeError) { - weakRef->runIfAlive([&](auto& ref) { ref.releaseSource(js, kj::mv(maybeError)); }); + KJ_IF_SOME(ref, weakRef) { + ref->releaseSource(js, kj::mv(maybeError)); + } } void WritableStreamInternalController::Pipe::releaseSource( @@ -2350,13 +2366,13 @@ void WritableStreamInternalController::drain(jsg::Lock& js, jsg::JsValue reason) auto promise = kj::mv(writeRequest.promise); maybeRejectPromise(js, promise, reason); } - KJ_CASE_ONEOF(pipeRequest, Pipe) { - if (!pipeRequest.flags.preventCancel) { - KJ_IF_SOME(sourceRef, pipeRequest.source) { + KJ_CASE_ONEOF(pipeRequest, kj::Own) { + if (!pipeRequest->flags.preventCancel) { + KJ_IF_SOME(sourceRef, pipeRequest->source) { sourceRef.cancel(js, reason); } } - auto promise = pipeRequest.takePromise(); + auto promise = pipeRequest->takePromise(); maybeRejectPromise(js, promise, reason); } KJ_CASE_ONEOF(closeRequest, Close) { @@ -2384,8 +2400,8 @@ void WritableStreamInternalController::visitForGc(jsg::GcVisitor& visitor) { KJ_CASE_ONEOF(flush, Flush) { visitor.visit(flush.promise); } - KJ_CASE_ONEOF(pipe, Pipe) { - pipe.visitForGc(visitor); + KJ_CASE_ONEOF(pipe, kj::Own) { + pipe->visitForGc(visitor); } } } diff --git a/src/workerd/api/streams/internal.h b/src/workerd/api/streams/internal.h index 787967dc4d8..a8b41bbeee2 100644 --- a/src/workerd/api/streams/internal.h +++ b/src/workerd/api/streams/internal.h @@ -371,17 +371,17 @@ class WritableStreamInternalController: public WritableStreamController { tracker.trackField("promise", promise); } }; - struct Pipe { + struct Pipe: kj::PtrTarget { struct State: public kj::Refcounted { jsg::Ref owner; - kj::Rc> weakRef; + kj::Weak weakRef; - State(jsg::Ref owner, kj::Rc> weakRef) + State(jsg::Ref owner, kj::Weak weakRef) : owner(kj::mv(owner)), weakRef(kj::mv(weakRef)) {} inline bool isAborted() const { - return !weakRef->isValid(); + return weakRef == nullptr; } bool checkSignal(jsg::Lock& js); jsg::Promise pipeLoop(jsg::Lock& js); @@ -405,7 +405,6 @@ class WritableStreamInternalController: public WritableStreamController { Flags flags{}; kj::Maybe> maybeSignal; kj::Maybe> capturedSourceError; - kj::Maybe>> selfRef; Pipe(WritableStreamInternalController& parent, ReadableStreamController::PipeController& source, @@ -417,39 +416,18 @@ class WritableStreamInternalController: public WritableStreamController { : parent(parent), source(source), promise(kj::mv(promise)), - maybeSignal(kj::mv(maybeSignal)), - selfRef(kj::rc>(kj::Badge(), *this)) { + maybeSignal(kj::mv(maybeSignal)) { flags.preventAbort = preventAbort; flags.preventClose = preventClose; flags.preventCancel = preventCancel; } - Pipe(Pipe&& other) noexcept(false) - : parent(other.parent), - source(kj::mv(other.source)), - promise(kj::mv(other.promise)), - flags(other.flags), - maybeSignal(kj::mv(other.maybeSignal)), - capturedSourceError(kj::mv(other.capturedSourceError)), - selfRef(kj::rc>(kj::Badge(), *this)) { - // Invalidate the old Pipe's weak ref — any State objects pointing to it - // will see isAborted() = true. - KJ_IF_SOME(ref, other.selfRef) { - ref->invalidate(); - other.selfRef = kj::none; - } - } - - ~Pipe() noexcept(false) { - KJ_IF_SOME(ref, selfRef) { - ref->invalidate(); - } - } + ~Pipe() noexcept(false) {} - KJ_DISALLOW_COPY(Pipe); + KJ_DISALLOW_COPY_AND_MOVE(Pipe); kj::Rc getState() { - return kj::rc(parent.addRef(), KJ_ASSERT_NONNULL(selfRef).addRef()); + return kj::rc(parent.addRef(), addWeakToThis()); } void visitForGc(jsg::GcVisitor& visitor) { @@ -479,14 +457,14 @@ class WritableStreamInternalController: public WritableStreamController { }; struct WriteEvent { kj::Maybe>> outputLock; // must wait for this before actually writing - kj::OneOf event; + kj::OneOf, Close, Flush> event; bool isCloseOrFlush() const { return event.is() || event.is(); } bool isPipe() const { - return event.is(); + return event.is>(); } JSG_MEMORY_INFO(WriteEvent) { @@ -497,8 +475,8 @@ class WritableStreamInternalController: public WritableStreamController { KJ_CASE_ONEOF(w, Write) { tracker.trackField("inner", w); } - KJ_CASE_ONEOF(p, Pipe) { - tracker.trackField("inner", p); + KJ_CASE_ONEOF(p, kj::Own) { + tracker.trackField("inner", *p); } KJ_CASE_ONEOF(c, Close) { tracker.trackField("inner", c); From 69cb7bb29571315d4da1f5b0b16357f5f906c07b Mon Sep 17 00:00:00 2001 From: Felix Hanau Date: Thu, 2 Jul 2026 21:43:20 -0400 Subject: [PATCH 018/101] Clean up JsRpcProperty MAX_PROPERTY_DEPTH --- src/workerd/api/tests/js-rpc-test.js | 18 +++++++++--------- src/workerd/api/worker-rpc.c++ | 3 --- src/workerd/api/worker-rpc.h | 3 +-- 3 files changed, 10 insertions(+), 14 deletions(-) diff --git a/src/workerd/api/tests/js-rpc-test.js b/src/workerd/api/tests/js-rpc-test.js index e4c35eabfe2..a5f237dc121 100644 --- a/src/workerd/api/tests/js-rpc-test.js +++ b/src/workerd/api/tests/js-rpc-test.js @@ -2200,7 +2200,7 @@ export let eOrderTest = { // Unbounded JsRpcProperty parent chain causes native stack overflow // (SIGSEGV) on destruction. Building a deep chain of pipelined // property accesses must be rejected once the depth exceeds -// MAX_PROPERTY_DEPTH (5120). +// MAX_PROPERTY_DEPTH (64). export let stubDepthLimitTest = { async test() { // Create a local RPC stub wrapping a plain object. @@ -2210,12 +2210,12 @@ export let stubDepthLimitTest = { // this would create an unbounded linked list of native // JsRpcProperty objects whose recursive destruction overflows // the native stack. After the fix, getProperty() throws a - // TypeError once depth >= 5120. + // TypeError once depth >= 64. let p = stub; let threw = false; let depthReached = 0; try { - for (let i = 0; i < 10000; i++) { + for (let i = 0; i < 100; i++) { p = p.x; depthReached = i + 1; } @@ -2236,16 +2236,16 @@ export let stubDepthLimitTest = { 'Expected TypeError to be thrown at depth limit, ' + `but reached depth ${depthReached} without error` ); - // The depth limit is 5120, so we should have reached at least 5120 + // The depth limit is 64, so we should have reached at least 64 // before the throw. assert.ok( - depthReached >= 5120, - `Expected to reach at least depth 5120, only reached ${depthReached}` + depthReached >= 64, + `Expected to reach at least depth 64, only reached ${depthReached}` ); - // And we should NOT have reached 10000 (the full loop). + // And we should NOT have reached 100 (the full loop). assert.ok( - depthReached < 10000, - 'Should not have reached depth 10000 without error' + depthReached < 100, + 'Should not have reached depth 100 without error' ); }, }; diff --git a/src/workerd/api/worker-rpc.c++ b/src/workerd/api/worker-rpc.c++ index 36d0a0606d5..fade4977092 100644 --- a/src/workerd/api/worker-rpc.c++ +++ b/src/workerd/api/worker-rpc.c++ @@ -541,9 +541,6 @@ jsg::JsValue JsRpcPromise::finally(jsg::Lock& js, v8::Local onFina } kj::Maybe> JsRpcProperty::getProperty(jsg::Lock& js, kj::String name) { - if (depth >= MAX_PROPERTY_WARNING_DEPTH) { - LOG_PERIODICALLY(WARNING, "NOSENTRY VULN-136589 exceeded RPC property warning depth", depth); - } JSG_REQUIRE(depth < MAX_PROPERTY_DEPTH, TypeError, "RPC pipelined property path is too deep (max ", MAX_PROPERTY_DEPTH, ")."); return js.alloc(JSG_THIS, kj::mv(name), depth + 1); diff --git a/src/workerd/api/worker-rpc.h b/src/workerd/api/worker-rpc.h index 9404072b958..6cc4a4cb226 100644 --- a/src/workerd/api/worker-rpc.h +++ b/src/workerd/api/worker-rpc.h @@ -270,8 +270,7 @@ class JsRpcProperty: public JsRpcClientProvider { // Maximum depth of pipelined property chains. Prevents stack overflow when a chain of // JsRpcProperty objects is destructed recursively. 64 is beyond any legitimate RPC pipelining // depth. - static constexpr uint MAX_PROPERTY_DEPTH = 5120; - static constexpr uint MAX_PROPERTY_WARNING_DEPTH = 64; + static constexpr uint MAX_PROPERTY_DEPTH = 64; JsRpcProperty(jsg::Ref parent, kj::String name, uint depth = 0) : parent(kj::mv(parent)), From 71f80366f9f12377a646da352a1450bd56896f2a Mon Sep 17 00:00:00 2001 From: Ashley Peacock Date: Sat, 27 Jun 2026 17:47:44 +0100 Subject: [PATCH 019/101] STOR-5304: Add actor-call delivery-position exception detail IDs Define two exception DetailTypeIds in jsg/util.h that record whether a DISCONNECTED actor-call failure happened before (ACTOR_CALL_NOT_DELIVERED_TO_SANDBOX_DETAIL_ID) or after (ACTOR_CALL_DELIVERED_TO_SANDBOX_DETAIL_ID) the call reached the receiving sandbox, so the caller can distinguish failures that are safe to retry as a fresh attempt from those that may have run user code. Set the DELIVERED detail in WorkerEntrypoint's actor exception catch (request + connect) before exceptionToPropagate(), so it survives the internal-exception description rewrite and serializes back across the RPC boundary. No behavior change; the detail is currently only read for metrics classification. Release note: None --- src/workerd/io/worker-entrypoint.c++ | 22 ++++++++++++++++++++++ src/workerd/jsg/util.h | 14 ++++++++++++++ 2 files changed, 36 insertions(+) diff --git a/src/workerd/io/worker-entrypoint.c++ b/src/workerd/io/worker-entrypoint.c++ index af3441e1503..dd70f47d3d8 100644 --- a/src/workerd/io/worker-entrypoint.c++ +++ b/src/workerd/io/worker-entrypoint.c++ @@ -564,6 +564,16 @@ kj::Promise WorkerEntrypoint::requestImpl(kj::HttpMethod method, // 4. Otherwise -> synthesize a 5xx response. if (isActor) { + // Reaching this catch means the request reached the actor (we are past + // `delivered()` in Stage 1), so user code may have run. Annotate DISCONNECTED failures so the + // caller-side actor-call classifier knows this failure must not be retried as a fresh + // delivery. Only DISCONNECTED failures participate in the delivery-position metric, so other + // exception types need no annotation. Set before exceptionToPropagate() so it survives the + // internal-exception description rewrite; the detail serializes back across the RPC boundary. + if (exception.getType() == kj::Exception::Type::DISCONNECTED) { + exception.setDetail( + jsg::REQUEST_DELIVERED_TO_ACTOR_DETAIL_ID, kj::heapArray(0)); + } // TODO(cleanup): We'd really like to tunnel exceptions any time a worker is calling another // worker, not just for actors (and W2W below), but getting that right will require cleaning // up error handling more generally. @@ -713,6 +723,18 @@ kj::Promise WorkerEntrypoint::connect(kj::StringPtr host, } if (isActor || tunnelExceptions) { + if (isActor) { + // Reaching this catch means the request reached the actor (we are past + // `delivered()` above), so user code may have run. Annotate DISCONNECTED failures so the + // caller-side actor-call classifier knows this failure must not be retried as a fresh + // delivery. Only DISCONNECTED failures participate in the delivery-position metric, so other + // exception types need no annotation. Set before exceptionToPropagate() so it survives the + // internal-exception description rewrite; the detail serializes back across the RPC boundary. + if (exception.getType() == kj::Exception::Type::DISCONNECTED) { + exception.setDetail( + jsg::REQUEST_DELIVERED_TO_ACTOR_DETAIL_ID, kj::heapArray(0)); + } + } // We want to tunnel exceptions from actors back to the caller. // TODO(cleanup): We'd really like to tunnel exceptions any time a worker is calling another // worker, not just for actors (and W2W below), but getting that right will require cleaning diff --git a/src/workerd/jsg/util.h b/src/workerd/jsg/util.h index 1bf773ec2ce..87fe907a928 100644 --- a/src/workerd/jsg/util.h +++ b/src/workerd/jsg/util.h @@ -95,6 +95,20 @@ constexpr kj::Exception::DetailTypeId TUNNELED_EXCEPTION_DETAIL_ID = 0xe80272921 // Detail type for JavaScript exception metadata (error type and stack trace) constexpr kj::Exception::DetailTypeId JS_EXCEPTION_METADATA_DETAIL_ID = 0xa9ae63464030fcefull; +// Set on a DISCONNECTED actor-call failure that is known to have occurred BEFORE the call reached +// the actor (i.e. user code definitely never ran). Such failures are safe to retry as a fresh +// attempt. Set by edgeworker's pre-delivery routing/getActor failure paths; read by the caller-side +// actor-call classifier. The payload is a zero-length array (marker only). +constexpr kj::Exception::DetailTypeId REQUEST_NOT_DELIVERED_TO_ACTOR_DETAIL_ID = + 0x1a07d0b6559baea6ull; + +// Set on a failure that is known to have occurred AFTER the call reached the actor (user code may +// have run). Must not be retried as a delivery failure. Set at the receiving entrypoint's actor +// catch so it survives the internal-exception description rewrite and serializes back across the RPC +// boundary. The payload is a zero-length array (marker only). +constexpr kj::Exception::DetailTypeId REQUEST_DELIVERED_TO_ACTOR_DETAIL_ID = + 0x7f6e0bece261e8eeull; + // Add a serialized copy of the exception value to the KJ exception, as a "detail". void addExceptionDetail(Lock& js, kj::Exception& exception, v8::Local handle); From 8cf8bb7488928ff0196e8d34a29904931dbdf739 Mon Sep 17 00:00:00 2001 From: Florent Collin Date: Wed, 1 Jul 2026 17:48:44 +0100 Subject: [PATCH 020/101] CFSQL-1705: D1 add support for the JSRPC `.query` method Adds the `d1_binding_jsrpc` compatibility flag and routes D1 query requests through the JSRPC query API when enabled. Keeps the legacy calls to the `fetch` function when the flag is disabled. --- src/cloudflare/internal/d1-api.ts | 353 +++++++++++++++++++----- src/workerd/io/compatibility-date.capnp | 8 +- 2 files changed, 296 insertions(+), 65 deletions(-) diff --git a/src/cloudflare/internal/d1-api.ts b/src/cloudflare/internal/d1-api.ts index 31a8d0e94d2..0b195310c66 100644 --- a/src/cloudflare/internal/d1-api.ts +++ b/src/cloudflare/internal/d1-api.ts @@ -5,14 +5,55 @@ import { withSpan } from 'cloudflare-internal:tracing-helpers'; import type { Span } from './tracing'; -interface D1Meta { - duration: number; - size_after: number; - rows_read: number; - rows_written: number; - last_row_id: number; - changed_db: boolean; - changes: number; +const d1BindingJsrpc = !!Cloudflare.compatibilityFlags['d1_binding_jsrpc']; + +///////////////////////////////////////////////////////////////////////////// +// Binding contract to create a custom binding // +///////////////////////////////////////////////////////////////////////////// +type D1BindingService = { + query(params: QueryRequest): Promise; +}; + +type D1SessionBookmark = string; +type D1SessionBookmarkOrConstraint = string; + +type QueryRequest = { + queries: QuerySql[]; + bookmark?: D1SessionBookmarkOrConstraint; +}; + +type QueryResponse = RpcResponse<{ + queryResults: QuerySqlResult[]; + bookmark?: D1SessionBookmark; +}>; + +type QuerySql = { + sql: string; + params?: unknown[]; +}; + +type QuerySqlResult = { + /** Metadata for this query. */ + meta: D1Meta; + data: { + /** Discriminator for the query result payload shape. */ + kind: 'raw'; + /** Column names for the returned rows. */ + columns: string[]; + /** Opaque row payload consumed by the D1 wrapper. */ + rows: unknown[][]; + }; +}; + +type RpcResponse = + | { success: true; results: T } + | { success: false; error: Error }; + +type D1Meta = { + /** + * True if-and-only-if the database instance that executed the query was the primary. + */ + served_by_primary?: boolean; /** * The region of the database instance that executed the query. @@ -24,26 +65,31 @@ interface D1Meta { */ served_by_colo?: string; - /** - * True if-and-only-if the database instance that executed the query was the primary. - */ - served_by_primary?: boolean; - timings?: { /** * The duration of the SQL query execution by the database instance. It doesn't include any network time. */ sql_duration_ms: number; }; + duration: number; + rows_read: number; + rows_written: number; + changes: number; + last_row_id: number; + changed_db: boolean; + size_after: number; /** * Number of total attempts to execute the query, due to automatic retries. * Note: All other fields in the response like `timings` only apply to the last attempt. */ total_attempts?: number; -} +}; +///////////////////////////////////////////////////////////////////////////// +// End of the binding contract // +///////////////////////////////////////////////////////////////////////////// -interface Fetcher { +interface Fetcher extends D1BindingService { fetch: typeof fetch; } @@ -93,8 +139,6 @@ type SQLError = { type ResultsFormat = 'ARRAY_OF_OBJECTS' | 'ROWS_AND_COLUMNS' | 'NONE'; -type D1SessionBookmarkOrConstraint = string; -type D1SessionBookmark = string; // Indicates that the first query should go to the primary, and the rest queries // using the same D1DatabaseSession will go to any replica that is consistent with // the bookmark maintained by the session (returned by the first query). @@ -147,6 +191,33 @@ class D1Database { return new D1DatabaseSession(this.fetcher, constraintOrBookmark); } + /** + * Send one or more SQL queries directly to D1 and return raw rows-and-columns results. + * + * Unlike `prepare().all()`, `prepare().raw()`, `prepare().first()`, or + * `batch()`, this method does not reshape rows into objects or apply the + * public convenience return types. It returns the lower-level D1 response + * format so callers can inspect result metadata and manage session bookmarks + * directly. + * + * @example + * ``` + * await env.D1.query({ + * queries: [{ sql: "select * from users where id=?", params: [1] }], + * bookmark: "" + * }); + * ``` + */ + async query(request: QueryRequest): Promise { + if (!d1BindingJsrpc) { + const message = + 'D1Database.query() requires the d1_binding_jsrpc compatibility flag. Enable d1_binding_jsrpc before calling query().'; + throw new Error(`D1_ERROR: ${message}`); + } + + return await fetcherQuery(this.fetcher, request); + } + /** * @deprecated */ @@ -212,7 +283,10 @@ class D1DatabaseSession { span.setAttribute('db.operation.name', 'batch'); span.setAttribute( 'db.query.text', - statements.map((s: D1PreparedStatement) => s.statement).join('\n') + // TODO: Joining all the statement without truncating it can lead to truncated + // span tags. We should find out if that's the case and where is the limit, and + // then decide if we need to truncate it ourselves. + statements.map((s) => s.statement).join('\n') ); span.setAttribute('db.operation.batch.size', statements.length); span.setAttribute('cloudflare.binding.type', 'D1'); @@ -221,13 +295,23 @@ class D1DatabaseSession { this.getBookmark() ?? undefined ); - const exec = (await this._sendOrThrow( - '/query', - statements.map((s: D1PreparedStatement) => s.statement), - statements.map((s: D1PreparedStatement) => s.params), - 'ROWS_AND_COLUMNS', - span - )) as D1UpstreamSuccess[]; + const exec = (d1BindingJsrpc + ? await this._queryOrThrow( + { + queries: statements.map((s) => ({ + sql: s.statement, + params: s.params, + })), + }, + span + ) + : await this._sendOrThrow( + '/query', + statements.map((s) => s.statement), + statements.map((s) => s.params), + 'ROWS_AND_COLUMNS', + span + )) as D1UpstreamSuccess[]; span.setAttribute( 'cloudflare.d1.response.bookmark', @@ -362,6 +446,60 @@ class D1DatabaseSession { }); } } + + async _queryOrThrow( + request: QueryRequest, + span: Span + ): Promise { + const results = await this._query(request, span); + for (const result of results) { + if (!result.success) { + span.setAttribute('error.type', result.error); + throw new Error(`D1_ERROR: ${result.error}`); + } + } + // The loop above rejects every failure result; TypeScript cannot narrow the + // array itself after checking each element. It's fine to cast here to avoid + // the allocation of a new array. + return results as D1RowsColumns[]; + } + + async _query( + request: QueryRequest, + span: Span + ): Promise> { + try { + const queryParams: QueryRequest = this.bookmarkOrConstraint + ? { ...request, bookmark: this.bookmarkOrConstraint } + : request; + const response = await fetcherQuery(this.fetcher, queryParams); + + if (!response.success) { + return [ + { + success: false, + error: response.error.message, + }, + ]; + } + + if (response.results.bookmark) { + this._updateBookmark(response.results.bookmark); + } + + return response.results.queryResults.map(mapQueryResult); + } catch (e: unknown) { + // TODO(cleanup): Split protocol validation from transport error wrapping so + // internal D1_ERRORs thrown above do not need this guard to avoid being + // double-wrapped as `D1_ERROR: Error: D1_ERROR: ...`. + if (e instanceof Error && e.message.startsWith('D1_ERROR:')) { + throw e; + } + const message = String(e); + span.setAttribute('error.type', message); + throw new Error(`D1_ERROR: ${message}`, { cause: e }); + } + } } class D1DatabaseSessionAlwaysPrimary extends D1DatabaseSession { @@ -401,25 +539,37 @@ class D1DatabaseSessionAlwaysPrimary extends D1DatabaseSession { // can contain multiple statements (ex: `select 1; select 2;`). // Also, a statement can span multiple lines... // Either, we should do a more reasonable job to split the query into multiple statements - // like we do in the D1 codebase or we report a simpler error without the line number. + // like we do in the D1 codebase. const lines = query.trim().split('\n'); - const _exec = await this._send('/execute', lines, [], 'NONE', span); - const exec = Array.isArray(_exec) ? _exec : [_exec]; + const execResults = d1BindingJsrpc + ? await this._query( + { + queries: lines.map((sql) => ({ sql })), + }, + span + ) + : await this._send('/execute', lines, [], 'NONE', span); + const results = Array.isArray(execResults) ? execResults : [execResults]; let duration = 0; const metas: D1Meta[] = []; - for (let i = 0; i < exec.length; i++) { - const res = exec[i]; - if (!res?.success) { - span.setAttribute('error.type', `Error in line ${i + 1}`); - throw new Error( - `D1_EXEC_ERROR: Error in line ${i + 1}: ${lines[i]}${res?.error ? `: ${res.error}` : ''}`, - { - cause: new Error( - `Error in line ${i + 1}: ${lines[i]}${res?.error ? `: ${res.error}` : ''}` - ), - } - ); + for (const [i, res] of results.entries()) { + if (!res.success) { + const message = res.error || 'Something went wrong'; + if (!d1BindingJsrpc) { + // Keep the old behavior even if the line number is always going to be 1 + // because the DO is only returning the first error only. + const lineNumber = i + 1; + const line = lines[lineNumber - 1]; + const legacyMessage = `Error in line ${lineNumber}: ${line}: ${message}`; + span.setAttribute('error.type', `Error in line ${lineNumber}`); + throw new Error(`D1_EXEC_ERROR: ${legacyMessage}`, { + cause: new Error(legacyMessage), + }); + } + + span.setAttribute('error.type', message); + throw new Error(`D1_EXEC_ERROR: ${message}`); } duration += res.meta.duration; @@ -430,7 +580,7 @@ class D1DatabaseSessionAlwaysPrimary extends D1DatabaseSession { addAggregatedD1MetaToSpan(span, metas); } return { - count: exec.length, + count: results.length, duration, }; }); @@ -441,6 +591,10 @@ class D1DatabaseSessionAlwaysPrimary extends D1DatabaseSession { * Only applies to the deprecated v1 alpha databases. */ async dump(): Promise { + // dump() is no longer callable, our D1 eyeball worker is throwing + // an error when using the method. + // TODO: Replace the body of this method and just throw an Error similar + // to what we return in our eyeball. const response = await this._wrappedFetch('http://d1/dump', { method: 'POST', headers: { @@ -540,13 +694,18 @@ class D1PreparedStatement { ); const info = firstIfArray( - await this.dbSession._sendOrThrow>( - '/query', - this.statement, - this.params, - 'ROWS_AND_COLUMNS', - span - ) + d1BindingJsrpc + ? await this.dbSession._queryOrThrow( + { queries: [{ sql: this.statement, params: this.params }] }, + span + ) + : await this.dbSession._sendOrThrow>( + '/query', + this.statement, + this.params, + 'ROWS_AND_COLUMNS', + span + ) ); span.setAttribute( @@ -555,7 +714,7 @@ class D1PreparedStatement { ); addD1MetaToSpan(span, info.meta); - const results = toArrayOfObjects(info).results; + const results = toArrayOfObjects>(info).results; const hasResults = results.length > 0; if (!hasResults) return null; @@ -574,6 +733,8 @@ class D1PreparedStatement { }); } + // TODO: Update this return type because we always includes a `results` array + // and people are relying on that. /* eslint-disable-next-line @typescript-eslint/no-unnecessary-type-parameters */ async run>(): Promise { return withSpan('d1_run', async (span) => { @@ -587,13 +748,18 @@ class D1PreparedStatement { ); const result = firstIfArray( - await this.dbSession._sendOrThrow( - '/execute', - this.statement, - this.params, - 'NONE', - span - ) + d1BindingJsrpc + ? await this.dbSession._queryOrThrow( + { queries: [{ sql: this.statement, params: this.params }] }, + span + ) + : await this.dbSession._sendOrThrow( + '/execute', + this.statement, + this.params, + 'NONE', + span + ) ); span.setAttribute( @@ -601,7 +767,7 @@ class D1PreparedStatement { this.dbSession.getBookmark() ?? undefined ); addD1MetaToSpan(span, result.meta); - return result; + return d1BindingJsrpc ? toArrayOfObjects(result) : result; }); } @@ -617,13 +783,18 @@ class D1PreparedStatement { ); const result = firstIfArray( - await this.dbSession._sendOrThrow( - '/query', - this.statement, - this.params, - 'ROWS_AND_COLUMNS', - span - ) + d1BindingJsrpc + ? await this.dbSession._queryOrThrow( + { queries: [{ sql: this.statement, params: this.params }] }, + span + ) + : await this.dbSession._sendOrThrow( + '/query', + this.statement, + this.params, + 'ROWS_AND_COLUMNS', + span + ) ); span.setAttribute( @@ -647,6 +818,26 @@ class D1PreparedStatement { this.dbSession.getBookmark() ?? undefined ); + if (d1BindingJsrpc) { + const s = firstIfArray( + await this.dbSession._queryOrThrow( + { queries: [{ sql: this.statement, params: this.params }] }, + span + ) + ); + + span.setAttribute( + 'cloudflare.d1.response.bookmark', + this.dbSession.getBookmark() ?? undefined + ); + addD1MetaToSpan(span, s.meta); + + return [ + ...(options?.columnNames ? [s.results.columns as T] : []), + ...(s.results.rows as T[]), + ]; + } + const s = firstIfArray( await this.dbSession._sendOrThrow>( '/query', @@ -690,6 +881,40 @@ class D1PreparedStatement { } } +/** + * Use this helper for every D1 `fetcher.query()` call instead of calling the + * fetcher directly, so shared query behavior has a single place to live. + */ +async function fetcherQuery( + fetcher: Fetcher, + request: QueryRequest +): Promise { + return await fetcher.query(request); +} + +// Adapt the lower-level query response into the existing rows-and-columns shape. +function mapQueryResult(queryResult: QuerySqlResult): D1RowsColumns { + const { data } = queryResult; + switch (data.kind) { + // Keep the switch so future result kinds have an obvious extension point. + // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition + case 'raw': + return { + success: true, + meta: queryResult.meta, + results: { + columns: data.columns, + rows: data.rows, + }, + }; + default: { + data.kind satisfies never; + const message = `Unsupported D1 query result kind: ${String(data.kind)}`; + throw new Error(`D1_ERROR: ${message}`); + } + } +} + function firstIfArray(results: T | T[]): T { return Array.isArray(results) ? (results.at(0) as T) : results; } diff --git a/src/workerd/io/compatibility-date.capnp b/src/workerd/io/compatibility-date.capnp index e88f443499c..5a680782009 100644 --- a/src/workerd/io/compatibility-date.capnp +++ b/src/workerd/io/compatibility-date.capnp @@ -1572,8 +1572,8 @@ struct CompatibilityFlags @0x8f8c1b68151b6cef { $compatDisableFlag("no_python_workers_20260610") $pythonSnapshotRelease $experimental; - # Enables Python Workers using Pyodide 314.0.0 (CPython 3.14.2, Emscripten 5.0.3). + # Enables Python Workers using Pyodide 314.0.0 (CPython 3.14.2, Emscripten 5.0.3). enableNodeJsInspectorLocalDev @180 :Bool $compatEnableFlag("enable_nodejs_inspector_local_dev") $experimental; @@ -1596,4 +1596,10 @@ struct CompatibilityFlags @0x8f8c1b68151b6cef { # This flag is not enabled by default in local dev on purpose, we want local dev to mimic # the production environment as much as possible, this flag is meant to be used by test # frameworks when running tests that require the inspector to be functional, and not by end users. + + d1BindingJsrpc @181 :Bool + $compatEnableFlag("d1_binding_jsrpc"); + # When enabled, D1 bindings use the internal JSRPC binding API for queries + # instead of issuing `fetch` calls to the D1 binding service. Without this + # flag, D1 bindings continue to use the `fetch` method of the Fetcher. } From bf60ab74036fcdce29886ec87c5d4882759b7296 Mon Sep 17 00:00:00 2001 From: Florent Collin Date: Wed, 1 Jul 2026 18:46:21 +0100 Subject: [PATCH 021/101] CFSQL-1705: D1 add JSRPC binding tests Add an explicit `d1_binding_jsrpc` D1 test config and teach the D1 mock to serve query() over DO RPC. --- src/cloudflare/internal/test/d1/BUILD.bazel | 17 +++++ .../test/d1/d1-api-instrumentation-test.js | 14 +++- .../test/d1/d1-api-test-jsrpc.wd-test | 53 ++++++++++++++ .../d1-api-test-with-sessions-jsrpc.wd-test | 49 +++++++++++++ .../test/d1/d1-api-test-with-sessions.js | 29 +++++--- .../internal/test/d1/d1-api-test.js | 34 +++++++++ src/cloudflare/internal/test/d1/d1-mock.js | 73 ++++++++++++++++++- 7 files changed, 254 insertions(+), 15 deletions(-) create mode 100644 src/cloudflare/internal/test/d1/d1-api-test-jsrpc.wd-test create mode 100644 src/cloudflare/internal/test/d1/d1-api-test-with-sessions-jsrpc.wd-test diff --git a/src/cloudflare/internal/test/d1/BUILD.bazel b/src/cloudflare/internal/test/d1/BUILD.bazel index 159d4e3b389..3f9564a2aba 100644 --- a/src/cloudflare/internal/test/d1/BUILD.bazel +++ b/src/cloudflare/internal/test/d1/BUILD.bazel @@ -11,6 +11,16 @@ wd_test( ) + ["//src/cloudflare/internal/test:instrumentation-test-helper.js"], ) +wd_test( + size = "large", + src = "d1-api-test-jsrpc.wd-test", + args = ["--experimental"], + data = glob( + ["*.js"], + exclude = ["d1-api-test-with-sessions.js"], + ) + ["//src/cloudflare/internal/test:instrumentation-test-helper.js"], +) + wd_test( size = "enormous", src = "d1-api-test-with-sessions.wd-test", @@ -18,6 +28,13 @@ wd_test( data = glob(["*.js"]), ) +wd_test( + size = "enormous", + src = "d1-api-test-with-sessions-jsrpc.wd-test", + args = ["--experimental"], + data = glob(["*.js"]), +) + py_wd_test( size = "large", src = "python-d1-api-test.wd-test", diff --git a/src/cloudflare/internal/test/d1/d1-api-instrumentation-test.js b/src/cloudflare/internal/test/d1/d1-api-instrumentation-test.js index 0184c2a3d99..1cda894f8c0 100644 --- a/src/cloudflare/internal/test/d1/d1-api-instrumentation-test.js +++ b/src/cloudflare/internal/test/d1/d1-api-instrumentation-test.js @@ -10,6 +10,7 @@ import { // Create module-level state using the helper const state = createInstrumentationState(); +const d1BindingJsrpc = !!Cloudflare.compatibilityFlags['d1_binding_jsrpc']; export default { tailStream: createTailStreamHandler(state), @@ -30,7 +31,7 @@ export const test = { }, }; -const expectedSpans = [ +const allExpectedSpans = [ // testExec: exec() happy path and error handling (regression test for #5218). { name: 'd1_exec', @@ -65,7 +66,9 @@ const expectedSpans = [ 'db.operation.name': 'exec', 'db.query.text': 'INVALID SQL', 'cloudflare.binding.type': 'D1', - 'error.type': 'Error in line 1', + 'error.type': d1BindingJsrpc + ? 'near "INVALID": syntax error at offset 0: SQLITE_ERROR' + : 'Error in line 1', closed: true, }, { @@ -1082,3 +1085,10 @@ const expectedSpans = [ closed: true, }, ]; + +const expectedSpans = d1BindingJsrpc + // The JSRPC binding path does not issue internal HTTP requests to the D1 + // service, so it should not emit the transport-level fetch spans that the + // legacy path produces. The public D1 operation spans are still asserted. + ? allExpectedSpans.filter((span) => span.name !== 'fetch') + : allExpectedSpans; diff --git a/src/cloudflare/internal/test/d1/d1-api-test-jsrpc.wd-test b/src/cloudflare/internal/test/d1/d1-api-test-jsrpc.wd-test new file mode 100644 index 00000000000..34971ee8fa4 --- /dev/null +++ b/src/cloudflare/internal/test/d1/d1-api-test-jsrpc.wd-test @@ -0,0 +1,53 @@ +using Workerd = import "/workerd/workerd.capnp"; + +const unitTests :Workerd.Config = ( + services = [ + ( name = "TEST_TMPDIR", disk = (writable = true) ), + ( name = "d1-api-test", + worker = ( + modules = [ + (name = "worker", esModule = embed "d1-api-test.js"), + (name = "d1-api-test-common", esModule = embed "d1-api-test-common.js"), + ], + compatibilityFlags = ["experimental", "nodejs_compat", "d1_binding_jsrpc"], + streamingTails = ["tail"], + bindings = [ + ( + name = "d1", + wrapped = ( + moduleName = "cloudflare-internal:d1-api", + innerBindings = [( + name = "fetcher", + service = "d1-mock" + )], + ) + ) + ], + ) + ), + ( name = "d1-mock", + worker = ( + compatibilityFlags = ["experimental", "nodejs_compat"], + modules = [ + (name = "worker", esModule = embed "d1-mock.js") + ], + durableObjectNamespaces = [ + (className = "D1MockDO", uniqueKey = "210bd0cbd803ef7883a1ee9d86cce06e"), + ], + durableObjectStorage = (localDisk = "TEST_TMPDIR"), + bindings = [ + (name = "db", durableObjectNamespace = "D1MockDO"), + ], + ) + ), + ( name = "tail", worker = .tailWorker, ), + ] +); + +const tailWorker :Workerd.Worker = ( + modules = [ + (name = "worker", esModule = embed "d1-api-instrumentation-test.js"), + (name = "instrumentation-test-helper", esModule = embed "../instrumentation-test-helper.js") + ], + compatibilityFlags = ["experimental", "nodejs_compat", "d1_binding_jsrpc"], +); diff --git a/src/cloudflare/internal/test/d1/d1-api-test-with-sessions-jsrpc.wd-test b/src/cloudflare/internal/test/d1/d1-api-test-with-sessions-jsrpc.wd-test new file mode 100644 index 00000000000..b40189b09ef --- /dev/null +++ b/src/cloudflare/internal/test/d1/d1-api-test-with-sessions-jsrpc.wd-test @@ -0,0 +1,49 @@ +using Workerd = import "/workerd/workerd.capnp"; + +const unitTests :Workerd.Config = ( + services = [ + ( name = "TEST_TMPDIR", disk = (writable = true) ), + ( name = "d1-api-test-with-sessions", + worker = ( + modules = [ + (name = "worker", esModule = embed "d1-api-test-with-sessions.js"), + (name = "d1-api-test-common", esModule = embed "d1-api-test-common.js"), + ], + + compatibilityFlags = ["experimental", "nodejs_compat", "d1_binding_jsrpc"], + + bindings = [ + ( + name = "d1", + wrapped = ( + moduleName = "cloudflare-internal:d1-api", + innerBindings = [( + name = "fetcher", + service = "d1-mock" + )], + ), + ), + ( + name = "d1MockFetcher", + service = "d1-mock" + ), + ], + ) + ), + ( name = "d1-mock", + worker = ( + compatibilityFlags = ["experimental", "nodejs_compat"], + modules = [ + (name = "worker", esModule = embed "d1-mock.js") + ], + durableObjectNamespaces = [ + (className = "D1MockDO", uniqueKey = "210bd0cbd803ef7883a1ee9d86cce06e"), + ], + durableObjectStorage = (localDisk = "TEST_TMPDIR"), + bindings = [ + (name = "db", durableObjectNamespace = "D1MockDO"), + ], + ) + ) + ] +); diff --git a/src/cloudflare/internal/test/d1/d1-api-test-with-sessions.js b/src/cloudflare/internal/test/d1/d1-api-test-with-sessions.js index b4609b938e5..7923a9ac4f4 100644 --- a/src/cloudflare/internal/test/d1/d1-api-test-with-sessions.js +++ b/src/cloudflare/internal/test/d1/d1-api-test-with-sessions.js @@ -90,7 +90,6 @@ async function testD1ApiWithSessionsTokensHandling(DB, envD1MockFetcher) { tokens.every((t) => t === null), true ); - // Make sure we also received nothing from the eyeball worker since we never sent any commit tokens. assert.deepEqual( await getCommitTokensReturnedFromEyeball(envD1MockFetcher), [] @@ -118,7 +117,7 @@ export const test_d1_api_withsessions_old_token_skipped = test( ); async function testD1ApiWithSessionsOldTokensSkipped(DB, envD1MockFetcher) { - const runTest = async (session) => { + const runTest = async (session, firstTokenFromBinding) => { await resetCommitTokens(envD1MockFetcher); await session.prepare(`SELECT * FROM sqlite_master;`).all(); await session.prepare(`SELECT * FROM sqlite_master;`).all(); @@ -137,8 +136,16 @@ async function testD1ApiWithSessionsOldTokensSkipped(DB, envD1MockFetcher) { await getCommitTokensSentFromBinding(envD1MockFetcher); const tokensFromEyeball = await getCommitTokensReturnedFromEyeball(envD1MockFetcher); + + // The default DB has no session, so it never sends or receives bookmarks. + if (firstTokenFromBinding === null) { + assert.deepEqual(tokensFromBinding, [null, null, null, null, null]); + assert.deepEqual(tokensFromEyeball, []); + return { ok: true }; + } + const expectedTokensFromBinding = [ - 'first-unconstrained', + firstTokenFromBinding, tokensFromEyeball[0], tokensFromEyeball[1], // We skip the commit token "------", since the previously received one was more recent. @@ -151,16 +158,20 @@ async function testD1ApiWithSessionsOldTokensSkipped(DB, envD1MockFetcher) { return { ok: true }; }; - itShould('default DB', () => runTest(DB), { ok: true }); - itShould('withSession()', () => runTest(DB.withSession()), { ok: true }); - itShould( + await itShould('default DB', () => runTest(DB, null), { ok: true }); + await itShould( + 'withSession()', + () => runTest(DB.withSession(), 'first-unconstrained'), + { ok: true } + ); + await itShould( 'withSession(first-unconstrained)', - () => runTest(DB.withSession('first-unconstrained')), + () => runTest(DB.withSession('first-unconstrained'), 'first-unconstrained'), { ok: true } ); - itShould( + await itShould( 'withSession(first-primary)', - () => runTest(DB.withSession('first-primary')), + () => runTest(DB.withSession('first-primary'), 'first-primary'), { ok: true } ); } diff --git a/src/cloudflare/internal/test/d1/d1-api-test.js b/src/cloudflare/internal/test/d1/d1-api-test.js index b721f6db97f..c71650e6b31 100644 --- a/src/cloudflare/internal/test/d1/d1-api-test.js +++ b/src/cloudflare/internal/test/d1/d1-api-test.js @@ -2,6 +2,8 @@ // Licensed under the Apache 2.0 license found in the LICENSE file or at: // https://opensource.org/licenses/Apache-2.0 +import * as assert from 'node:assert'; + import { testD1ApiQueriesHappyPath, testD1Exec } from './d1-api-test-common'; export const testWithoutSessions = { @@ -15,3 +17,35 @@ export const testExec = { await testD1Exec(env.d1); }, }; + +export const testDirectQuery = { + async test(_ctr, env) { + if (!Cloudflare.compatibilityFlags['d1_binding_jsrpc']) { + return; + } + + const response = await env.d1.query({ + queries: [{ sql: 'select 42 as answer' }], + bookmark: 'first-unconstrained', + }); + + assert.deepEqual(response, { + success: true, + results: { + bookmark: response.results.bookmark, + queryResults: [ + { + meta: response.results.queryResults[0].meta, + data: { + kind: 'raw', + columns: ['answer'], + rows: [[42]], + }, + }, + ], + }, + }); + assert.equal(typeof response.results.bookmark, 'string'); + assert.equal(typeof response.results.queryResults[0].meta.duration, 'number'); + }, +}; diff --git a/src/cloudflare/internal/test/d1/d1-mock.js b/src/cloudflare/internal/test/d1/d1-mock.js index e4057e56ae1..a97c3146515 100644 --- a/src/cloudflare/internal/test/d1/d1-mock.js +++ b/src/cloudflare/internal/test/d1/d1-mock.js @@ -2,8 +2,11 @@ // Licensed under the Apache 2.0 license found in the LICENSE file or at: // https://opensource.org/licenses/Apache-2.0 -export class D1MockDO { +import { DurableObject } from 'cloudflare:workers'; + +export class D1MockDO extends DurableObject { constructor(state, env) { + super(state, env); this.state = state; this.sql = this.state.storage.sql; } @@ -26,7 +29,7 @@ export class D1MockDO { return this.runQuery(query, resultsFormat); } catch (e) { // Reproduce the production behavior by catching any error and returning a V4Failure - return { success: false, error: String(e.message) }; + return { success: false, error: String(e instanceof Error ? e.message : e) }; } }; return Response.json( @@ -39,6 +42,35 @@ export class D1MockDO { } } + async query(params) { + const results = params.queries.map((query) => { + try { + return this.runQuery(query, 'ROWS_AND_COLUMNS'); + } catch (e) { + // Reproduce the production behavior by catching any error and returning a V4Failure + return { success: false, error: String(e instanceof Error ? e.message : e) }; + } + }); + const failure = results.find((result) => !result.success); + if (failure) { + return { success: false, error: new Error(failure.error) }; + } + + return { + success: true, + results: { + queryResults: results.map((result) => ({ + meta: result.meta, + data: { + kind: 'raw', + columns: result.results.columns, + rows: result.results.rows, + }, + })), + }, + }; + } + runQuery(query, resultsFormat) { const { sql, params = [] } = query; @@ -89,7 +121,6 @@ export class D1MockDO { results, meta: { duration: Math.random() * 0.01, - served_by: 'd1-mock', served_by_colo: 'DFW', changes: num_changes, last_row_id: last_row_id_after, @@ -108,6 +139,35 @@ export default { commitTokensReturned: [], nextTokenExpected: null, + async query(params, env) { + this.commitTokensReceived.push(params.bookmark ?? null); + + try { + const stub = env.db.get(env.db.idFromName('test')); + const queryResult = await stub.query(params); + if (!queryResult.success) { + return queryResult; + } + + const results = { + queryResults: queryResult.results.queryResults, + }; + if (params.bookmark) { + results.bookmark = this.nextCommitToken(); + } + + return { + success: true, + results, + }; + } catch (err) { + return { + success: false, + error: err instanceof Error ? err : new Error(String(err)), + }; + } + }, + async fetch(request, env, ctx) { if (request.url.startsWith('http://d1-api-test/commitTokens')) { return this.handleD1ApiTestRoutes(request); @@ -133,7 +193,7 @@ export default { } }, - async buildResponseWithCommitToken(resp) { + nextCommitToken() { let newToken = `token-${(++this.commitTokenNum).toLocaleString('en-US', { minimumIntegerDigits: 4, // no commas @@ -144,6 +204,11 @@ export default { this.nextTokenExpected = null; } this.commitTokensReturned.push(newToken); + return newToken; + }, + + async buildResponseWithCommitToken(resp) { + const newToken = this.nextCommitToken(); // Append an ever increasing commit token to the response. // Simulating the D1 eyeball worker. const newHeaders = new Headers(resp.headers); From 8ab143b2e82ab7b6a16b8c88b9dd49c8e21cfb5f Mon Sep 17 00:00:00 2001 From: Erik Corry Date: Fri, 3 Jul 2026 12:57:50 +0000 Subject: [PATCH 022/101] Use non-sandbox buffer to read in internal.c++ * Use non-sandbox buffer to read in internal.c++ In case the sandbox array buffers are under MPK protection we provide a temporary buffer for the kj sink to write into. See merge request cloudflare/ew/workerd!422 --- src/workerd/api/streams/internal.c++ | 61 ++++++++++------------------ 1 file changed, 21 insertions(+), 40 deletions(-) diff --git a/src/workerd/api/streams/internal.c++ b/src/workerd/api/streams/internal.c++ index 7d50a8ab55a..a8c479263fa 100644 --- a/src/workerd/api/streams/internal.c++ +++ b/src/workerd/api/streams/internal.c++ @@ -528,27 +528,14 @@ kj::Maybe> ReadableStreamInternalController::read( js.typeError("Unable to allocate memory for read"_kj)); } - // In the case the ArrayBuffer is detached/transfered while the read is pending, we - // need to make sure that the ptr remains stable, so we grab a shared ptr to the - // backing store and use that to get the pointer to the data. If the buffer is detached - // while the read is pending, this does mean that the read data will end up being lost, - // but there's not really a better option. The best we can do here is warn the user - // that this is happening so they can avoid doing it in the future. - // Also, the user really shouldn't do this because the read will end up completing into - // the detached backing store still which could cause issues with whatever code now actually - // owns the transfered buffer. Below we'll warn the user about this if it happens so they - // can avoid doing it in the future. - auto backing = theStore->GetBackingStore(); - - // For resizable ArrayBuffers, the buffer may be resized while the read is - // pending, decommitting memory pages and making the pointer invalid (SIGSEGV). - // We read into a temporary buffer and copy the data back in the .then() - // callback, where we can validate the buffer is still large enough. - bool isResizable = theStore->IsResizableByUserJavaScript(); - - kj::Array tempBuffer; - kj::byte* readPtr; - if (isResizable) { + // All reads go through a temporary kj-heap buffer outside the V8 sandbox + // in case the sandbox is MPK protected with a key not held by the kj + // sink called from the event loop. + // + // We will memcpy back into the user's BackingStore in the .then() + // continuation, after re-validating the BackingStore is still attached + // and large enough. + if (theStore->IsResizableByUserJavaScript()) { auto currentByteLength = theStore->ByteLength(); if (byteOffset >= currentByteLength) { readPending = false; @@ -564,19 +551,15 @@ kj::Maybe> ReadableStreamInternalController::read( atLeast = byteLength > 0 ? byteLength : 1; } } - tempBuffer = kj::heapArray(byteLength); - readPtr = tempBuffer.begin(); - } else { - auto ptr = static_cast(backing->Data()); - readPtr = ptr + byteOffset; } - auto bytes = kj::arrayPtr(readPtr, byteLength); + + auto tempBuffer = kj::heapArray(byteLength); + auto bytes = tempBuffer.asPtr(); KJ_ASSERT(atLeast <= bytes.size(), "minBytes must not exceed maxBytes in tryRead"); - auto promise = kj::evalNow([&] { - return readable->tryRead(bytes.begin(), atLeast, bytes.size()).attach(kj::mv(backing)); - }); + auto promise = + kj::evalNow([&] { return readable->tryRead(bytes.begin(), atLeast, bytes.size()); }); KJ_IF_SOME(readerLock, readState.tryGetUnsafe()) { promise = KJ_ASSERT_NONNULL(readerLock.getCanceler())->wrap(kj::mv(promise)); } @@ -592,10 +575,10 @@ kj::Maybe> ReadableStreamInternalController::read( auto& ioContext = IoContext::current(); return ioContext.awaitIoLegacy(js, kj::mv(promise)) .then(js, - ioContext.addFunctor([ref = addRef(), store = js.v8Ref(store), byteOffset, byteLength, - isByob = maybeByobOptions != kj::none, isResizable, readPtr, - tempBuffer = kj::mv(tempBuffer)](jsg::Lock& js, - size_t amount) mutable -> jsg::Promise { + ioContext.addFunctor( + [ref = addRef(), store = js.v8Ref(store), byteOffset, byteLength, + isByob = maybeByobOptions != kj::none, tempBuffer = kj::mv(tempBuffer)]( + jsg::Lock& js, size_t amount) mutable -> jsg::Promise { auto& controller = static_cast(ref->getController()); controller.readPending = false; KJ_ASSERT(amount <= byteLength); @@ -663,12 +646,10 @@ kj::Maybe> ReadableStreamInternalController::read( amount = handle->ByteLength() - byteOffset; } - if (isResizable && byteOffset + amount <= handle->ByteLength()) { - // For resizable buffers, the data was read into a temporary buffer. - // Copy it back into the user's (still valid) buffer region. - auto destPtr = static_cast(handle->GetBackingStore()->Data()); - memcpy(destPtr + byteOffset, readPtr, amount); - } + // Copy the data from the kj-heap temporary buffer into the user's + // BackingStore. + auto destPtr = static_cast(handle->GetBackingStore()->Data()); + memcpy(destPtr + byteOffset, tempBuffer.begin(), amount); auto u8 = v8::Uint8Array::New(store.getHandle(js), byteOffset, amount); return js.resolvedPromise(ReadResult{ From 6102c8546b8568fdab9824f040417f4a685d333a Mon Sep 17 00:00:00 2001 From: Dominik Picheta Date: Wed, 1 Jul 2026 16:05:36 +0000 Subject: [PATCH 023/101] Implements Worker::Isolate::runInLockScope. --- src/workerd/io/worker.c++ | 8 ++++++++ src/workerd/io/worker.h | 6 ++++++ 2 files changed, 14 insertions(+) diff --git a/src/workerd/io/worker.c++ b/src/workerd/io/worker.c++ index f2729d34c12..4d5d0d2d60b 100644 --- a/src/workerd/io/worker.c++ +++ b/src/workerd/io/worker.c++ @@ -2736,6 +2736,14 @@ Worker::Isolate::AsyncWaiterList::~AsyncWaiterList() noexcept { KJ_ASSERT(tail == &head, "tail pointer corrupted?"); } +void Worker::Isolate::runInLockScope( + LockType lockType, kj::FunctionParam callback) const { + jsg::runInV8Stack([&](jsg::V8StackScope& stackScope) { + Isolate::Impl::Lock recordedLock(*this, lockType, stackScope); + callback(*recordedLock.lock); + }); +} + kj::Promise Worker::Isolate::takeAsyncLockWithoutRequest( SpanParent parentSpan) const { auto lockTiming = getMetrics().tryCreateLockTiming(kj::mv(parentSpan)); diff --git a/src/workerd/io/worker.h b/src/workerd/io/worker.h index 3a4b12c806f..2b995ccbb96 100644 --- a/src/workerd/io/worker.h +++ b/src/workerd/io/worker.h @@ -499,6 +499,12 @@ class Worker::Isolate: public kj::AtomicRefcounted { // See Worker::takeAsyncLock(). kj::Promise takeAsyncLock(RequestObserver&) const; + // Enters the isolate under `lockType` and runs `callback` with the resulting `jsg::Lock`. Unlike + // `Worker::runInLockScope`, this does not require a `Worker` -- it locks the isolate directly, + // which is useful for whole-isolate operations such as CPU profiling that are not tied to any + // particular Worker instance. + void runInLockScope(LockType lockType, kj::FunctionParam callback) const; + bool isInspectorEnabled() const; // Returns the isolate's V8 inspector, if one exists. An inspector is created either because a From b2a4fadadf7cf12077bcdcab825ab4402425a5cb Mon Sep 17 00:00:00 2001 From: Nicholas Paun Date: Fri, 3 Jul 2026 03:16:18 +0000 Subject: [PATCH 024/101] Respect no_build for rust test compilation --- build/wd_rust_crate.bzl | 4 ++++ build/wd_rust_proc_macro.bzl | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/build/wd_rust_crate.bzl b/build/wd_rust_crate.bzl index de14bbd4fbc..a4c53eaa221 100644 --- a/build/wd_rust_crate.bzl +++ b/build/wd_rust_crate.bzl @@ -172,6 +172,10 @@ def wd_rust_crate( "@//build/config:rust_cc_common_link": 1, "//conditions:default": -1, }), + target_compatible_with = select({ + "@//build/config:no_build": ["@platforms//:incompatible"], + "//conditions:default": [], + }), ) if len(proc_macro_deps) + len(cxx_bridge_srcs) > 0: diff --git a/build/wd_rust_proc_macro.bzl b/build/wd_rust_proc_macro.bzl index ed909ceac29..1b5f2c80e52 100644 --- a/build/wd_rust_proc_macro.bzl +++ b/build/wd_rust_proc_macro.bzl @@ -47,4 +47,8 @@ def wd_rust_proc_macro( } | test_env, tags = test_tags + ["no-coverage"], deps = test_deps, + target_compatible_with = select({ + "@//build/config:no_build": ["@platforms//:incompatible"], + "//conditions:default": [], + }), ) From 570a6e2ab6822724c91ea805f4308e128a14aa33 Mon Sep 17 00:00:00 2001 From: Ketan Gupta Date: Mon, 6 Jul 2026 10:01:36 +0000 Subject: [PATCH 025/101] [ci] Stream internal build logs --- ci/build.yml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/ci/build.yml b/ci/build.yml index 937cae857fb..2bbc41720d1 100644 --- a/ci/build.yml +++ b/ci/build.yml @@ -66,7 +66,6 @@ include: runOnBranches: "^$" SCRIPT: | import os - import runpy import subprocess import sys @@ -77,7 +76,9 @@ include: "requests", ]) - sys.argv = [ + subprocess.check_call([ + sys.executable, + "-u", "./tools/cross/internal_build.py", os.environ["CI_MERGE_REQUEST_IID"], os.environ["CI_COMMIT_SHA"], @@ -87,9 +88,7 @@ include: os.environ["CI_URL"], os.environ["CI_CLIENT_ID"], os.environ["CI_CLIENT_SECRET"], - ] - - runpy.run_path("./tools/cross/internal_build.py", run_name="__main__") + ]) linux-x64-build: <<: *job-template From e54a45305f3176033063e43018dbcaf26a182bdc Mon Sep 17 00:00:00 2001 From: Caio Nogueira Date: Mon, 15 Jun 2026 20:39:04 +0100 Subject: [PATCH 026/101] WOR-1263: migrate workflow wrapped binding into jsrpc Gates the JSRPC transport for the env.WORKFLOW binding while the RPC callee is rolled out. Experimental opt-in only (no enable date); the legacy HTTP transport remains the default. feat(workflows): route binding through JSRPC behind compat flag When workflows_bindings_rpc is enabled, WorkflowImpl and InstanceImpl dispatch their methods as JSRPC calls directly on the inner fetcher instead of issuing HTTP requests. The wrapper classes are preserved on both transports (results are always wrapped in InstanceImpl), so prototypes and instanceof stability are unchanged. The Fetcher interface now declares the RPC method surface the callee must implement. The legacy callFetcher HTTP path is retained for un-flagged workers. test(workflows): cover both binding transports end-to-end Upgrade the mock to a dual-transport WorkerEntrypoint (RPC methods + fetch handler) sharing one set of business-logic helpers so both transports behave identically, including an error-trigger id. The wd-test now runs the same suite against a flag-off (HTTP) service and a flag-on (RPC) service pointed at the WorkflowsMock entrypoint. Tests add instance-method, prototype-preservation, and error-propagation coverage. --- .../test/workflows/workflows-api-rpc-test.js | 118 +++++++++++++ .../test/workflows/workflows-api-test.js | 32 ++++ .../test/workflows/workflows-api-test.wd-test | 28 ++- .../internal/test/workflows/workflows-mock.js | 160 +++++++++++------- src/cloudflare/internal/workflows-api.ts | 70 ++++++-- src/workerd/io/compatibility-date.capnp | 8 + 6 files changed, 344 insertions(+), 72 deletions(-) create mode 100644 src/cloudflare/internal/test/workflows/workflows-api-rpc-test.js diff --git a/src/cloudflare/internal/test/workflows/workflows-api-rpc-test.js b/src/cloudflare/internal/test/workflows/workflows-api-rpc-test.js new file mode 100644 index 00000000000..215d206afb8 --- /dev/null +++ b/src/cloudflare/internal/test/workflows/workflows-api-rpc-test.js @@ -0,0 +1,118 @@ +// Copyright (c) 2024 Cloudflare, Inc. +// Licensed under the Apache 2.0 license found in the LICENSE file or at: +// https://opensource.org/licenses/Apache-2.0 + +import * as assert from 'node:assert'; + +async function lastRestartBody(env, id) { + return env.mock.lastRestart(id); +} + +export const tests = { + async test(_, env) { + { + const instance = await env.workflow.create({ + id: 'foo', + payload: { bar: 'baz' }, + }); + assert.deepStrictEqual(instance.id, 'foo'); + } + + { + const instance = await env.workflow.get('bar'); + assert.deepStrictEqual(instance.id, 'bar'); + } + + { + const instances = await env.workflow.createBatch([ + { id: 'foo', payload: { bar: 'baz' } }, + { id: 'bar', payload: { bar: 'baz' } }, + ]); + assert.deepStrictEqual(instances[0].id, 'foo'); + assert.deepStrictEqual(instances[1].id, 'bar'); + } + + { + const instance = await env.workflow.get('inst'); + await instance.pause(); + await instance.resume(); + await instance.terminate(); + await instance.sendEvent({ + type: 'my-event', + payload: { hello: 'world' }, + }); + } + + { + const instance = await env.workflow.get('status-rpc'); + const status = await instance.status(); + assert.deepStrictEqual(status.status, 'running'); + assert.strictEqual(status.transport, 'rpc'); + assert.strictEqual(status.output, 'status-rpc'); + } + + { + for (const method of ['get', 'create', 'createBatch']) { + assert.strictEqual(typeof env.workflow[method], 'function'); + } + + const fromGet = await env.workflow.get('a'); + const fromCreate = await env.workflow.create({ id: 'b' }); + const [fromBatch] = await env.workflow.createBatch([{ id: 'c' }]); + + const proto = Object.getPrototypeOf(fromGet); + assert.strictEqual(Object.getPrototypeOf(fromCreate), proto); + assert.strictEqual(Object.getPrototypeOf(fromBatch), proto); + + for (const method of [ + 'pause', + 'resume', + 'terminate', + 'restart', + 'status', + 'sendEvent', + ]) { + assert.strictEqual(typeof fromGet[method], 'function'); + } + } + + { + await assert.rejects(env.workflow.get('throw'), { + message: 'workflow instance not found', + }); + } + }, + + async testRestartNoOptions(_, env) { + const instance = await env.workflow.get('rpc-restart-basic'); + await instance.restart(); + + const body = await lastRestartBody(env, 'rpc-restart-basic'); + assert.deepStrictEqual(body.id, 'rpc-restart-basic'); + assert.strictEqual(body.from, undefined); + }, + + async testRestartFromStepNameOnly(_, env) { + const instance = await env.workflow.get('rpc-restart-step'); + await instance.restart({ from: { name: 'fetch data' } }); + + const body = await lastRestartBody(env, 'rpc-restart-step'); + assert.deepStrictEqual(body.id, 'rpc-restart-step'); + assert.deepStrictEqual(body.from, { name: 'fetch data' }); + }, + + async testRestartFromStepAllOptions(_, env) { + const instance = await env.workflow.get('rpc-restart-full'); + await instance.restart({ + from: { name: 'process item', count: 3, type: 'do' }, + }); + + const body = await lastRestartBody(env, 'rpc-restart-full'); + assert.deepStrictEqual(body.id, 'rpc-restart-full'); + assert.deepStrictEqual(body.from, { + name: 'process item', + count: 3, + type: 'do', + }); + }, +}; diff --git a/src/cloudflare/internal/test/workflows/workflows-api-test.js b/src/cloudflare/internal/test/workflows/workflows-api-test.js index 662acf76b01..6cd31067478 100644 --- a/src/cloudflare/internal/test/workflows/workflows-api-test.js +++ b/src/cloudflare/internal/test/workflows/workflows-api-test.js @@ -44,6 +44,38 @@ export const tests = { assert.deepStrictEqual(instances[0].id, 'foo'); assert.deepStrictEqual(instances[1].id, 'bar'); } + + { + const instance = await env.workflow.get('status-http'); + const status = await instance.status(); + assert.deepStrictEqual(status.status, 'running'); + assert.strictEqual(status.transport, 'http'); + } + + { + for (const method of ['get', 'create', 'createBatch']) { + assert.strictEqual(typeof env.workflow[method], 'function'); + } + + const fromGet = await env.workflow.get('a'); + const fromCreate = await env.workflow.create({ id: 'b' }); + const [fromBatch] = await env.workflow.createBatch([{ id: 'c' }]); + + const proto = Object.getPrototypeOf(fromGet); + assert.strictEqual(Object.getPrototypeOf(fromCreate), proto); + assert.strictEqual(Object.getPrototypeOf(fromBatch), proto); + + for (const method of [ + 'pause', + 'resume', + 'terminate', + 'restart', + 'status', + 'sendEvent', + ]) { + assert.strictEqual(typeof fromGet[method], 'function'); + } + } }, async testRestartNoOptions(_, env) { diff --git a/src/cloudflare/internal/test/workflows/workflows-api-test.wd-test b/src/cloudflare/internal/test/workflows/workflows-api-test.wd-test index cf29c59e4d4..4b6b03854d3 100644 --- a/src/cloudflare/internal/test/workflows/workflows-api-test.wd-test +++ b/src/cloudflare/internal/test/workflows/workflows-api-test.wd-test @@ -26,9 +26,35 @@ const unitTests :Workerd.Config = ( ], ) ), + ( name = "workflows-api-rpc-test", + worker = ( + modules = [ + (name = "worker", esModule = embed "workflows-api-rpc-test.js") + ], + compatibilityFlags = ["nodejs_compat", "workflows_bindings_rpc", + "experimental", "service_binding_extra_handlers", + "rpc"], + bindings = [ + ( + name = "workflow", + wrapped = ( + moduleName = "cloudflare-internal:workflows-api", + innerBindings = [( + name = "fetcher", + service = "workflows-mock" + )], + ) + ), + ( + name = "mock", + service = "workflows-mock" + ) + ], + ) + ), ( name = "workflows-mock", worker = ( - compatibilityFlags = ["nodejs_compat"], + compatibilityFlags = ["experimental", "nodejs_compat"], modules = [ (name = "worker", esModule = embed "workflows-mock.js") ], diff --git a/src/cloudflare/internal/test/workflows/workflows-mock.js b/src/cloudflare/internal/test/workflows/workflows-mock.js index 055297c861d..7cd53df5514 100644 --- a/src/cloudflare/internal/test/workflows/workflows-mock.js +++ b/src/cloudflare/internal/test/workflows/workflows-mock.js @@ -2,70 +2,110 @@ // Licensed under the Apache 2.0 license found in the LICENSE file or at: // https://opensource.org/licenses/Apache-2.0 +import { WorkerEntrypoint } from 'cloudflare:workers'; + const restartBodies = new Map(); -export default { - async fetch(request, env, ctx) { - const data = await request.json(); - const reqUrl = new URL(request.url); - - if (reqUrl.pathname === '/get' && request.method === 'POST') { - return Response.json( - { - result: { - id: data.id, - }, - }, - { - status: 200, - headers: { - 'content-type': 'application/json', - }, - } - ); - } +const THROW_ID = 'throw'; - if (reqUrl.pathname === '/create' && request.method === 'POST') { - return Response.json( - { - result: { - id: data.id, - }, - }, - { - status: 201, - headers: { - 'content-type': 'application/json', - }, - } - ); - } +function getInstance(id) { + if (id === THROW_ID) { + throw new Error('workflow instance not found'); + } + return { id }; +} - if (reqUrl.pathname === '/createBatch' && request.method === 'POST') { - return Response.json( - { - result: data.map((val) => ({ id: val.id })), - }, - { - status: 201, - headers: { - 'content-type': 'application/json', - }, - } - ); - } +function createInstance(options) { + return { id: options?.id }; +} - if (reqUrl.pathname === '/restart' && request.method === 'POST') { - restartBodies.set(data.id, data); - return Response.json({}, { status: 200 }); - } +function createBatchInstances(options) { + return options.map((val) => ({ id: val.id })); +} - // Test-only: returns the body from the last /restart call for a given id - if (reqUrl.pathname === '/last-restart' && request.method === 'POST') { - return Response.json( - { result: restartBodies.get(data.id) ?? null }, - { status: 200 } - ); +function instanceStatus(id, transport) { + return { status: 'running', output: id, transport }; +} + +async function handleHttp(request) { + const data = await request.json(); + const reqUrl = new URL(request.url); + + if (request.method !== 'POST') { + return Response.json({ success: false }, { status: 500 }); + } + + try { + switch (reqUrl.pathname) { + case '/get': + return Response.json({ result: getInstance(data.id) }, { status: 200 }); + case '/create': + return Response.json({ result: createInstance(data) }, { status: 201 }); + case '/createBatch': + return Response.json( + { result: createBatchInstances(data) }, + { status: 201 } + ); + case '/pause': + case '/resume': + case '/terminate': + case '/send-event': + return Response.json({ result: null }, { status: 200 }); + case '/restart': + restartBodies.set(data.id, data); + return Response.json({ result: null }, { status: 200 }); + case '/status': + return Response.json( + { result: instanceStatus(data.id, 'http') }, + { status: 200 } + ); + case '/last-restart': + return Response.json( + { result: restartBodies.get(data.id) ?? null }, + { status: 200 } + ); + default: + return Response.json({ success: false }, { status: 404 }); } - }, -}; + } catch (err) { + return Response.json({ error: { message: err.message } }, { status: 500 }); + } +} + +export default class WorkflowsMock extends WorkerEntrypoint { + async getInstance(id) { + return getInstance(id); + } + + async create(options) { + return createInstance(options); + } + + async createBatch(options) { + return createBatchInstances(options); + } + + async pause(_id) {} + + async resume(_id) {} + + async terminate(_id) {} + + async restart(id, options) { + restartBodies.set(id, { ...options, id }); + } + + async status(id) { + return instanceStatus(id, 'rpc'); + } + + async sendEvent(_id, _event) {} + + async lastRestart(id) { + return restartBodies.get(id) ?? null; + } + + async fetch(request) { + return handleHttp(request); + } +} diff --git a/src/cloudflare/internal/workflows-api.ts b/src/cloudflare/internal/workflows-api.ts index 5eea22eef05..2c3109885f1 100644 --- a/src/cloudflare/internal/workflows-api.ts +++ b/src/cloudflare/internal/workflows-api.ts @@ -9,8 +9,26 @@ export class NonRetryableError extends Error { } } +const workflowsBindingsRpc = + !!Cloudflare.compatibilityFlags['workflows_bindings_rpc']; + interface Fetcher { fetch: typeof fetch; + getInstance(id: string): Promise<{ id: string }>; + create(options?: WorkflowInstanceCreateOptions): Promise<{ id: string }>; + createBatch( + options: WorkflowInstanceCreateOptions[] + ): Promise<{ id: string }[]>; + + pause(id: string): Promise; + resume(id: string): Promise; + terminate(id: string, options?: WorkflowInstanceTerminateOptions): Promise; + restart(id: string, options?: WorkflowInstanceRestartOptions): Promise; + status(id: string): Promise; + sendEvent( + id: string, + event: { type: string; payload: unknown } + ): Promise; } async function callFetcher( @@ -51,17 +69,29 @@ class InstanceImpl implements WorkflowInstance { } async pause(): Promise { + if (workflowsBindingsRpc) { + await this.fetcher.pause(this.id); + return; + } await callFetcher(this.fetcher, '/pause', { id: this.id, }); } async resume(): Promise { + if (workflowsBindingsRpc) { + await this.fetcher.resume(this.id); + return; + } await callFetcher(this.fetcher, '/resume', { id: this.id, }); } async terminate(options?: WorkflowInstanceTerminateOptions): Promise { + if (workflowsBindingsRpc) { + await this.fetcher.terminate(this.id, options); + return; + } await callFetcher(this.fetcher, '/terminate', { id: this.id, ...(options?.rollback === true ? { rollback: true } : {}), @@ -69,6 +99,10 @@ class InstanceImpl implements WorkflowInstance { } async restart(options?: WorkflowInstanceRestartOptions): Promise { + if (workflowsBindingsRpc) { + await this.fetcher.restart(this.id, options); + return; + } await callFetcher(this.fetcher, '/restart', { ...options, id: this.id, @@ -76,6 +110,9 @@ class InstanceImpl implements WorkflowInstance { } async status(): Promise { + if (workflowsBindingsRpc) { + return await this.fetcher.status(this.id); + } const result = await callFetcher(this.fetcher, '/status', { id: this.id, }); @@ -89,6 +126,10 @@ class InstanceImpl implements WorkflowInstance { type: string; payload: unknown; }): Promise { + if (workflowsBindingsRpc) { + await this.fetcher.sendEvent(this.id, { type, payload }); + return; + } await callFetcher(this.fetcher, '/send-event', { type, payload, @@ -107,9 +148,12 @@ class WorkflowImpl { } async get(id: string): Promise { - const result = await callFetcher<{ - id: string; - }>(this.fetcher, '/get', { id }); + const result = workflowsBindingsRpc + ? // getInstance, not get: avoids colliding with the built-in Fetcher.get(url). + await this.fetcher.getInstance(id) + : await callFetcher<{ + id: string; + }>(this.fetcher, '/get', { id }); return new InstanceImpl(result.id, this.fetcher); } @@ -117,9 +161,11 @@ class WorkflowImpl { async create( options?: WorkflowInstanceCreateOptions ): Promise { - const result = await callFetcher<{ - id: string; - }>(this.fetcher, '/create', options ?? {}); + const result = workflowsBindingsRpc + ? await this.fetcher.create(options) + : await callFetcher<{ + id: string; + }>(this.fetcher, '/create', options ?? {}); return new InstanceImpl(result.id, this.fetcher); } @@ -127,11 +173,13 @@ class WorkflowImpl { async createBatch( options: WorkflowInstanceCreateOptions[] ): Promise { - const results = await callFetcher< - { - id: string; - }[] - >(this.fetcher, '/createBatch', options); + const results = workflowsBindingsRpc + ? await this.fetcher.createBatch(options) + : await callFetcher< + { + id: string; + }[] + >(this.fetcher, '/createBatch', options); return results.map((result) => new InstanceImpl(result.id, this.fetcher)); } diff --git a/src/workerd/io/compatibility-date.capnp b/src/workerd/io/compatibility-date.capnp index 5a680782009..3bd565deee3 100644 --- a/src/workerd/io/compatibility-date.capnp +++ b/src/workerd/io/compatibility-date.capnp @@ -1602,4 +1602,12 @@ struct CompatibilityFlags @0x8f8c1b68151b6cef { # When enabled, D1 bindings use the internal JSRPC binding API for queries # instead of issuing `fetch` calls to the D1 binding service. Without this # flag, D1 bindings continue to use the `fetch` method of the Fetcher. + + workflowsBindingsRpc @182 :Bool + $compatEnableFlag("workflows_bindings_rpc") + $experimental; + # When enabled, the `env.WORKFLOW` binding (cloudflare-internal:workflows-api) + # dispatches its methods as JSRPC calls on the inner fetcher instead of HTTP + # requests against the binding-shim worker. Without the flag the legacy HTTP + # transport is used. } From 92361ffcbf6666d51d43a31b2be87c20a3d282ce Mon Sep 17 00:00:00 2001 From: LongYinan Date: Mon, 29 Jun 2026 20:54:24 +0800 Subject: [PATCH 027/101] Use simdutf for remaining base64 decodes --- src/workerd/api/BUILD.bazel | 2 +- src/workerd/api/data-url.c++ | 40 ++++++++++++++++++++++++++++---- src/workerd/api/global-scope.c++ | 12 ++++++---- 3 files changed, 45 insertions(+), 9 deletions(-) diff --git a/src/workerd/api/BUILD.bazel b/src/workerd/api/BUILD.bazel index e548f11c99d..b56993485ce 100644 --- a/src/workerd/api/BUILD.bazel +++ b/src/workerd/api/BUILD.bazel @@ -455,7 +455,6 @@ wd_cc_library( srcs = ["data-url.c++"], hdrs = ["data-url.h"], implementation_deps = [ - ":encoding", "//src/workerd/util:strings", ], visibility = ["//visibility:public"], @@ -463,6 +462,7 @@ wd_cc_library( "//src/workerd/jsg:url", "//src/workerd/util:mimetype", "@capnp-cpp//src/kj", + "@simdutf", ], ) diff --git a/src/workerd/api/data-url.c++ b/src/workerd/api/data-url.c++ index 0fe648f202c..5ea494df9a1 100644 --- a/src/workerd/api/data-url.c++ +++ b/src/workerd/api/data-url.c++ @@ -1,11 +1,44 @@ #include "data-url.h" -#include +#include "simdutf.h" + #include -#include +#include namespace workerd::api { +namespace { + +kj::Array decodeDataUrlBase64(kj::ArrayPtr input) { + kj::Vector filtered(input.size()); + + for (kj::byte c: input) { + if (isAlpha(c) || isDigit(c) || c == '+' || c == '/') { + filtered.add(static_cast(c)); + } + } + + auto base64 = filtered.asPtr(); + if (base64.size() % 4 == 1) { + base64 = base64.first(base64.size() - 1); + } + + if (base64.size() == 0) { + return nullptr; + } + + auto size = simdutf::maximal_binary_length_from_base64(base64.begin(), base64.size()); + auto decoded = kj::heapArray(size); + auto result = simdutf::base64_to_binary( + base64.begin(), base64.size(), decoded.asChars().begin(), simdutf::base64_default); + if (result.error != simdutf::SUCCESS || result.count == 0) { + return nullptr; + } + KJ_ASSERT(result.count <= size); + return decoded.first(result.count).attach(kj::mv(decoded)); +} + +} // namespace kj::Maybe DataUrl::tryParse(kj::StringPtr url) { KJ_IF_SOME(url, jsg::Url::tryParse(url)) { @@ -53,8 +86,7 @@ kj::Maybe DataUrl::from(const jsg::Url& url) { kj::Array decoded = nullptr; if (isBase64(unparsed)) { unparsed = unparsed.first(KJ_ASSERT_NONNULL(unparsed.findLast(';'))); - decoded = - kj::decodeBase64(stripInnerWhitespace(jsg::Url::percentDecode(data.asBytes())).asChars()); + decoded = decodeDataUrlBase64(jsg::Url::percentDecode(data.asBytes())); } else { decoded = jsg::Url::percentDecode(data.asBytes()); } diff --git a/src/workerd/api/global-scope.c++ b/src/workerd/api/global-scope.c++ index 434ea9c3236..53b7763d5f0 100644 --- a/src/workerd/api/global-scope.c++ +++ b/src/workerd/api/global-scope.c++ @@ -928,17 +928,21 @@ void ServiceWorkerGlobalScope::fuzzilli(jsg::Lock& js, jsg::Arguments(size); + auto result = simdutf::base64_to_binary( + data.begin(), data.size(), decoded.asChars().begin(), simdutf::base64_default); - JSG_REQUIRE(!decoded.hadErrors, DOMInvalidCharacterError, + JSG_REQUIRE(result.error == simdutf::SUCCESS, DOMInvalidCharacterError, "atob() called with invalid base64-encoded data. (Only whitespace, '+', '/', alphanumeric " "ASCII, and up to two terminal '=' signs when the input data length is divisible by 4 are " "allowed.)"); // Similar to btoa() taking a v8::Value, we return a v8::String directly, as this allows us to - // construct a string from the non-nul-terminated array returned from decodeBase64(). This avoids + // construct a string from the non-nul-terminated array returned from base64_to_binary(). This avoids // making a copy purely to append a nul byte. - return js.str(decoded.asBytes()); + KJ_ASSERT(result.count <= size); + return js.str(decoded.first(result.count)); } void ServiceWorkerGlobalScope::queueMicrotask(jsg::Lock& js, jsg::Function task) { From 3cd5c6c01b377c155877243082cb316190217c26 Mon Sep 17 00:00:00 2001 From: Logan Gatlin Date: Mon, 6 Jul 2026 19:29:30 +0000 Subject: [PATCH 028/101] Make new module registry error classes consistent across resolve paths * Expand error-class consistency tests to cover all paths and conditions Address review feedback on MR !374: * errorClassConsistency now verifies "Module not found" is a plain Error on all three paths (dynamic import, static import, and require), and that a circular dependency surfaces as an Error (not a TypeError). * Add invalidModuleSpecifier, asserting an unparseable specifier is a TypeError on both the dynamic-import and static-import paths. The static case (static-invalid-spec) directly exercises the path whose class was corrected from Error to TypeError. * Clarify that invalidUrlAsSpecifier ('zebra: ...') deliberately tests the not-found path: that specifier parses as a URL and fails as not-found, so it never reaches the invalid-specifier branch. Adds bundle modules static-import-missing, circular-a/b/c, and static-invalid-spec to drive the static-import and circular paths. * Make new module registry error classes consistent across resolve paths The new module registry chose different JS error classes for the same failure depending on which resolution path detected it: * "Module not found": Error on the static-import/require paths but TypeError on the dynamic-import path. * "Invalid module specifier": Error on the static-import path but TypeError on the dynamic-import path. * "Circular dependency when resolving module": Error on the require path but TypeError on the static/dynamic instantiation guards. Semantically a TypeError means a value is of the wrong type, so: * not found -> Error (a lookup failure; matches Node's ERR_MODULE_NOT_FOUND, which extends Error) * invalid spec -> TypeError (a malformed value; matches Node's ERR_INVALID_MODULE_SPECIFIER, a TypeError) * circular dep -> Error (a module-graph/loading error; matches Node's ERR_REQUIRE_CYCLE_MODULE, an Error) Normalize all paths accordingly and add regression coverage. See merge request cloudflare/ew/workerd!374 --- .../api/tests/new-module-registry-test.js | 91 +++++++++++++++++++ .../tests/new-module-registry-test.wd-test | 15 +++ src/workerd/jsg/modules-new-test.c++ | 2 +- src/workerd/jsg/modules-new.c++ | 20 +++- 4 files changed, 123 insertions(+), 5 deletions(-) diff --git a/src/workerd/api/tests/new-module-registry-test.js b/src/workerd/api/tests/new-module-registry-test.js index d893cfc38fb..cb7a49a8cfb 100644 --- a/src/workerd/api/tests/new-module-registry-test.js +++ b/src/workerd/api/tests/new-module-registry-test.js @@ -271,6 +271,10 @@ export const importAssertionsFail = { }, }; +// Note: 'zebra: ...' parses as a URL (scheme "zebra" with an opaque path), so it +// resolves successfully and then fails as *not found*. This is deliberately the +// not-found path, NOT the invalid-specifier path (covered by +// `invalidModuleSpecifier` below). export const invalidUrlAsSpecifier = { async test() { await rejects(import('zebra: not a \x00 valid URL'), { @@ -279,6 +283,93 @@ export const invalidUrlAsSpecifier = { }, }; +// The error class for a given module-resolution failure should be consistent +// regardless of which path (static-import, dynamic-import, or require()) +// detects it: +// * "Module not found" -> Error (a lookup failure, not a type error) +// * "Invalid module specifier" -> TypeError (a malformed value); see +// `invalidModuleSpecifier` +// * "Circular dependency ..." -> Error (a module-graph/loading error) +// See the resolve/dynamicResolve/require paths in jsg/modules-new.c++. The exact +// "Circular dependency when resolving module" message+class is pinned by the C++ +// test in jsg/modules-new-test.c++; at the JS boundary it surfaces as a (still +// Error-classed) "Failed to instantiate module". +export const errorClassConsistency = { + async test() { + const myRequire = createRequire(import.meta.url); + + const assertPlainError = (err, label) => { + ok(err instanceof Error, `${label}: expected an Error, got ${err}`); + strictEqual( + Object.getPrototypeOf(err), + Error.prototype, + `${label}: expected a plain Error, not a subclass like TypeError` + ); + }; + + // "Module not found" is a plain Error on every path. + + // Dynamic import: + const dynNotFound = await import('module-not-found').then( + () => null, + (e) => e + ); + assertPlainError(dynNotFound, 'dynamic import not-found'); + ok(/Module not found/.test(dynNotFound.message), dynNotFound.message); + + // Static import: importing a module whose own static import is missing + // surfaces the static-import resolution failure. + const staticNotFound = await import('static-import-missing').then( + () => null, + (e) => e + ); + assertPlainError(staticNotFound, 'static import not-found'); + ok(/Module not found/.test(staticNotFound.message), staticNotFound.message); + + // require(): + let reqNotFound = null; + try { + myRequire('module-not-found'); + } catch (e) { + reqNotFound = e; + } + assertPlainError(reqNotFound, 'require not-found'); + ok(/Module not found/.test(reqNotFound.message), reqNotFound.message); + + // A circular dependency is also a plain Error (not a TypeError). + const circular = await import('circular-a').then( + () => null, + (e) => e + ); + assertPlainError(circular, 'circular dependency'); + }, +}; + +// A malformed/unparseable specifier is a TypeError on both the dynamic-import +// and static-import paths (matching Node's ERR_INVALID_MODULE_SPECIFIER, which +// extends TypeError). 'https://' is a special-scheme URL with no host, so it +// fails to parse rather than resolving to a (missing) module. +export const invalidModuleSpecifier = { + async test() { + // Dynamic import: + const dyn = await import('https://').then( + () => null, + (e) => e + ); + ok(dyn instanceof TypeError, `expected TypeError, got ${dyn && dyn.name}`); + ok(/Invalid module specifier/.test(dyn.message), dyn.message); + + // Static import: a bundle ESM that statically imports the unparseable + // specifier surfaces the same TypeError. + const stat = await import('static-invalid-spec').then( + () => null, + (e) => e + ); + ok(stat instanceof TypeError, `expected TypeError, got ${stat && stat.name}`); + ok(/Invalid module specifier/.test(stat.message), stat.message); + }, +}; + export const evalErrorsInEsmTopLevel = { async test() { await rejects(import('esm-error'), { diff --git a/src/workerd/api/tests/new-module-registry-test.wd-test b/src/workerd/api/tests/new-module-registry-test.wd-test index 2435e8cea12..3c557290278 100644 --- a/src/workerd/api/tests/new-module-registry-test.wd-test +++ b/src/workerd/api/tests/new-module-registry-test.wd-test @@ -71,6 +71,21 @@ const unitTests :Workerd.Config = ( (name = "部品", esModule = "export default 1;"), (name = "tla", esModule = "export default await import('text')"), + + # An ESM that statically imports a module that does not exist. Importing + # this exercises the static-import resolution path's not-found error. + (name = "static-import-missing", esModule = "import 'definitely-not-a-real-module-xyz'; export default 1;"), + + # ESM (circular-a) -> CJS (circular-b) -> require(ESM circular-c) which + # statically imports circular-b back while it is still evaluating. This + # exercises the circular-dependency guard on the instantiation path. + (name = "circular-a", esModule = "import b from 'circular-b'; export default b;"), + (name = "circular-b", commonJsModule = "module.exports = require('circular-c')"), + (name = "circular-c", esModule = "import b from 'circular-b'; export default b;"), + + # An ESM that statically imports an unparseable specifier, to exercise + # the static-import "Invalid module specifier" path. + (name = "static-invalid-spec", esModule = "import 'https://'; export default 1;"), ], compatibilityFlags = [ "nodejs_compat_v2", diff --git a/src/workerd/jsg/modules-new-test.c++ b/src/workerd/jsg/modules-new-test.c++ index 3bcca5e52e6..519ceb1c9c6 100644 --- a/src/workerd/jsg/modules-new-test.c++ +++ b/src/workerd/jsg/modules-new-test.c++ @@ -1442,7 +1442,7 @@ KJ_TEST("ESM -> CJS -> require(ESM) -> static import CJS circular dependency fai JSG_FAIL_REQUIRE(Error, "Should have thrown"); }, [&](Value exception) { auto str = kj::str(exception.getHandle(js)); - KJ_ASSERT(str == "TypeError: Circular dependency when resolving module: b"); + KJ_ASSERT(str == "Error: Circular dependency when resolving module: b"); }); }); } diff --git a/src/workerd/jsg/modules-new.c++ b/src/workerd/jsg/modules-new.c++ index 625a00749cf..34f97b3a4ea 100644 --- a/src/workerd/jsg/modules-new.c++ +++ b/src/workerd/jsg/modules-new.c++ @@ -554,7 +554,10 @@ class IsolateModuleRegistry final { } // Nothing found? Aw... fail! - JSG_FAIL_REQUIRE(TypeError, kj::str("Module not found: ", normalizedSpecifier.getHref())); + // A module that cannot be resolved is a lookup failure, not a type error, + // so this is an Error (matching the static-import and require paths, and + // Node's ERR_MODULE_NOT_FOUND which extends Error). + JSG_FAIL_REQUIRE(Error, kj::str("Module not found: ", normalizedSpecifier.getHref())); }, [&](Value exception) -> Promise { return js.rejectedPromise(kj::mv(exception)); })); @@ -1116,8 +1119,11 @@ v8::MaybeLocal> resolv return {}; } if (resolved->GetStatus() == v8::Module::kEvaluating) { + // A circular dependency is a module-graph/loading error, not a type + // error, so this is an Error (matching the require path and Node's + // ERR_REQUIRE_CYCLE_MODULE which extends Error). js.throwException( - js.typeError(kj::str("Circular dependency when resolving module: ", spec))); + js.error(kj::str("Circular dependency when resolving module: ", spec))); return {}; } // Validate import type attribute against the resolved module's content type. @@ -1152,8 +1158,11 @@ v8::MaybeLocal> resolv return {}; } if (resolved->GetStatus() == v8::Module::kEvaluating) { + // A circular dependency is a module-graph/loading error, not a type + // error, so this is an Error (matching the require path and Node's + // ERR_REQUIRE_CYCLE_MODULE which extends Error). js.throwException( - js.typeError(kj::str("Circular dependency when resolving module: ", spec))); + js.error(kj::str("Circular dependency when resolving module: ", spec))); return v8::MaybeLocal(); } @@ -1202,7 +1211,10 @@ v8::MaybeLocal> resolv KJ_UNREACHABLE; } - js.throwException(js.error(kj::str("Invalid module specifier: "_kj, specifier))); + // A malformed/unparseable specifier is a bad-value error, so this is a + // TypeError (matching the dynamic-import path and Node's + // ERR_INVALID_MODULE_SPECIFIER which extends TypeError). + js.throwException(js.typeError(kj::str("Invalid module specifier: "_kj, specifier))); return {}; }, [&](Value exception) -> v8::MaybeLocal { // If there are any synchronously thrown exceptions, we want to catch them From afc1df62c5dcb6c3259937d18c318c5be1c0a9b3 Mon Sep 17 00:00:00 2001 From: Aaron Snell Date: Mon, 6 Jul 2026 21:54:30 +0000 Subject: [PATCH 029/101] CC-8007: add abort signal to container exec * CC-8007: add abort signal to container exec Signed-off-by: asnell See merge request cloudflare/ew/workerd!342 --- src/workerd/api/container.c++ | 50 ++++++++++++++++--- src/workerd/api/container.h | 20 ++++++-- .../server/tests/container-client/test.js | 21 ++++++++ .../experimental/index.d.ts | 1 + .../generated-snapshot/experimental/index.ts | 1 + types/generated-snapshot/index.d.ts | 1 + types/generated-snapshot/index.ts | 1 + 7 files changed, 86 insertions(+), 9 deletions(-) diff --git a/src/workerd/api/container.c++ b/src/workerd/api/container.c++ index b7e11af4277..6725d83c69d 100644 --- a/src/workerd/api/container.c++ +++ b/src/workerd/api/container.c++ @@ -78,16 +78,44 @@ jsg::JsArrayBuffer ExecOutput::getStderr(jsg::Lock& js) { return jsg::JsArrayBuffer::create(js, stderrBytes); } -ExecProcess::ExecProcess(jsg::Optional> stdinStream, +ExecProcess::ExecProcess(jsg::Lock& js, + IoContext& ioContext, + jsg::Optional> stdinStream, jsg::Optional> stdoutStream, jsg::Optional> stderrStream, int pid, - rpc::Container::ProcessHandle::Client handle) + rpc::Container::ProcessHandle::Client handle, + kj::Maybe> abortSignal) : stdinStream(kj::mv(stdinStream)), stdoutStream(kj::mv(stdoutStream)), stderrStream(kj::mv(stderrStream)), pid(pid), - handle(IoContext::current().addObject(kj::heap(kj::mv(handle)))) {} + handle(ioContext.addObject(kj::heap(kj::mv(handle)))) { + KJ_IF_SOME(signal, abortSignal) { + constexpr int kSigKill = 9; + + auto& canceler = signal->getCanceler(); + + // exec() calls throwIfAborted() before sending the RPC, but the signal can still fire while the + // RPC is in flight, i.e. before this constructor runs in the RPC's continuation. If that + // happened, kill the freshly-started process immediately; there's no point registering a + // listener. + if (canceler.isCanceled()) { + sendKill(kSigKill); + } else { + // Hold a strong reference to the canceler so it outlives the AbortSignal's own IoOwn, then register + // a listener that kills the process when the signal is later triggered. + auto own = kj::addRef(canceler); + auto& ref = *own; + abortCanceler = ioContext.addObject(kj::mv(own)); + abortListener.emplace(ref, [self = JSG_THIS_WEAK(js)]() { + KJ_IF_SOME(process, self.tryGet()) { + process.sendKill(kSigKill); + } + }); + } + } +} jsg::Optional> ExecProcess::getStdin() { return stdinStream.map([](jsg::Ref& stream) { return stream.addRef(); }); @@ -192,10 +220,15 @@ jsg::Promise> ExecProcess::output(jsg::Lock& js) { void ExecProcess::kill(jsg::Lock& js, jsg::Optional signal) { auto signo = signal.orDefault(15); JSG_REQUIRE(signo > 0 && signo <= 64, RangeError, "Invalid signal number."); + sendKill(signo); +} +void ExecProcess::sendKill(int signo) { + // Grab the IoContext first so we fail fast if there is no active IoContext. + auto& ioContext = IoContext::current(); auto req = handle->killRequest(capnp::MessageSize{4, 0}); req.setSigno(signo); - IoContext::current().addTask(req.sendIgnoringResult()); + ioContext.addTask(req.sendIgnoringResult()); } // ======================================================================================= // Basic lifecycle methods @@ -459,6 +492,11 @@ jsg::Promise> Container::exec( JSG_REQUIRE(!combinedOutput || stdoutMode == "pipe", TypeError, "stderr: \"combined\" requires stdout to be \"pipe\"."); + // If an already-aborted signal is provided, fail fast before spawning anything. + KJ_IF_SOME(signal, options.signal) { + signal->throwIfAborted(js); + } + auto& ioContext = IoContext::current(); auto& byteStreamFactory = ioContext.getByteStreamFactory(); @@ -574,8 +612,8 @@ jsg::Promise> Container::exec( } // return the instance to the process after getting pipeline of the process handle - return js.alloc( - kj::mv(stdinStream), kj::mv(stdoutStream), kj::mv(stderrStream), pid, kj::mv(handle)); + return js.alloc(js, ioContext, kj::mv(stdinStream), kj::mv(stdoutStream), + kj::mv(stderrStream), pid, kj::mv(handle), kj::mv(options.signal)); }); } diff --git a/src/workerd/api/container.h b/src/workerd/api/container.h index 0f8cf0c499b..6c64953910d 100644 --- a/src/workerd/api/container.h +++ b/src/workerd/api/container.h @@ -5,12 +5,14 @@ #pragma once // Container management API for Durable Object-attached containers. // +#include #include #include #include #include #include #include +#include namespace workerd::api { @@ -57,8 +59,9 @@ struct ExecOptions { jsg::Optional cwd; jsg::Optional> env; jsg::Optional user; + jsg::Optional> signal; - JSG_STRUCT($stdin, $stdout, $stderr, cwd, env, user); + JSG_STRUCT($stdin, $stdout, $stderr, cwd, env, user, signal); JSG_STRUCT_TS_OVERRIDE(ContainerExecOptions { stdin?: ReadableStream | "pipe"; stdout?: "pipe" | "ignore"; @@ -66,6 +69,7 @@ struct ExecOptions { cwd?: string; env?: Record; user?: string; + signal?: AbortSignal; $stdin: never; $stdout: never; $stderr: never; @@ -74,11 +78,14 @@ struct ExecOptions { class ExecProcess: public jsg::Object { public: - ExecProcess(jsg::Optional> stdinStream, + ExecProcess(jsg::Lock& js, + IoContext& ioContext, + jsg::Optional> stdinStream, jsg::Optional> stdoutStream, jsg::Optional> stderrStream, int pid, - rpc::Container::ProcessHandle::Client handle); + rpc::Container::ProcessHandle::Client handle, + kj::Maybe> abortSignal = kj::none); jsg::Optional> getStdin(); jsg::Optional> getStdout(); @@ -123,6 +130,10 @@ class ExecProcess: public jsg::Object { void ensureExitCodePromise(jsg::Lock& js); jsg::Promise getExitCodeForOutput(jsg::Lock& js); + // Sends a kill signal to the underlying process. Used both by the public kill() method and by + // the AbortSignal handler. + void sendKill(int signo); + jsg::Optional> stdinStream; jsg::Optional> stdoutStream; jsg::Optional> stderrStream; @@ -133,6 +144,9 @@ class ExecProcess: public jsg::Object { kj::Maybe resolvedExitCode; bool outputCalled = false; + kj::Maybe> abortCanceler; + kj::Maybe abortListener; + void visitForGc(jsg::GcVisitor& visitor) { visitor.visit(stdinStream, stdoutStream, stderrStream, exitCodePromise, exitCodePromiseCopy); } diff --git a/src/workerd/server/tests/container-client/test.js b/src/workerd/server/tests/container-client/test.js index 6ce1574b4ee..dd8e494a7ac 100644 --- a/src/workerd/server/tests/container-client/test.js +++ b/src/workerd/server/tests/container-client/test.js @@ -305,6 +305,27 @@ export class DurableObjectExample extends DurableObject { await container.exec(['cat']).then((p) => p.output()); } + // 13. An already-aborted signal causes exec() to throw synchronously. + { + const ac = new AbortController(); + ac.abort(); + assert.throws(() => container.exec(['echo', 'hello'], { signal: ac.signal }), { + name: 'AbortError', + }); + } + + // 14. Aborting the signal while the process is running kills it (SIGKILL). + { + const ac = new AbortController(); + const proc = await container.exec(['sh', '-lc', 'sleep 60'], { + signal: ac.signal, + stdout: 'ignore', + }); + ac.abort(); + // A process killed by SIGKILL (9) reports exit code 128 + 9 = 137. + assert.strictEqual(await proc.exitCode, 137); + } + await container.destroy(); await monitor; assert.strictEqual(container.running, false); diff --git a/types/generated-snapshot/experimental/index.d.ts b/types/generated-snapshot/experimental/index.d.ts index 14f98616cac..d7def830f2a 100755 --- a/types/generated-snapshot/experimental/index.d.ts +++ b/types/generated-snapshot/experimental/index.d.ts @@ -3981,6 +3981,7 @@ interface ContainerExecOptions { cwd?: string; env?: Record; user?: string; + signal?: AbortSignal; stdin?: ReadableStream | "pipe"; stdout?: "pipe" | "ignore"; stderr?: "pipe" | "ignore" | "combined"; diff --git a/types/generated-snapshot/experimental/index.ts b/types/generated-snapshot/experimental/index.ts index f995722123e..e5858f07b3c 100755 --- a/types/generated-snapshot/experimental/index.ts +++ b/types/generated-snapshot/experimental/index.ts @@ -3991,6 +3991,7 @@ export interface ContainerExecOptions { cwd?: string; env?: Record; user?: string; + signal?: AbortSignal; stdin?: ReadableStream | "pipe"; stdout?: "pipe" | "ignore"; stderr?: "pipe" | "ignore" | "combined"; diff --git a/types/generated-snapshot/index.d.ts b/types/generated-snapshot/index.d.ts index 4304016cb19..a95dca4a88b 100755 --- a/types/generated-snapshot/index.d.ts +++ b/types/generated-snapshot/index.d.ts @@ -3866,6 +3866,7 @@ interface ContainerExecOptions { cwd?: string; env?: Record; user?: string; + signal?: AbortSignal; stdin?: ReadableStream | "pipe"; stdout?: "pipe" | "ignore"; stderr?: "pipe" | "ignore" | "combined"; diff --git a/types/generated-snapshot/index.ts b/types/generated-snapshot/index.ts index c6511876bca..64f510017c5 100755 --- a/types/generated-snapshot/index.ts +++ b/types/generated-snapshot/index.ts @@ -3876,6 +3876,7 @@ export interface ContainerExecOptions { cwd?: string; env?: Record; user?: string; + signal?: AbortSignal; stdin?: ReadableStream | "pipe"; stdout?: "pipe" | "ignore"; stderr?: "pipe" | "ignore" | "combined"; From bd97b5349455edfc8e4c09fea15b5b33446c8042 Mon Sep 17 00:00:00 2001 From: James M Snell Date: Mon, 6 Jul 2026 15:26:50 -0700 Subject: [PATCH 030/101] Fix for vuln 136997 --- src/workerd/io/bundle-fs-test.c++ | 66 +++++++++++++++++++++++++++++++ src/workerd/io/bundle-fs.c++ | 37 +++++++++++++++-- src/workerd/io/worker-fs.c++ | 35 +++++++++++++++- src/workerd/io/worker-fs.h | 8 ++++ 4 files changed, 140 insertions(+), 6 deletions(-) diff --git a/src/workerd/io/bundle-fs-test.c++ b/src/workerd/io/bundle-fs-test.c++ index babd5b2b99f..34b99a53753 100644 --- a/src/workerd/io/bundle-fs-test.c++ +++ b/src/workerd/io/bundle-fs-test.c++ @@ -266,5 +266,71 @@ KJ_TEST("Module names exceeding max bundle path depth are skipped") { }); } +KJ_TEST("Transpiled bundle module bodies outlive the WorkerSource (VULN-136997)") { + // Regression test for VULN-136997: a heap use-after-free. + // + // When a module is transpiled at load time (e.g. TypeScript type-stripping), + // the resulting bytes are owned by WorkerSource::EsModule::ownBody -- a + // transient rust::String that lives only as long as the WorkerSource. The + // /bundle directory is materialized lazily, long after the WorkerSource has + // been destroyed, so the File must take ownership of those bytes rather than + // aliasing them. + // + // This test builds such a source, obtains the (lazy) bundle directory, then + // destroys the source *before* the directory is materialized and read. Under + // the bug the read aliases freed memory; run under ASAN (`just test-asan`) to + // detect the regression deterministically. The content assertions additionally + // verify that the copied bytes are correct. + TestFixture fixture; + + fixture.runInIoContext([&](const TestFixture::Environment& env) { + kj::StringPtr kBody = "export default 42; // transpiled at load time"_kj; + + kj::Maybe> maybeDir; + { + // `ownBody` owns the bytes; `body` is a non-owning view into them -- exactly + // how workerd-api.c++ sets up a module transpiled during load. + ::rust::String ownBody(kBody.begin(), kBody.size()); + kj::ArrayPtr bodyView(ownBody.data(), ownBody.size()); + + kj::Vector modules(1); + modules.add(WorkerSource::Module{ + .name = "a/transpiled.js"_kj, + .content = WorkerSource::EsModule{.body = bodyView, .ownBody = kj::mv(ownBody)}, + }); + + auto config = WorkerSource(WorkerSource::ModulesSource{ + .mainModule = "a/transpiled.js"_kj, .modules = modules.releaseAsArray()}); + + maybeDir = getBundleDirectory(config); + + // config -- and thus ownBody's backing buffer -- is destroyed here, before + // the lazy directory below is ever materialized. + } + + auto& dir = KJ_ASSERT_NONNULL(maybeDir); + + // The first tryOpen materializes the whole tree (creating the File and + // discarding the builder closure), so the subsequent reads exercise the + // File's own storage with both the source and the closure long gone. + auto maybeFile = dir->tryOpen(env.js, kj::Path({"a", "transpiled.js"})); + auto& file = KJ_ASSERT_NONNULL(maybeFile).get>(); + + auto stat = file->stat(env.js); + KJ_EXPECT(stat.type == FsType::FILE); + KJ_EXPECT(stat.size == kBody.size()); + + auto readText = file->readAllText(env.js).get(); + KJ_EXPECT(readText == env.js.str(kBody)); + + auto readBytes = file->readAllBytes(env.js).get>(); + KJ_EXPECT(readBytes.getHandle(env.js).asArrayPtr() == kBody.asArray().asBytes()); + + // The bundle remains read-only even for files that own their body. + auto error = dir->remove(env.js, kj::Path({"a", "transpiled.js"})).get(); + KJ_EXPECT(error == FsError::READ_ONLY); + }); +} + } // namespace } // namespace workerd diff --git a/src/workerd/io/bundle-fs.c++ b/src/workerd/io/bundle-fs.c++ index 868f5b482f5..827500472d8 100644 --- a/src/workerd/io/bundle-fs.c++ +++ b/src/workerd/io/bundle-fs.c++ @@ -14,6 +14,17 @@ kj::Rc getBundleDirectory(const WorkerSource& conf) { struct Entry { kj::StringPtr name; kj::ArrayPtr data; + // When the module body is owned by the (transient) WorkerSource -- as is the case for + // TypeScript transpiled at load time, where the body lives in EsModule::ownBody -- we must copy + // it here, because the WorkerSource (and thus ownBody) is destroyed once worker setup completes. + // At materialization this owned copy is handed to the File itself (File::newReadable(kj::Array)), + // so the bytes outlive both the source and the lazy closure -- note the closure that holds these + // entries is itself destroyed after the directory is first materialized (see + // LazyDirectory::getDirectory), so a non-owning File would still dangle. When kj::none, `data` + // points into memory the caller guarantees outlives the directory (process-lifetime capnp + // buffers or a retained source clone). See VULN-136997 and the matching copy in + // worker-modules.h. + kj::Maybe> ownedData; }; kj::Vector entries; KJ_SWITCH_ONEOF(conf.variant) { @@ -27,10 +38,19 @@ kj::Rc getBundleDirectory(const WorkerSource& conf) { for (auto& module: modules.modules) { KJ_SWITCH_ONEOF(module.content) { KJ_CASE_ONEOF(esModule, WorkerSource::EsModule) { - entries.add(Entry{ + Entry entry{ .name = module.name, .data = esModule.body.asBytes(), - }); + }; + // If the body was transpiled at load time it is owned by the transient WorkerSource; + // copy it so the lazy directory does not read freed memory after the source is + // destroyed (VULN-136997). + if (esModule.ownBody != kj::none) { + auto owned = kj::heapArray(esModule.body.asBytes()); + entry.data = owned.asPtr(); + entry.ownedData = kj::mv(owned); + } + entries.add(kj::mv(entry)); } KJ_CASE_ONEOF(commonJsModule, WorkerSource::CommonJsModule) { entries.add(Entry{ @@ -80,7 +100,10 @@ kj::Rc getBundleDirectory(const WorkerSource& conf) { } } - return getLazyDirectoryImpl([entries = entries.releaseAsArray()] { + // `mutable` so we can move the owned module bytes out of the captured entries into their Files + // below. This is safe because the closure is invoked at most once: getLazyDirectoryImpl memoizes + // the produced Directory and discards the closure after the first call. + return getLazyDirectoryImpl([entries = entries.releaseAsArray()]() mutable { Directory::Builder builder; kj::Path kRoot{}; // Defense-in-depth: reject module names whose parsed path exceeds a sane @@ -99,7 +122,13 @@ kj::Rc getBundleDirectory(const WorkerSource& conf) { KJ_LOG(WARNING, "Skipping overly deep module path", path.size()); continue; } - builder.addPath(path, File::newReadable(entry.data)); + KJ_IF_SOME(owned, entry.ownedData) { + // The bytes are owned by the transient WorkerSource; transfer ownership to the File so + // they survive both the source's destruction and this closure's (VULN-136997). + builder.addPath(path, File::newReadable(kj::mv(owned))); + } else { + builder.addPath(path, File::newReadable(entry.data)); + } } return builder.finish(); }); diff --git a/src/workerd/io/worker-fs.c++ b/src/workerd/io/worker-fs.c++ index c718608d156..97f80392d9f 100644 --- a/src/workerd/io/worker-fs.c++ +++ b/src/workerd/io/worker-fs.c++ @@ -665,6 +665,14 @@ class FileImpl final: public File { FileImpl(kj::ArrayPtr data): ownedOrView(data), lastModified(kj::UNIX_EPOCH) {} FileImpl(kj::Array&& owned) = delete; + // Constructor used to create a read-only file that owns its backing buffer. + // Unlike the writable Owned mode, this does not require a jsg::Lock and its + // contents are not counted toward the isolate external memory usage (matching + // the non-owning read-only file). See File::newReadable(kj::Array). + FileImpl(kj::Array owned) + : ownedOrView(kj::mv(owned)), + lastModified(kj::UNIX_EPOCH) {} + // Constructor used to create a writable file. FileImpl(jsg::Lock& js, kj::Array owned) : ownedOrView(Owned(js, kj::mv(owned))), @@ -772,7 +780,9 @@ class FileImpl final: public File { } void jsgGetMemoryInfo(jsg::MemoryTracker& tracker) const override { - // We only track the memory if we own the data. + // We only track the memory if we own the data as a writable file. Read-only + // files (whether a view or an owned buffer) are controlled by the runtime + // and are intentionally not counted toward the isolate external memory. KJ_SWITCH_ONEOF(ownedOrView) { KJ_CASE_ONEOF(owned, Owned) { tracker.trackField("owned", owned.data); @@ -781,6 +791,9 @@ class FileImpl final: public File { KJ_CASE_ONEOF(view, kj::ArrayPtr) { return; } + KJ_CASE_ONEOF(ownedView, kj::Array) { + return; + } } } @@ -801,6 +814,13 @@ class FileImpl final: public File { kj::Rc file = kj::rc(js, kj::heapArray(view)); return kj::mv(file); } + KJ_CASE_ONEOF(ownedView, kj::Array) { + if (ownedView.size() > maxSize) [[unlikely]] { + return FsError::FILE_SIZE_LIMIT_EXCEEDED; + } + kj::Rc file = kj::rc(js, kj::heapArray(ownedView)); + return kj::mv(file); + } } KJ_UNREACHABLE; } @@ -849,7 +869,11 @@ class FileImpl final: public File { : data(kj::mv(data)), adjustment(js.getExternalMemoryAdjustment(this->data.size())) {} }; - kj::OneOf> ownedOrView; + // - Owned: writable, isolate-memory-tracked buffer (see Owned). + // - kj::ArrayPtr: read-only view into caller-owned memory. + // - kj::Array: read-only buffer owned by this file. + // Only the Owned alternative is writable (see isWritable()). + kj::OneOf, kj::Array> ownedOrView; kj::Date lastModified; mutable kj::Maybe maybeUniqueId; mutable kj::Maybe maybeMemoryAdjustment; @@ -875,6 +899,9 @@ class FileImpl final: public File { KJ_CASE_ONEOF(owned, Owned) { return owned.data.asPtr().asConst(); } + KJ_CASE_ONEOF(ownedView, kj::Array) { + return ownedView.asPtr(); + } } KJ_UNREACHABLE; } @@ -1265,6 +1292,10 @@ kj::Rc File::newReadable(kj::ArrayPtr data) { return kj::rc(data); } +kj::Rc File::newReadable(kj::Array data) { + return kj::rc(kj::mv(data)); +} + kj::Own newVirtualFileSystem( kj::Own fsMap, kj::Rc&& root, kj::Own observer) { return kj::heap(kj::mv(fsMap), kj::mv(root), kj::mv(observer)); diff --git a/src/workerd/io/worker-fs.h b/src/workerd/io/worker-fs.h index 484d5c463f6..a7ffcf0a3d2 100644 --- a/src/workerd/io/worker-fs.h +++ b/src/workerd/io/worker-fs.h @@ -289,6 +289,14 @@ class File: public kj::Refcounted { // will not count towards the isolate external memory usage. static kj::Rc newReadable(kj::ArrayPtr data) KJ_WARN_UNUSED_RESULT; + // Same as newReadable(kj::ArrayPtr) above, but the file takes ownership of the + // data. Use this when the backing buffer would not otherwise outlive the file + // -- e.g. bundle module bodies transpiled at load time (TypeScript), whose + // storage is freed once worker setup completes (VULN-136997). Like the + // non-owning overload, the file is read-only and its contents are not tracked + // and do not count towards the isolate external memory usage. + static kj::Rc newReadable(kj::Array data) KJ_WARN_UNUSED_RESULT; + virtual kj::StringPtr jsgGetMemoryName() const = 0; virtual size_t jsgGetMemorySelfSize() const = 0; virtual void jsgGetMemoryInfo(jsg::MemoryTracker& tracker) const = 0; From b51733965c6a07da7e55b9363a5389378e1b645d Mon Sep 17 00:00:00 2001 From: Guy Bedford Date: Mon, 6 Jul 2026 23:25:39 +0000 Subject: [PATCH 031/101] Add synthetic IP for Hyperdrive bindings, resolvable via node:dns * sockets: use ArrayPtr::first() instead of slice(0, ...) * ipv6 fix * ipv6 support, various fixes * Add synthetic IP for Hyperdrive bindings, resolvable via node:dns Some database drivers require the connection host to be an IP literal rather than a hostname, which meant they could not use the magic `.hyperdrive.local` hostname exposed by `Hyperdrive.host`. Each Hyperdrive binding now also generates a synthetic IPv4 address in the reserved 240.0.0.0/4 range, exposed as `Hyperdrive.ip`. Connecting to `ip:port` routes to the same place as the hostname via a connect override, and `node:dns` `lookup()` resolves the magic hostname to this synthetic IP so drivers that resolve-then-connect work as well. See merge request cloudflare/ew/workerd!424 --- src/node/internal/internal_dns.ts | 45 ++++ src/node/internal/sockets.d.ts | 2 + src/workerd/api/global-scope.c++ | 8 + src/workerd/api/global-scope.h | 6 + src/workerd/api/hyperdrive.c++ | 49 +++- src/workerd/api/hyperdrive.h | 12 + src/workerd/api/sockets.c++ | 93 ++++++- src/workerd/api/sockets.h | 4 + src/workerd/server/server-test.c++ | 249 ++++++++++++++++++ types/defines/hyperdrive.d.ts | 9 + .../experimental/index.d.ts | 9 + .../generated-snapshot/experimental/index.ts | 9 + types/generated-snapshot/index.d.ts | 9 + types/generated-snapshot/index.ts | 9 + 14 files changed, 502 insertions(+), 11 deletions(-) diff --git a/src/node/internal/internal_dns.ts b/src/node/internal/internal_dns.ts index 70c3a8ed842..a856dba4ea5 100644 --- a/src/node/internal/internal_dns.ts +++ b/src/node/internal/internal_dns.ts @@ -38,6 +38,7 @@ import { } from 'node-internal:validators'; import * as errorCodes from 'node-internal:internal_dns_constants'; import { isIP } from 'node-internal:internal_net'; +import inner from 'cloudflare-internal:sockets'; import type dns from 'node:dns'; type DnsOrder = 'verbatim' | 'ipv4first' | 'ipv6first'; @@ -51,6 +52,22 @@ export const validDnsOrders: DnsOrder[] = [ let defaultDnsOrder: DnsOrder = 'verbatim'; +// Magic hostnames (e.g. Hyperdrive's) resolve to a synthetic IPv4 that routes via a connect +// override. Gate on the suffix so ordinary lookups stay entirely in JS rather than crossing into +// C++ on every resolution. +function getMagicHostOverride(hostname: string): string | undefined { + if (!hostname.endsWith('.hyperdrive.local')) { + return undefined; + } + return inner.getCallerDnsOverride(hostname); +} + +// The synthetic address is IPv4; its IPv4-mapped IPv6 form is a known, derived value that connect() +// normalizes back to the IPv4 override key, so no separate registration is needed. +function toMappedIpv6(ipv4: string): string { + return `::ffff:${ipv4}`; +} + export function getServers(): ReturnType<(typeof dns)['getServers']> { return ['1.1.1.1', '2606:4700:4700::1111', '1.0.0.1', '2606:4700:4700::1001']; } @@ -142,6 +159,23 @@ export function lookup( return; } + const overrideIp = getMagicHostOverride(hostname); + if (overrideIp != null) { + // family 6 gets the IPv4-mapped form; family 0/4 gets the raw IPv4. + const address = family === 6 ? toMappedIpv6(overrideIp) : overrideIp; + const resultFamily = family === 6 ? 6 : 4; + // Deliver via queueMicrotask rather than process.nextTick: this path is reachable in + // node:dns configs where the `process` global is not defined. + queueMicrotask(() => { + if (all) { + callback(null, [{ address, family: resultFamily }]); + } else { + callback(null, address, resultFamily); + } + }); + return; + } + // If all is true and family is 0, we need to query both A and AAAA records if (all && family === 0) { Promise.all([ @@ -291,6 +325,11 @@ export function resolve4( // The following change is done to comply with Node.js behavior const ttl = !!options?.ttl; + const overrideIp = getMagicHostOverride(name); + if (overrideIp != null) { + return Promise.resolve([ttl ? { ttl: 0, address: overrideIp } : overrideIp]); + } + // Validation errors needs to be sync. // Return a promise rather than using async qualifier. return sendDnsRequest(name, 'A').then((json) => { @@ -311,6 +350,12 @@ export function resolve6( // The following change is done to comply with Node.js behavior const ttl = !!options?.ttl; + const overrideIp = getMagicHostOverride(name); + if (overrideIp != null) { + const address = toMappedIpv6(overrideIp); + return Promise.resolve([ttl ? { ttl: 0, address } : address]); + } + // Validation errors needs to be sync. // Return a promise rather than using async qualifier. return sendDnsRequest(name, 'AAAA').then((json) => { diff --git a/src/node/internal/sockets.d.ts b/src/node/internal/sockets.d.ts index 51c86852a70..7826ee7d06e 100644 --- a/src/node/internal/sockets.d.ts +++ b/src/node/internal/sockets.d.ts @@ -41,5 +41,7 @@ declare namespace sockets { secureTransport: 'on' | 'off' | 'starttls'; } ): Socket; + + function getCallerDnsOverride(hostname: string): string | undefined; } export default sockets; diff --git a/src/workerd/api/global-scope.c++ b/src/workerd/api/global-scope.c++ index 434ea9c3236..ede0606fca0 100644 --- a/src/workerd/api/global-scope.c++ +++ b/src/workerd/api/global-scope.c++ @@ -906,6 +906,14 @@ kj::Maybe ServiceWorkerGlobalScope::getCon return connectOverrides.find(networkAddress); } +void ServiceWorkerGlobalScope::setDnsOverride(kj::String hostname, kj::String ip) { + dnsOverrides.upsert(kj::mv(hostname), kj::mv(ip)); +} + +kj::Maybe ServiceWorkerGlobalScope::getDnsOverride(kj::StringPtr hostname) { + return dnsOverrides.find(hostname).map([](kj::String& ip) -> kj::StringPtr { return ip; }); +} + jsg::JsString ServiceWorkerGlobalScope::btoa(jsg::Lock& js, jsg::JsString str) { // We could implement btoa() by accepting a kj::String, but then we'd have to check that it // doesn't have any multibyte code points. Easier to perform that test using v8::String's diff --git a/src/workerd/api/global-scope.h b/src/workerd/api/global-scope.h index 4b1c28cd2ed..ef8d67cee30 100644 --- a/src/workerd/api/global-scope.h +++ b/src/workerd/api/global-scope.h @@ -752,6 +752,11 @@ class ServiceWorkerGlobalScope: public WorkerGlobalScope { void setConnectOverride(kj::String networkAddress, ConnectFn connectFn); kj::Maybe getConnectOverride(kj::StringPtr networkAddress); + // hostname->IP overrides so node:dns can resolve magic hostnames (e.g. Hyperdrive's) to a + // synthetic IP that has a corresponding connect override registered above. + void setDnsOverride(kj::String hostname, kj::String ip); + kj::Maybe getDnsOverride(kj::StringPtr hostname); + // --------------------------------------------------------------------------- // JS API @@ -1167,6 +1172,7 @@ class ServiceWorkerGlobalScope: public WorkerGlobalScope { kj::Maybe> bufferValue; kj::Maybe> defaultFetcher; kj::HashMap connectOverrides; + kj::HashMap dnsOverrides; void visitForGc(jsg::GcVisitor& visitor) { visitor.visit(processValue, bufferValue, defaultFetcher); diff --git a/src/workerd/api/hyperdrive.c++ b/src/workerd/api/hyperdrive.c++ index 7792faadb3f..ecdaf393d42 100644 --- a/src/workerd/api/hyperdrive.c++ +++ b/src/workerd/api/hyperdrive.c++ @@ -66,20 +66,49 @@ kj::StringPtr Hyperdrive::getScheme() { return this->scheme; } -kj::StringPtr Hyperdrive::getHost() { - if (!registeredConnectOverride) { - // Returns the random hostname and ensures the connect override is registered on the - // ServiceWorkerGlobalScope for the Worker. This getter has a side effect: it registers (or - // re-registers) an entry in the ServiceWorkerGlobalScope's connectOverrides HashMap so that - // cloudflare:sockets's connect() will route connections to this magic hostname through Hyperdrive. - auto& globalScope = IoContext::current().getCurrentLock().getGlobalScope(); - globalScope.setConnectOverride(kj::str(randomHost, ":", getPort()), - [self = JSG_THIS](jsg::Lock& js) mutable { return self->connect(js); }); - registeredConnectOverride = true; +void Hyperdrive::registerConnectOverride() { + if (registeredConnectOverride) { + return; + } + auto& globalScope = IoContext::current().getCurrentLock().getGlobalScope(); + auto port = getPort(); + + // Assign a synthetic IPv4 from 240.0.0.0/16 (within the reserved, non-routable 240.0.0.0/4 block) + // so it can never collide with a real host the Worker might legitimately connect to. The low 16 + // bits identify this binding; start at a random suffix and walk until we find one not already + // claimed by another Hyperdrive binding. + kj::FixedArray suffixBytes; + getEntropy(suffixBytes.asPtr()); + uint16_t start = (static_cast(suffixBytes[0]) << 8) | suffixBytes[1]; + uint16_t suffix = start; + for (;;) { + auto candidate = kj::str("240.0.", suffix >> 8, ".", suffix & 0xff); + if (globalScope.getConnectOverride(kj::str(candidate, ":", port)) == kj::none) { + randomIp = kj::mv(candidate); + break; + } + JSG_REQUIRE(++suffix != start, Error, "no free Hyperdrive address available"); } + + globalScope.setConnectOverride(kj::str(randomHost, ":", port), + [self = JSG_THIS](jsg::Lock& js) mutable { return self->connect(js); }); + globalScope.setConnectOverride(kj::str(randomIp, ":", port), + [self = JSG_THIS](jsg::Lock& js) mutable { return self->connect(js); }); + globalScope.setDnsOverride(kj::str(randomHost), kj::str(randomIp)); + registeredConnectOverride = true; +} + +kj::StringPtr Hyperdrive::getHost() { + // Reading the host/IP lazily registers the overrides that route it through Hyperdrive. + registerConnectOverride(); return randomHost; } +kj::StringPtr Hyperdrive::getIP() { + registerConnectOverride(); + return randomIp; +} + // We currently only support Postgres and MySQL uint16_t Hyperdrive::getPort() { if (scheme == "mysql") { diff --git a/src/workerd/api/hyperdrive.h b/src/workerd/api/hyperdrive.h index 94aca1b5cc7..35efa8e44a7 100644 --- a/src/workerd/api/hyperdrive.h +++ b/src/workerd/api/hyperdrive.h @@ -34,6 +34,11 @@ class Hyperdrive: public jsg::Object { kj::StringPtr getScheme(); kj::StringPtr getHost(); + + // A synthetic IPv4 (in the reserved 240.0.0.0/4 block) routing to this binding, for drivers that + // require an IP literal rather than a hostname. Connecting to it reaches the same place as host. + kj::StringPtr getIP(); + uint16_t getPort(); kj::String getConnectionString(); @@ -43,6 +48,7 @@ class Hyperdrive: public jsg::Object { JSG_LAZY_READONLY_INSTANCE_PROPERTY(user, getUser); JSG_LAZY_READONLY_INSTANCE_PROPERTY(password, getPassword); JSG_LAZY_READONLY_INSTANCE_PROPERTY(host, getHost); + JSG_LAZY_READONLY_INSTANCE_PROPERTY(ip, getIP); JSG_LAZY_READONLY_INSTANCE_PROPERTY(port, getPort); JSG_LAZY_READONLY_INSTANCE_PROPERTY(connectionString, getConnectionString); @@ -51,6 +57,7 @@ class Hyperdrive: public jsg::Object { void visitForMemoryInfo(jsg::MemoryTracker& tracker) const { tracker.trackField("randomHost", randomHost); + tracker.trackField("randomIp", randomIp); tracker.trackField("database", database); tracker.trackField("user", user); tracker.trackField("password", password); @@ -60,12 +67,17 @@ class Hyperdrive: public jsg::Object { private: uint clientIndex; kj::String randomHost; + kj::String randomIp; kj::String database; kj::String user; kj::String password; kj::String scheme; bool registeredConnectOverride = false; + // Registers the connect/dns overrides for the hostname and synthetic IP. Requires an active + // IoContext, so it is done lazily on first access rather than in the constructor. + void registerConnectOverride(); + kj::Promise> connectToDb(); }; #define EW_HYPERDRIVE_ISOLATE_TYPES api::Hyperdrive diff --git a/src/workerd/api/sockets.c++ b/src/workerd/api/sockets.c++ index 38b4fc41c71..3fb4cd069ea 100644 --- a/src/workerd/api/sockets.c++ +++ b/src/workerd/api/sockets.c++ @@ -79,6 +79,74 @@ kj::Maybe getWritableHighWaterMark(jsg::Optional& opts) return kj::none; } +kj::Maybe parseHex16(kj::ArrayPtr s) { + if (s.size() == 0 || s.size() > 4) { + return kj::none; + } + uint16_t v = 0; + for (char c: s) { + v <<= 4; + if (c >= '0' && c <= '9') { + v |= (c - '0'); + } else if (c >= 'a' && c <= 'f') { + v |= (c - 'a' + 10); + } else { + // Input is lowercased before parsing, so uppercase never reaches here. + return kj::none; + } + } + return v; +} + +// If `host` is an IPv4-mapped IPv6 address, return its dotted-quad IPv4 form; otherwise kj::none. +// The 96-bit `::ffff` prefix has three textual spellings (compressed, minimal, and fully-padded), +// and the trailing 32 bits may be written dotted (`192.168.1.1`) or as two hex groups +// (`c0a8:0101`). We match the prefix case-insensitively then decode the tail. Note +// `::ffff:0:0:c0a8:0101` is a *different* address and is correctly rejected (its tail has too many +// groups). Used solely to normalize the connect-override lookup key so a synthetic IPv4 override +// (e.g. Hyperdrive's) matches every mapped spelling; the real connection path is left untouched. +kj::Maybe tryGetMappedIpv4(kj::ArrayPtr host) { + // Strip surrounding brackets (the URL parser emits IPv6 hostnames bracketed) and lowercase for + // case-insensitive prefix/hex matching. + if (host.size() >= 2 && host.front() == '[' && host.back() == ']') { + host = host.slice(1, host.size() - 1); + } + auto lower = kj::heapString(host); + for (char& c: lower) { + if (c >= 'A' && c <= 'Z') { + c = c - 'A' + 'a'; + } + } + + static constexpr kj::StringPtr prefixes[] = { + "::ffff:"_kj, + "0:0:0:0:0:ffff:"_kj, + "0000:0000:0000:0000:0000:ffff:"_kj, + }; + kj::ArrayPtr tail; + bool matched = false; + for (auto prefix: prefixes) { + if (lower.startsWith(prefix)) { + tail = lower.asArray().slice(prefix.size()); + matched = true; + break; + } + } + if (!matched) { + return kj::none; + } + + // Dotted spelling: the tail is already the IPv4. + if (tail.findFirst('.') != kj::none) { + return kj::str(tail); + } + // Hex spelling: exactly two groups `hi:lo`. + auto colon = KJ_UNWRAP_OR_RETURN(tail.findFirst(':'), kj::none); + auto hi = KJ_UNWRAP_OR_RETURN(parseHex16(tail.first(colon)), kj::none); + auto lo = KJ_UNWRAP_OR_RETURN(parseHex16(tail.slice(colon + 1, tail.size())), kj::none); + return kj::str(hi >> 8, ".", hi & 0xff, ".", lo >> 8, ".", lo & 0xff); +} + } // namespace // Forward declarations @@ -197,6 +265,11 @@ jsg::Ref connectImpl(jsg::Lock& js, kj::String domain; bool isDefaultFetchPort = false; + // If the address is an IPv4-mapped IPv6 address, this holds the dotted-quad key its connect + // override would be registered under, so a synthetic IPv4 override (e.g. Hyperdrive's) also + // matches the mapped spelling. + kj::Maybe mappedOverrideKey; + KJ_SWITCH_ONEOF(address) { KJ_CASE_ONEOF(str, kj::String) { // We need just the hostname part of the address, i.e. we want to strip out the port. @@ -209,9 +282,15 @@ jsg::Ref connectImpl(jsg::Lock& js, JSG_REQUIRE(host != ""_kj, TypeError, "Specified address is missing hostname."); JSG_REQUIRE(port != ""_kj, TypeError, "Specified address is missing port."); isDefaultFetchPort = port == "443"_kj || port == "80"_kj; + KJ_IF_SOME(ipv4, tryGetMappedIpv4(host)) { + mappedOverrideKey = kj::str(ipv4, ":", port); + } domain = kj::str(host); } KJ_CASE_ONEOF(record, SocketAddress) { + KJ_IF_SOME(ipv4, tryGetMappedIpv4(record.hostname)) { + mappedOverrideKey = kj::str(ipv4, ":", record.port); + } domain = kj::heapString(record.hostname); isDefaultFetchPort = record.port == 443 || record.port == 80; } @@ -238,9 +317,15 @@ jsg::Ref connectImpl(jsg::Lock& js, // Support calling into arbitrary callbacks for any registered "magic" addresses for which // custom connect() logic is needed. Note that these overrides should only apply to calls of the // global connect() method, not for fetcher->connect(), hence why we check for them here. - KJ_IF_SOME(fn, ioContext.getCurrentLock().getGlobalScope().getConnectOverride(addressStr)) { + auto& globalScope = ioContext.getCurrentLock().getGlobalScope(); + KJ_IF_SOME(fn, globalScope.getConnectOverride(addressStr)) { return fn(js); } + KJ_IF_SOME(key, mappedOverrideKey) { + KJ_IF_SOME(fn, globalScope.getConnectOverride(key)) { + return fn(js); + } + } actualFetcher = js.alloc(IoContext::NULL_CLIENT_CHANNEL, Fetcher::RequiresHostAndProtocol::YES); } @@ -567,6 +652,12 @@ jsg::Ref SocketsModule::connect( return connectImpl(js, kj::none, kj::mv(address), kj::mv(options)); } +jsg::Optional SocketsModule::getCallerDnsOverride( + jsg::Lock& js, kj::String hostname) { + auto& ioContext = IoContext::current(); + return ioContext.getCurrentLock().getGlobalScope().getDnsOverride(hostname); +} + kj::Own Socket::takeConnectionStream(jsg::Lock& js) { // Set this so that if `close` is called after this, that no closure steps are taken and instead // the `close` is a no-op. diff --git a/src/workerd/api/sockets.h b/src/workerd/api/sockets.h index c25bb8c13ab..37b8cf3f450 100644 --- a/src/workerd/api/sockets.h +++ b/src/workerd/api/sockets.h @@ -276,8 +276,12 @@ class SocketsModule final: public jsg::Object { // Creates a Fetcher from a Socket that can perform HTTP requests over the socket connection jsg::Promise> internalNewHttpClient(jsg::Lock& js, jsg::Ref socket); + // Returns the synthetic IP registered for a magic hostname, or undefined. Used by node:dns. + jsg::Optional getCallerDnsOverride(jsg::Lock& js, kj::String hostname); + JSG_RESOURCE_TYPE(SocketsModule, CompatibilityFlags::Reader flags) { JSG_METHOD(connect); + JSG_METHOD(getCallerDnsOverride); if (flags.getWorkerdExperimental()) { JSG_METHOD(internalNewHttpClient); diff --git a/src/workerd/server/server-test.c++ b/src/workerd/server/server-test.c++ index 3fdc4b8749c..c4468daa7ac 100644 --- a/src/workerd/server/server-test.c++ +++ b/src/workerd/server/server-test.c++ @@ -1483,6 +1483,255 @@ KJ_TEST("Server: capability bindings") { )"_blockquote); } +KJ_TEST("Server: Hyperdrive connect via synthetic IP") { + TestServer test(R"(( + services = [ + ( name = "hello", + worker = ( + compatibilityDate = "2022-08-17", + modules = [ + ( name = "main.js", + esModule = + `import { connect } from 'cloudflare:sockets'; + `export default { + ` async fetch(request, env) { + ` const ip = env.hyperdrive.ip; + ` if (!/^240\.0\.\d{1,3}\.\d{1,3}$/.test(ip)) { + ` throw new Error(`unexpected hyperdrive ip: ${ip}`); + ` } + ` const connection = connect(`${ip}:5432`); + ` const encoded = new TextEncoder().encode("hyperdrive-ip-test"); + ` await connection.writable.getWriter().write(new Uint8Array(encoded)); + ` return new Response("OK"); + ` } + `} + ) + ], + bindings = [ + ( name = "hyperdrive", + hyperdrive = ( + designator = "hyperdrive-outbound", + database = "test-db", + user = "test-user", + password = "test-password", + scheme = "postgresql" + ) + ) + ] + ) + ), + ( name = "hyperdrive-outbound", external = ( + address = "hyperdrive-host", + tcp = () + )) + ], + sockets = [ + ( name = "main", + address = "test-addr", + service = "hello" + ) + ] + ))"_kj); + + test.start(); + auto conn = test.connect("test-addr"); + conn.sendHttpGet("/"); + + { + auto subreq = test.receiveSubrequest("hyperdrive-host"); + subreq.recv("hyperdrive-ip-test"); + } + conn.recvHttp200("OK"); +} + +KJ_TEST("Server: Hyperdrive host resolves via node:dns to synthetic IP") { + TestServer test(R"(( + services = [ + ( name = "hello", + worker = ( + compatibilityDate = "2022-08-17", + compatibilityFlags = ["nodejs_compat"], + modules = [ + ( name = "main.js", + esModule = + `import { connect } from 'cloudflare:sockets'; + `import { promises as dns } from 'node:dns'; + `export default { + ` async fetch(request, env) { + ` // Reading host primes the connect/dns overrides. + ` const host = env.hyperdrive.host; + ` const { address, family } = await dns.lookup(host); + ` if (family !== 4 || !/^240\.0\.\d{1,3}\.\d{1,3}$/.test(address)) { + ` throw new Error(`unexpected lookup result: ${address}/${family}`); + ` } + ` const connection = connect(`${address}:5432`); + ` const encoded = new TextEncoder().encode("hyperdrive-dns-test"); + ` await connection.writable.getWriter().write(new Uint8Array(encoded)); + ` return new Response("OK"); + ` } + `} + ) + ], + bindings = [ + ( name = "hyperdrive", + hyperdrive = ( + designator = "hyperdrive-outbound", + database = "test-db", + user = "test-user", + password = "test-password", + scheme = "postgresql" + ) + ) + ] + ) + ), + ( name = "hyperdrive-outbound", external = ( + address = "hyperdrive-host", + tcp = () + )) + ], + sockets = [ + ( name = "main", + address = "test-addr", + service = "hello" + ) + ] + ))"_kj); + + test.start(); + auto conn = test.connect("test-addr"); + conn.sendHttpGet("/"); + + { + auto subreq = test.receiveSubrequest("hyperdrive-host"); + subreq.recv("hyperdrive-dns-test"); + } + conn.recvHttp200("OK"); +} + +KJ_TEST("Server: Hyperdrive host resolves via node:dns resolve4 to synthetic IP") { + TestServer test(R"(( + services = [ + ( name = "hello", + worker = ( + compatibilityDate = "2022-08-17", + compatibilityFlags = ["nodejs_compat"], + modules = [ + ( name = "main.js", + esModule = + `import { connect } from 'cloudflare:sockets'; + `import { promises as dns } from 'node:dns'; + `export default { + ` async fetch(request, env) { + ` // Reading host primes the connect/dns overrides. + ` const host = env.hyperdrive.host; + ` const [address] = await dns.resolve4(host); + ` if (!/^240\.0\.\d{1,3}\.\d{1,3}$/.test(address)) { + ` throw new Error(`unexpected resolve4 result: ${address}`); + ` } + ` const connection = connect(`${address}:5432`); + ` const encoded = new TextEncoder().encode("hyperdrive-resolve4-test"); + ` await connection.writable.getWriter().write(new Uint8Array(encoded)); + ` return new Response("OK"); + ` } + `} + ) + ], + bindings = [ + ( name = "hyperdrive", + hyperdrive = ( + designator = "hyperdrive-outbound", + database = "test-db", + user = "test-user", + password = "test-password", + scheme = "postgresql" + ) + ) + ] + ) + ), + ( name = "hyperdrive-outbound", external = ( + address = "hyperdrive-host", + tcp = () + )) + ], + sockets = [ + ( name = "main", + address = "test-addr", + service = "hello" + ) + ] + ))"_kj); + + test.start(); + auto conn = test.connect("test-addr"); + conn.sendHttpGet("/"); + + { + auto subreq = test.receiveSubrequest("hyperdrive-host"); + subreq.recv("hyperdrive-resolve4-test"); + } + conn.recvHttp200("OK"); +} + +KJ_TEST("Server: Hyperdrive connect via IPv4-mapped IPv6") { + TestServer test(R"(( + services = [ + ( name = "hello", + worker = ( + compatibilityDate = "2022-08-17", + modules = [ + ( name = "main.js", + esModule = + `import { connect } from 'cloudflare:sockets'; + `export default { + ` async fetch(request, env) { + ` // The IPv4-mapped IPv6 spelling must route to the same override as the IPv4. + ` const connection = connect(`[::ffff:${env.hyperdrive.ip}]:5432`); + ` const encoded = new TextEncoder().encode("hyperdrive-mapped-test"); + ` await connection.writable.getWriter().write(new Uint8Array(encoded)); + ` return new Response("OK"); + ` } + `} + ) + ], + bindings = [ + ( name = "hyperdrive", + hyperdrive = ( + designator = "hyperdrive-outbound", + database = "test-db", + user = "test-user", + password = "test-password", + scheme = "postgresql" + ) + ) + ] + ) + ), + ( name = "hyperdrive-outbound", external = ( + address = "hyperdrive-host", + tcp = () + )) + ], + sockets = [ + ( name = "main", + address = "test-addr", + service = "hello" + ) + ] + ))"_kj); + + test.start(); + auto conn = test.connect("test-addr"); + conn.sendHttpGet("/"); + + { + auto subreq = test.receiveSubrequest("hyperdrive-host"); + subreq.recv("hyperdrive-mapped-test"); + } + conn.recvHttp200("OK"); +} + KJ_TEST("Server: cyclic bindings") { TestServer test(R"(( services = [ diff --git a/types/defines/hyperdrive.d.ts b/types/defines/hyperdrive.d.ts index e134e649216..3399f889b20 100644 --- a/types/defines/hyperdrive.d.ts +++ b/types/defines/hyperdrive.d.ts @@ -26,6 +26,15 @@ interface Hyperdrive { * for your database. */ readonly host: string; + /* + * A synthetic IPv4 address (in the reserved 240.0.0.0/4 range) that, like the + * host field, is only valid within the context of the currently running + * Worker and, when passed into the `connect()` function from the + * "cloudflare:sockets" module, will connect to the Hyperdrive instance for + * your database. This is provided for database drivers that require the host + * to be an IP literal rather than a hostname. + */ + readonly ip: string; /* * The port that must be paired the the host field when connecting. */ diff --git a/types/generated-snapshot/experimental/index.d.ts b/types/generated-snapshot/experimental/index.d.ts index d7def830f2a..04247055096 100755 --- a/types/generated-snapshot/experimental/index.d.ts +++ b/types/generated-snapshot/experimental/index.d.ts @@ -14573,6 +14573,15 @@ interface Hyperdrive { * for your database. */ readonly host: string; + /* + * A synthetic IPv4 address (in the reserved 240.0.0.0/4 range) that, like the + * host field, is only valid within the context of the currently running + * Worker and, when passed into the `connect()` function from the + * "cloudflare:sockets" module, will connect to the Hyperdrive instance for + * your database. This is provided for database drivers that require the host + * to be an IP literal rather than a hostname. + */ + readonly ip: string; /* * The port that must be paired the the host field when connecting. */ diff --git a/types/generated-snapshot/experimental/index.ts b/types/generated-snapshot/experimental/index.ts index e5858f07b3c..53d1dd39ae1 100755 --- a/types/generated-snapshot/experimental/index.ts +++ b/types/generated-snapshot/experimental/index.ts @@ -14600,6 +14600,15 @@ export interface Hyperdrive { * for your database. */ readonly host: string; + /* + * A synthetic IPv4 address (in the reserved 240.0.0.0/4 range) that, like the + * host field, is only valid within the context of the currently running + * Worker and, when passed into the `connect()` function from the + * "cloudflare:sockets" module, will connect to the Hyperdrive instance for + * your database. This is provided for database drivers that require the host + * to be an IP literal rather than a hostname. + */ + readonly ip: string; /* * The port that must be paired the the host field when connecting. */ diff --git a/types/generated-snapshot/index.d.ts b/types/generated-snapshot/index.d.ts index a95dca4a88b..2f12217385e 100755 --- a/types/generated-snapshot/index.d.ts +++ b/types/generated-snapshot/index.d.ts @@ -13925,6 +13925,15 @@ interface Hyperdrive { * for your database. */ readonly host: string; + /* + * A synthetic IPv4 address (in the reserved 240.0.0.0/4 range) that, like the + * host field, is only valid within the context of the currently running + * Worker and, when passed into the `connect()` function from the + * "cloudflare:sockets" module, will connect to the Hyperdrive instance for + * your database. This is provided for database drivers that require the host + * to be an IP literal rather than a hostname. + */ + readonly ip: string; /* * The port that must be paired the the host field when connecting. */ diff --git a/types/generated-snapshot/index.ts b/types/generated-snapshot/index.ts index 64f510017c5..929e893acfd 100755 --- a/types/generated-snapshot/index.ts +++ b/types/generated-snapshot/index.ts @@ -13952,6 +13952,15 @@ export interface Hyperdrive { * for your database. */ readonly host: string; + /* + * A synthetic IPv4 address (in the reserved 240.0.0.0/4 range) that, like the + * host field, is only valid within the context of the currently running + * Worker and, when passed into the `connect()` function from the + * "cloudflare:sockets" module, will connect to the Hyperdrive instance for + * your database. This is provided for database drivers that require the host + * to be an IP literal rather than a hostname. + */ + readonly ip: string; /* * The port that must be paired the the host field when connecting. */ From 6d429c9ee63e6d1193f6d2ebdb448fb1cda4b4d5 Mon Sep 17 00:00:00 2001 From: James M Snell Date: Thu, 25 Jun 2026 14:39:16 -0700 Subject: [PATCH 032/101] Initial TS streams impl --- src/per_isolate/AGENTS.md | 95 + src/per_isolate/main.ts | 146 + src/per_isolate/per_isolate-env.d.ts | 33 +- src/per_isolate/primordials.ts | 115 +- src/per_isolate/tsconfig.json | 4 + src/per_isolate/webstreams/encoding.ts | 229 + src/per_isolate/webstreams/identity.ts | 315 ++ src/per_isolate/webstreams/native.ts | 1413 +++++++ src/per_isolate/webstreams/queue.ts | 1317 ++++++ src/per_isolate/webstreams/readable.ts | 3668 +++++++++++++++++ src/per_isolate/webstreams/strategies.ts | 98 + src/per_isolate/webstreams/streams.ts | 72 + src/per_isolate/webstreams/transform.ts | 712 ++++ src/per_isolate/webstreams/types.d.ts | 213 + src/per_isolate/webstreams/writable.ts | 1323 ++++++ src/workerd/api/tests/BUILD.bazel | 6 + src/workerd/api/tests/ts-webstreams-test.js | 50 + .../api/tests/ts-webstreams-test.wd-test | 21 + src/workerd/io/compatibility-date.capnp | 6 + src/wpt/BUILD.bazel | 9 + src/wpt/harness/assertions.ts | 26 +- src/wpt/streams-test-ts.ts | 220 + 22 files changed, 10086 insertions(+), 5 deletions(-) create mode 100644 src/per_isolate/AGENTS.md create mode 100644 src/per_isolate/webstreams/encoding.ts create mode 100644 src/per_isolate/webstreams/identity.ts create mode 100644 src/per_isolate/webstreams/native.ts create mode 100644 src/per_isolate/webstreams/queue.ts create mode 100644 src/per_isolate/webstreams/readable.ts create mode 100644 src/per_isolate/webstreams/strategies.ts create mode 100644 src/per_isolate/webstreams/streams.ts create mode 100644 src/per_isolate/webstreams/transform.ts create mode 100644 src/per_isolate/webstreams/types.d.ts create mode 100644 src/per_isolate/webstreams/writable.ts create mode 100644 src/workerd/api/tests/ts-webstreams-test.js create mode 100644 src/workerd/api/tests/ts-webstreams-test.wd-test create mode 100644 src/wpt/streams-test-ts.ts diff --git a/src/per_isolate/AGENTS.md b/src/per_isolate/AGENTS.md new file mode 100644 index 00000000000..c4f7b31023c --- /dev/null +++ b/src/per_isolate/AGENTS.md @@ -0,0 +1,95 @@ +# src/per_isolate/ + +Per-isolate JavaScript/TypeScript bootstrap: scripts that run synchronously +at context creation, before any user code. Gated by the +`per-isolate-javascript-bootstrap` autogate (config: +`workerd-autogate-per-isolate-javascript-bootstrap`). C++ entry point: +`src/workerd/io/per-isolate-bootstrap.c++`. + +## EXECUTION MODEL + +- Scripts are compiled as functions with a context-extension object — the + pseudo-globals `require`, `module`, `exports`, `compatFlags`, + `autogates`, `primordials`, `utils` are in scope but NOT on + `globalThis` (see `per_isolate-env.d.ts`). +- Module system is bootstrap CommonJS: `require('webstreams/queue')` + + `module.exports = {...}`. The `src/node/` ESM-only rule does NOT apply + here. Circular requires are a FATAL startup error — keep modules + acyclic (e.g., `webstreams/native` is a deliberate leaf). +- TypeScript `import type` / `export type` are used freely for type + plumbing (fully erased; the loader only sees `module.exports`). +- `main.ts` installs the (TEMPORARY, dev-only) lazy `globalThis.streams` + surface. +- Files are auto-discovered (`BUILD.bazel` and `tsconfig.json` both glob + `**/*.ts`). No registration needed for new files. +- Local convention: no copyright headers in this directory's bootstrap + scripts (deviation from the repo-wide new-file rule; keep consistent). + +## CONVENTIONS + +- **Primordials discipline**: no bare prototype lookups on builtins after + bootstrap captures; no `for...of` over patchable iterables. Capture + methods/getters at module scope via `uncurryThis`. +- **JSG property capture trap**: readonly JSG properties live on the + PROTOTYPE only under modern compat + (`workers_api_getters_setters_on_prototype`, default-on 2022-01-31). + Under older dates JSG uses `SetNativeDataProperty` on the INSTANCE + template — an own native DATA property per instance; **no getter + function exists anywhere to capture**. Use `captureJsgGetter` in + the `primordials.ts` : prototype-accessor capture with a plain-read fallback + (an own data property read never consults the patchable prototype chain). + The bootstrap runs for every worker regardless of compat date — both layouts + must work. JSG *methods* are always on the prototype. +- Internal cross-module surfaces are exported as a single namespace + object (`internalsForPipe`, `nativeStreamInternals`), never + re-exported to users. +- Internal class dispatch uses private-brand `in` checks, NEVER + `instanceof` (classes reachable from the global have user-reachable + `Symbol.hasInstance`). + +## WEBSTREAMS ARCHITECTURE (webstreams/) + +TypeScript Streams implementation with a backend-blind reader layer and +two consumer backends behind the `StreamConsumer`/`ByteStreamConsumer` +fence (authoritative docs are IN-SOURCE — read these headers first): + +| File | Role | +| ------------- | -------------------------------------------------------------------------------------------- | +| `queue.ts` | QUEUED backend: single-queue/multi-cursor, JS sources; fence interfaces; invariant list | +| `native.ts` | NATIVE backend: C++-(eventually-)backed pull conduit; **the C++/JS contract** + invariants | +| `readable.ts` | Reader layer + queued controllers + the BACKEND-DISPATCH points (constructor, tee, chains, byte-capable gate, JS-to-C++ extraction) | +| `writable.ts` / `transform.ts` / `strategies.ts` | WHATWG writable/transform/strategies | +| `identity.ts` | IdentityTransformStream and FixedLengthStream (byte-capable identity transforms) | +| `encoding.ts` | TextEncoderStream and TextDecoderStream (pure JS codec transforms) | +| `streams.ts` | Module aggregator and temporary native-source exports | +| `types.d.ts` | TypeScript type definitions for the streams API | + +Key rules: + +- The reader layer must stay backend-blind; backend divergence is + confined to the fence interface and the marked BACKEND-DISPATCH points. +- Do not port logic across the fence without checking BOTH invariant + lists (`queue.ts` and `native.ts` headers). +- The native source contract (marker symbol, standard pull/cancel hooks, + byobRequest discrimination, once-per-pull delivery, per-pull abort + signal for cancellation, under-delivery = fused + `{done: true, value: partial}` EOF, tee hook, `expectedLength` + exact-total byte contract) is specified in the `native.ts` header. + The future C++ integration MUST conform to it; it is currently exercised + by JS mocks only. Key addition: `pull` receives an extension `signal` + argument — the source checks `signal.aborted` before delivery and stashes + bytes for redelivery if aborted (race buffering lives source-side; the JS + conduit is uniformly bufferless). +- `kNativeSource` is TEMPORARILY re-exported via `streams.ts` for tests; + it is removed when the real C++ handshake lands. + +## ANTI-PATTERNS + +- **NEVER** assume a JSG readonly property has a capturable getter (see + the capture trap above). +- **NEVER** add a runtime require cycle between bootstrap scripts. +- **NEVER** expose internals on user-visible exports (the temporary + `kNativeSource` exception is tracked for removal). +- **NEVER** rely on `readable-source-adapter.h` as a reference for the + native streams contract — it belongs to the original (non-enabled) + implementation; `native.ts` is authoritative. diff --git a/src/per_isolate/main.ts b/src/per_isolate/main.ts index 32ae51086f3..77c8f1ab98a 100644 --- a/src/per_isolate/main.ts +++ b/src/per_isolate/main.ts @@ -1,3 +1,4 @@ +'use strict'; // Per-isolate bootstrap entry point. // This script runs synchronously at context creation time, // before any user code. @@ -30,3 +31,148 @@ // const foo = require('./foo'); // (globalThis as any).Foo = foo.bootstrapFoo(compatFlags); // +const { ObjectDefineProperties } = primordials; + +if (compatFlags['typescript_implemented_streams']) { + const { + ReadableStream, + ReadableStreamDefaultReader, + ReadableStreamBYOBReader, + ReadableStreamDefaultController, + ReadableByteStreamController, + ReadableStreamBYOBRequest, + ByteLengthQueuingStrategy, + CountQueuingStrategy, + WritableStream, + WritableStreamDefaultWriter, + WritableStreamDefaultController, + TransformStream, + TransformStreamDefaultController, + IdentityTransformStream, + FixedLengthStream, + TextEncoderStream, + TextDecoderStream, + } = require('webstreams/streams'); + + ObjectDefineProperties(globalThis, { + ReadableStream: { + __proto__: null, + configurable: true, + enumerable: false, + writable: true, + value: ReadableStream, + }, + ReadableStreamDefaultReader: { + __proto__: null, + configurable: true, + enumerable: false, + writable: true, + value: ReadableStreamDefaultReader, + }, + ReadableStreamBYOBReader: { + __proto__: null, + configurable: true, + enumerable: false, + writable: true, + value: ReadableStreamBYOBReader, + }, + ReadableStreamDefaultController: { + __proto__: null, + configurable: true, + enumerable: false, + writable: true, + value: ReadableStreamDefaultController, + }, + ReadableByteStreamController: { + __proto__: null, + configurable: true, + enumerable: false, + writable: true, + value: ReadableByteStreamController, + }, + ReadableStreamBYOBRequest: { + __proto__: null, + configurable: true, + enumerable: false, + writable: true, + value: ReadableStreamBYOBRequest, + }, + ByteLengthQueuingStrategy: { + __proto__: null, + configurable: true, + enumerable: false, + writable: true, + value: ByteLengthQueuingStrategy, + }, + CountQueuingStrategy: { + __proto__: null, + configurable: true, + enumerable: false, + writable: true, + value: CountQueuingStrategy, + }, + WritableStream: { + __proto__: null, + configurable: true, + enumerable: false, + writable: true, + value: WritableStream, + }, + WritableStreamDefaultWriter: { + __proto__: null, + configurable: true, + enumerable: false, + writable: true, + value: WritableStreamDefaultWriter, + }, + WritableStreamDefaultController: { + __proto__: null, + configurable: true, + enumerable: false, + writable: true, + value: WritableStreamDefaultController, + }, + TransformStream: { + __proto__: null, + configurable: true, + enumerable: false, + writable: true, + value: TransformStream, + }, + TransformStreamDefaultController: { + __proto__: null, + configurable: true, + enumerable: false, + writable: true, + value: TransformStreamDefaultController, + }, + IdentityTransformStream: { + __proto__: null, + configurable: true, + enumerable: false, + writable: true, + value: IdentityTransformStream, + }, + FixedLengthStream: { + __proto__: null, + configurable: true, + enumerable: false, + writable: true, + value: FixedLengthStream, + }, + TextEncoderStream: { + __proto__: null, + configurable: true, + enumerable: false, + writable: true, + value: TextEncoderStream, + }, + TextDecoderStream: { + __proto__: null, + configurable: true, + enumerable: false, + writable: true, + value: TextDecoderStream, + }, + }); +} diff --git a/src/per_isolate/per_isolate-env.d.ts b/src/per_isolate/per_isolate-env.d.ts index a5d36137a1a..f269aae57aa 100644 --- a/src/per_isolate/per_isolate-env.d.ts +++ b/src/per_isolate/per_isolate-env.d.ts @@ -48,4 +48,35 @@ declare const autogates: { * * See primordials.ts for the full list of captures. */ -declare const primordials: Record; +declare const primordials: { + // Well-known symbols must be typed precisely so computed properties + // using them (e.g., [SymbolAsyncIterator]) satisfy interface constraints. + // NOTE: destructuring widens unique symbol → symbol. Files that need the + // precise type for indexing should use primordials.SymbolXxx directly. + readonly SymbolIterator: typeof Symbol.iterator; + readonly SymbolAsyncIterator: typeof Symbol.asyncIterator; + readonly SymbolToStringTag: typeof Symbol.toStringTag; + + // uncurryThis accepts Function (the typeof-narrowed type) in addition to + // properly typed callables, so callers don't need to cast after a + // `typeof fn === 'function'` guard. + readonly uncurryThis: any) | Function>( + fn: T + ) => T extends (...args: any[]) => any + ? (thisArg: ThisParameterType, ...args: Parameters) => ReturnType + : (...args: any[]) => any; + + // Everything else is loosely typed — add specific entries as needed. + readonly [key: string]: any; +}; + +declare const utils: { + isArrayBuffer(value: unknown): value is ArrayBuffer; + isArrayBufferView(value: unknown): value is ArrayBufferView; + isPromise(value: unknown): value is Promise; + isSharedArrayBuffer(value: unknown): value is SharedArrayBuffer; + isUint8Array(value: unknown): value is Uint8Array; + isAnyArrayBuffer(value: unknown): value is ArrayBuffer | SharedArrayBuffer; + markPromiseHandled(promise: Promise): void; + getApiSymbol(name: string): symbol; +}; diff --git a/src/per_isolate/primordials.ts b/src/per_isolate/primordials.ts index 1273cfdecd3..6d50b1c98e4 100644 --- a/src/per_isolate/primordials.ts +++ b/src/per_isolate/primordials.ts @@ -26,7 +26,9 @@ // Getters: {Type}Prototype{Prop}Get e.g. MapPrototypeSizeGet // --- Foundation: capture call/bind FIRST, before anything else --- -const { call, bind } = Function.prototype; +const ObjectPrototype = Object.prototype; +const FunctionPrototype = Function.prototype; +const { call, bind } = FunctionPrototype; // uncurryThis(Proto.method) returns (thisArg, ...args) => Proto.method.call(thisArg, ...args) // Safe even if Function.prototype.call is later polluted. @@ -73,7 +75,7 @@ const WeakSetCtor = WeakSet; // by per-isolate bootstrap scripts. // Function -const FunctionPrototypeBind = uncurryThis(Function.prototype.bind); +const FunctionPrototypeBind = uncurryThis(FunctionPrototype.bind); // Object const ObjectCreate = Object.create; @@ -191,6 +193,16 @@ ObjectDefineProperty(SafePromise, 'withResolvers', { configurable: false, }); +// Regular-Promise static methods (captured before user code runs). +// Use these for user-facing promises. SafePromise.resolve/reject/withResolvers +// remain available for internal-only promises that need species protection. +const PromiseResolve = FunctionPrototypeBind(PromiseCtor.resolve, PromiseCtor); +const PromiseReject = FunctionPrototypeBind(PromiseCtor.reject, PromiseCtor); +const PromiseWithResolvers = FunctionPrototypeBind( + PromiseCtor.withResolvers, + PromiseCtor +); + // Array const ArrayIsArray = Array.isArray; const ArrayFrom = Array.from; @@ -305,12 +317,16 @@ const SetIteratorPrototypeNext: (iter: any) => IteratorResult = ); // ArrayBuffer +const ArrayBufferPrototypeSlice = uncurryThis(ArrayBuffer.prototype.slice); const ArrayBufferPrototypeTransfer = uncurryThis( ArrayBuffer.prototype.transfer ); const ArrayBufferPrototypeByteLengthGet = getProtoGetter< (buffer: ArrayBuffer) => number >(ArrayBuffer.prototype, 'byteLength'); +const ArrayBufferPrototypeDetachedGet = getProtoGetter< + (buffer: ArrayBuffer) => boolean +>(ArrayBuffer.prototype, 'detached'); // TypedArray — instance methods live on %TypedArray%.prototype, the shared // prototype of all typed array types. Capturing via Uint8Array.prototype @@ -579,6 +595,68 @@ class SafeArrayIterator { } } +// Runtime built-ins that we also need to capture +function captureJsgGetter(proto: object, name: string): T { + const get = ObjectGetOwnPropertyDescriptor(proto, name)?.get; + if (get !== undefined) { + return uncurryThis(get) as T; + } + return ((instance: Record) => + instance[name]) as unknown as T; +} + +const AbortControllerCtor = globalThis.AbortController; +const AbortControllerAbort = uncurryThis( + AbortControllerCtor.prototype.abort +) as (controller: AbortController, reason?: unknown) => void; +const AbortControllerSignalGet = captureJsgGetter< + (controller: AbortController) => AbortSignal +>(AbortControllerCtor.prototype, 'signal'); + +const AbortSignalCtor = globalThis.AbortSignal; +const AbortSignalAbortedGet = captureJsgGetter< + (signal: AbortSignal) => boolean +>(AbortSignalCtor.prototype, 'aborted'); +const AbortSignalReasonGet = captureJsgGetter<(signal: AbortSignal) => unknown>( + AbortSignalCtor.prototype, + 'reason' +); + +const EventTargetCtor = globalThis.EventTarget; +const EventTargetAddEventListener = uncurryThis( + EventTargetCtor.prototype.addEventListener +) as (target: object, type: string, listener: () => void) => void; +const EventTargetRemoveEventListener = uncurryThis( + EventTargetCtor.prototype.removeEventListener +) as (target: object, type: string, listener: () => void) => void; + +const TextDecoderCtor = globalThis.TextDecoder; +const TextEncoderCtor = globalThis.TextEncoder; + +const textEncoderEncode = uncurryThis(TextEncoderCtor.prototype.encode) as ( + encoder: TextEncoder, + input: string +) => Uint8Array; + +const textDecoderDecode = uncurryThis(TextDecoderCtor.prototype.decode) as ( + decoder: TextDecoder, + input?: BufferSource, + options?: { stream?: boolean } +) => string; + +// TextDecoder readonly properties use the JSG layout trap: prototype +// accessors under modern compat, own data properties under old dates. +const textDecoderEncodingGet = captureJsgGetter< + (decoder: TextDecoder) => string +>(TextDecoderCtor.prototype, 'encoding'); +const textDecoderFatalGet = captureJsgGetter<(decoder: TextDecoder) => boolean>( + TextDecoderCtor.prototype, + 'fatal' +); +const textDecoderIgnoreBOMGet = captureJsgGetter< + (decoder: TextDecoder) => boolean +>(TextDecoderCtor.prototype, 'ignoreBOM'); + // Freeze the exports object to prevent accidental mutation by bootstrap scripts. // Uses the captured ObjectFreeze, not the potentially-polluted global. module.exports = ObjectFreeze({ @@ -587,6 +665,8 @@ module.exports = ObjectFreeze({ applyBind, // Global types + AbortController: AbortControllerCtor, + AbortSignal: AbortSignalCtor, AggregateError: AggregateErrorCtor, Array: ArrayCtor, ArrayBuffer: ArrayBufferCtor, @@ -602,6 +682,8 @@ module.exports = ObjectFreeze({ RegExp: RegExpCtor, Set: SetCtor, Symbol: SymbolCtor, + TextDecoder: TextDecoderCtor, + TextEncoder: TextEncoderCtor, TypeError: TypeErrorCtor, Uint8Array: Uint8ArrayCtor, WeakMap: WeakMapCtor, @@ -609,9 +691,11 @@ module.exports = ObjectFreeze({ WeakSet: WeakSetCtor, // Function + FunctionPrototype, FunctionPrototypeBind, // Object + ObjectPrototype, ObjectCreate, ObjectDefineProperty, ObjectDefineProperties, @@ -624,10 +708,15 @@ module.exports = ObjectFreeze({ // %AsyncIteratorPrototype% AsyncIteratorPrototype, - // Promise + // Promise — regular Promise (for user-facing promises) + PromiseResolve, + PromiseReject, + PromiseWithResolvers, + // Promise — captured prototype methods PromisePrototypeThen, PromisePrototypeCatch, PromisePrototypeFinally, + // Promise — SafePromise (for internal-only promises needing species protection) SafePromise, // Array @@ -693,8 +782,10 @@ module.exports = ObjectFreeze({ NumberIsNaN, // ArrayBuffer + ArrayBufferPrototypeSlice, ArrayBufferPrototypeTransfer, ArrayBufferPrototypeByteLengthGet, + ArrayBufferPrototypeDetachedGet, // TypedArray TypedArrayPrototypeSet, @@ -722,6 +813,24 @@ module.exports = ObjectFreeze({ SymbolIterator, SymbolToStringTag, + // EventTarget + EventTargetAddEventListener, + EventTargetRemoveEventListener, + + // AbortController + AbortControllerCtor, + AbortControllerAbort, + AbortControllerSignalGet, + AbortSignalAbortedGet, + AbortSignalReasonGet, + + // TextDecoder/TextEncoder + textDecoderEncodingGet, + textDecoderFatalGet, + textDecoderIgnoreBOMGet, + textEncoderEncode, + textDecoderDecode, + // Safe types — use normal method syntax, resistant to prototype pollution SafeMap, SafeSet, diff --git a/src/per_isolate/tsconfig.json b/src/per_isolate/tsconfig.json index 1c9c955c445..c275dd60543 100644 --- a/src/per_isolate/tsconfig.json +++ b/src/per_isolate/tsconfig.json @@ -6,6 +6,10 @@ "moduleResolution": "node", "verbatimModuleSyntax": false, "isolatedModules": false, + "baseUrl": ".", + "paths": { + "./*": ["./*"] + }, "types": [] }, "include": ["**/*.ts"] diff --git a/src/per_isolate/webstreams/encoding.ts b/src/per_isolate/webstreams/encoding.ts new file mode 100644 index 00000000000..cc9955c2da2 --- /dev/null +++ b/src/per_isolate/webstreams/encoding.ts @@ -0,0 +1,229 @@ +'use strict'; + +// TextEncoderStream and TextDecoderStream — WHATWG Encoding Standard +// stream wrappers, implemented purely in JS/TS over the captured +// TextEncoder/TextDecoder primitives. These are standard category-3 +// transforms (not elidable, not native-backed): the codec work +// dominates, and the readable side yields values (strings for TDS, +// Uint8Array for TES), not bytes — so they deliberately do NOT use +// the native-backed byte-stream path. +// +// The C++ stream wrappers these replace are legacy artifacts predating +// the per-isolate bootstrap; the codec primitives stay C++. + +import type { + ReadableStream as ReadableStreamType, + WritableStream as WritableStreamType, +} from './types'; + +const { + ObjectDefineProperties, + ObjectGetOwnPropertyDescriptor, + StringPrototypeCharCodeAt, + StringPrototypeSlice, + textDecoderEncodingGet, + TextDecoder, + TextEncoder, + textDecoderFatalGet, + textDecoderIgnoreBOMGet, + textEncoderEncode, + textDecoderDecode, + TypeError, + uncurryThis, +} = primordials; + +const { isArrayBuffer, isArrayBufferView } = utils; + +// Captured for primordials discipline — ToString coercion per spec. +const StringCoerce = String; + +const { TransformStream } = require('webstreams/transform'); + +// Capture TransformStream.prototype accessors at bootstrap time so +// internal reads do not go through the (user-patchable) prototype chain. +// Uses the same validation guard pattern as getProtoGetter in primordials.ts. +function getProtoGetter(proto: object, name: string): T { + const desc = ObjectGetOwnPropertyDescriptor(proto, name); + if (desc === undefined || desc.get === undefined) { + throw new TypeError(`Expected accessor property '${name}' on prototype`); + } + return uncurryThis(desc.get) as T; +} + +const transformStreamReadableGet = getProtoGetter< + (stream: object) => ReadableStreamType +>(TransformStream.prototype, 'readable'); +const transformStreamWritableGet = getProtoGetter< + (stream: object) => WritableStreamType +>(TransformStream.prototype, 'writable'); + +// --------------------------------------------------------------------------- +// TextEncoderStream +// +// Spec: https://encoding.spec.whatwg.org/#interface-textencoderstream +// +// Encodes strings to UTF-8. Stateful: a lone high surrogate at the end +// of a chunk is held and paired with the next chunk's leading low +// surrogate (spec §10.1.1 "encode and enqueue a chunk"). If the stream +// closes with a pending high surrogate, it's replaced with U+FFFD. + +class TextEncoderStream { + #transform: InstanceType; + #pendingHighSurrogate: string = ''; + + constructor() { + const self = this; + const encoder = new TextEncoder(); + this.#transform = new TransformStream({ + transform( + chunk: unknown, + controller: { enqueue: (c: Uint8Array) => void } + ) { + // Spec: "encode and enqueue a chunk" ToString-coerces the input. + const str = typeof chunk === 'string' ? chunk : StringCoerce(chunk); + const text = self.#pendingHighSurrogate + str; + const len = text.length; + if (len === 0) return; + + // Check if the last code unit is a lone high surrogate + // (0xD800–0xDBFF). If so, hold it for the next chunk. + const last = StringPrototypeCharCodeAt(text, len - 1) as number; + if (last >= 0xd800 && last <= 0xdbff) { + self.#pendingHighSurrogate = StringPrototypeSlice( + text, + len - 1 + ) as string; + const prefix = StringPrototypeSlice(text, 0, len - 1) as string; + if (prefix.length > 0) { + controller.enqueue(textEncoderEncode(encoder, prefix)); + } + } else { + self.#pendingHighSurrogate = ''; + controller.enqueue(textEncoderEncode(encoder, text)); + } + }, + flush(controller: { enqueue: (c: Uint8Array) => void }) { + // A pending high surrogate at end-of-stream is replaced with + // U+FFFD (the replacement character). + if (self.#pendingHighSurrogate !== '') { + controller.enqueue(textEncoderEncode(encoder, '\uFFFD')); + self.#pendingHighSurrogate = ''; + } + }, + }); + } + + get encoding(): string { + return 'utf-8'; + } + + get readable(): ReadableStreamType { + if (!(#transform in this)) throw new TypeError('Illegal invocation'); + return transformStreamReadableGet( + this.#transform + ) as ReadableStreamType; + } + + get writable(): WritableStreamType { + if (!(#transform in this)) throw new TypeError('Illegal invocation'); + return transformStreamWritableGet( + this.#transform + ) as WritableStreamType; + } +} + +// --------------------------------------------------------------------------- +// TextDecoderStream +// +// Spec: https://encoding.spec.whatwg.org/#interface-textdecoderstream +// +// Decodes bytes to strings. Wraps a stateful TextDecoder with +// {stream: true} in transform and a final decode() in flush to catch +// incomplete multi-byte sequences. + +interface TextDecoderStreamOptions { + fatal?: boolean; + ignoreBOM?: boolean; +} + +class TextDecoderStream { + #transform: InstanceType; + #decoder: TextDecoder; + + constructor(label: string = 'utf-8', options?: TextDecoderStreamOptions) { + const decoder = new TextDecoder(label, options); + this.#decoder = decoder; + + this.#transform = new TransformStream({ + transform(chunk: unknown, controller: { enqueue: (c: string) => void }) { + if (!isArrayBufferView(chunk) && !isArrayBuffer(chunk)) { + throw new TypeError( + 'TextDecoderStream: chunk must be a BufferSource' + ); + } + const decoded = textDecoderDecode(decoder, chunk as BufferSource, { + stream: true, + }); + if (decoded.length > 0) { + controller.enqueue(decoded); + } + }, + flush(controller: { enqueue: (c: string) => void }) { + // Final decode: flushes any incomplete multi-byte sequences. + // In fatal mode this throws if the sequence is incomplete. + const decoded = textDecoderDecode(decoder); + if (decoded.length > 0) { + controller.enqueue(decoded); + } + }, + }); + } + + get encoding(): string { + if (!(#decoder in this)) throw new TypeError('Illegal invocation'); + return textDecoderEncodingGet(this.#decoder); + } + + get fatal(): boolean { + if (!(#decoder in this)) throw new TypeError('Illegal invocation'); + return textDecoderFatalGet(this.#decoder); + } + + get ignoreBOM(): boolean { + if (!(#decoder in this)) throw new TypeError('Illegal invocation'); + return textDecoderIgnoreBOMGet(this.#decoder); + } + + get readable(): ReadableStreamType { + if (!(#transform in this)) throw new TypeError('Illegal invocation'); + return transformStreamReadableGet( + this.#transform + ) as ReadableStreamType; + } + + get writable(): WritableStreamType { + if (!(#transform in this)) throw new TypeError('Illegal invocation'); + return transformStreamWritableGet( + this.#transform + ) as WritableStreamType; + } +} + +ObjectDefineProperties(TextEncoderStream.prototype, { + encoding: { enumerable: true }, + readable: { enumerable: true }, + writable: { enumerable: true }, +}); + +ObjectDefineProperties(TextDecoderStream.prototype, { + encoding: { enumerable: true }, + fatal: { enumerable: true }, + ignoreBOM: { enumerable: true }, + readable: { enumerable: true }, + writable: { enumerable: true }, +}); + +module.exports = { + TextEncoderStream, + TextDecoderStream, +}; diff --git a/src/per_isolate/webstreams/identity.ts b/src/per_isolate/webstreams/identity.ts new file mode 100644 index 00000000000..3fe93c1aa3e --- /dev/null +++ b/src/per_isolate/webstreams/identity.ts @@ -0,0 +1,315 @@ +'use strict'; + +// IdentityTransformStream and FixedLengthStream — non-standard +// byte-capable identity transforms. +// +// IdentityTransformStream is semantically equivalent to +// `new TransformStream()` (no-op, elided) except: +// - The readable side is a BYTE STREAM (BYOB, min/atLeast, draining, +// extraction — all inherited from ReadableByteStreamController). +// - The writable side only accepts BYTES (ArrayBuffer, ArrayBufferView, +// SharedArrayBuffer) and STRINGS (→ UTF-8 via TextEncoder). +// Everything else: TypeError. +// - All writes COPY the data (never transfer/detach the input buffer). +// SAB input forces this anyway; uniform copy avoids a behavioral +// split and matches legacy parity with the old C++ ITS. +// - Zero-length writes are accepted as NO-OPS: the write resolves +// immediately without touching the readable queue (no zero-length +// chunk enqueued, no backpressure interaction, no pull). +// +// FixedLengthStream extends IdentityTransformStream with an +// `expectedLength` that flows through to the readable byte controller, +// giving `new Response(fixedLengthStream.readable)` its Content-Length +// header via the existing expectedLength machinery. +// +// Not a subclass of the standard TransformStream: the byte-stream +// readable, the write-side validation, and the copy-not-transfer +// semantics make it a distinct class with its own wiring. + +import type { + PromiseWithResolvers as PromiseWithResolversType, + QueuingStrategy, + ReadableStream as ReadableStreamType, + WritableStream as WritableStreamType, +} from './types'; + +const { + ArrayBuffer, + DataViewPrototypeGetBuffer, + DataViewPrototypeGetByteLength, + DataViewPrototypeGetByteOffset, + ObjectDefineProperties, + ObjectGetOwnPropertyDescriptor, + PromiseWithResolvers, + Symbol, + TextEncoder, + textEncoderEncode, + TypeError, + TypedArrayPrototypeGetBuffer, + TypedArrayPrototypeGetByteLength, + TypedArrayPrototypeGetByteOffset, + TypedArrayPrototypeGetSymbolToStringTag, + TypedArrayPrototypeSet, + Uint8Array, + uncurryThis, +} = primordials; + +const { + isArrayBufferView, + isArrayBuffer, + isSharedArrayBuffer, + markPromiseHandled, +} = utils; + +const { + ReadableStream, + ReadableByteStreamController, +} = require('webstreams/readable'); +const { + WritableStream, + WritableStreamDefaultController, + internalsForPipe: writableInternals, +} = require('webstreams/writable'); + +const writableControllerError = uncurryThis( + WritableStreamDefaultController.prototype.error +) as (controller: object, reason: unknown) => void; + +// --- Bootstrap captures (byte controller methods + TextEncoder) ---------- + +const byteControllerEnqueue = uncurryThis( + ReadableByteStreamController.prototype.enqueue +) as (controller: object, chunk: ArrayBufferView) => void; +const byteControllerClose = uncurryThis( + ReadableByteStreamController.prototype.close +) as (controller: object) => void; +const byteControllerError = uncurryThis( + ReadableByteStreamController.prototype.error +) as (controller: object, reason: unknown) => void; +const byteControllerDesiredSizeGet = (() => { + const desc = ObjectGetOwnPropertyDescriptor( + ReadableByteStreamController.prototype, + 'desiredSize' + ); + if (desc === undefined || desc.get === undefined) { + throw new TypeError( + "Expected accessor property 'desiredSize' on prototype" + ); + } + return uncurryThis(desc.get); +})() as (controller: object) => number | null; + +// TextEncoder instance for string → UTF-8 conversion. +const textEncoderInstance = new TextEncoder(); + +// --------------------------------------------------------------------------- +// Chunk validation and copy +// +// Returns a COPIED Uint8Array for byte inputs, a TextEncoder result for +// strings, or undefined for zero-length inputs (write no-op). Throws +// TypeError for anything else. Never detaches the input buffer. + +function validateAndCopyChunk(chunk: unknown): Uint8Array | undefined { + if (typeof chunk === 'string') { + if (chunk.length === 0) return undefined; + return textEncoderEncode(textEncoderInstance, chunk); + } + if (isArrayBuffer(chunk) || isSharedArrayBuffer(chunk)) { + // Wrap in Uint8Array for a uniform code path — Uint8Array accepts + // both ArrayBuffer and SharedArrayBuffer, and its byteLength getter + // works on both (unlike ArrayBuffer.prototype.byteLength, which + // throws on SAB receivers). + const src = new Uint8Array(chunk as ArrayBuffer); + const byteLength = TypedArrayPrototypeGetByteLength(src) as number; + if (byteLength === 0) return undefined; + const copy = new Uint8Array(new ArrayBuffer(byteLength)); + TypedArrayPrototypeSet(copy, src); + return copy; + } + if (isArrayBufferView(chunk)) { + const isDataView = + TypedArrayPrototypeGetSymbolToStringTag(chunk) === undefined; + const byteOffset = ( + isDataView + ? DataViewPrototypeGetByteOffset(chunk) + : TypedArrayPrototypeGetByteOffset(chunk) + ) as number; + const byteLength = ( + isDataView + ? DataViewPrototypeGetByteLength(chunk) + : TypedArrayPrototypeGetByteLength(chunk) + ) as number; + if (byteLength === 0) return undefined; + const buffer = ( + isDataView + ? DataViewPrototypeGetBuffer(chunk) + : TypedArrayPrototypeGetBuffer(chunk) + ) as ArrayBuffer; + const copy = new Uint8Array(new ArrayBuffer(byteLength)); + TypedArrayPrototypeSet( + copy, + new Uint8Array(buffer, byteOffset, byteLength) + ); + return copy; + } + throw new TypeError( + 'IdentityTransformStream: chunk must be a BufferSource or string' + ); +} + +// --------------------------------------------------------------------------- + +const kPrivateSymbol: symbol = Symbol('private'); + +class IdentityTransformStream { + #readable: ReadableStreamType; + #writable: WritableStreamType; + #readableController: object | undefined; + #writableController: object | undefined; + #backpressure: boolean = true; + #backpressureChange: PromiseWithResolversType; + + #setBackpressure(backpressure: boolean): void { + this.#backpressureChange.resolve(); + const replacement = + PromiseWithResolvers() as PromiseWithResolversType; + markPromiseHandled(replacement.promise); + this.#backpressureChange = replacement; + this.#backpressure = backpressure; + } + + #errorWritableAndUnblockWrite(reason: unknown): void { + const wc = this.#writableController; + if (wc !== undefined) { + writableControllerError(wc, reason); + } + if (this.#backpressure) { + this.#setBackpressure(false); + } + } + + constructor(writableStrategy?: QueuingStrategy); + // Internal: called by FixedLengthStream with (kPrivateSymbol, length). + constructor(internal: symbol, expectedLength: bigint | number); + constructor( + writableStrategyOrInternal?: QueuingStrategy | symbol, + internalExpectedLength?: bigint | number + ) { + // External: new IdentityTransformStream() or + // new IdentityTransformStream(writableStrategy). + // Internal (from FixedLengthStream): new ITS(kPrivateSymbol, length). + let writableStrategy: QueuingStrategy | undefined; + let expectedLength: bigint | number | undefined; + if (writableStrategyOrInternal === kPrivateSymbol) { + expectedLength = internalExpectedLength; + } else { + writableStrategy = writableStrategyOrInternal as + | QueuingStrategy + | undefined; + } + writableStrategy ??= {} as QueuingStrategy; + + const initialBackpressureChange = + PromiseWithResolvers() as PromiseWithResolversType; + markPromiseHandled(initialBackpressureChange.promise); + this.#backpressureChange = initialBackpressureChange; + + // --- Writable side (byte-only ingress) --- + const sinkWrite = async (chunk: unknown): Promise => { + const copied = validateAndCopyChunk(chunk); + if (copied === undefined) return; // zero-length no-op + while (this.#backpressure) { + await this.#backpressureChange.promise; + const state = writableInternals.getState(this.#writable); + if (state === 'erroring' || state === 'errored') { + throw writableInternals.getStoredError(this.#writable); + } + } + const rc = this.#readableController as object; + byteControllerEnqueue(rc, copied); + const desiredSize = byteControllerDesiredSizeGet(rc); + const backpressure = desiredSize !== null && desiredSize <= 0; + if (backpressure !== this.#backpressure) { + this.#setBackpressure(backpressure); + } + }; + const sinkClose = (): void => { + const rc = this.#readableController; + if (rc !== undefined) byteControllerClose(rc); + }; + const sinkAbort = (reason: unknown): void => { + const rc = this.#readableController; + if (rc !== undefined) byteControllerError(rc, reason); + }; + + this.#writable = new WritableStream( + { + start: (c: object) => { + this.#writableController = c; + }, + write: sinkWrite, + close: sinkClose, + abort: sinkAbort, + }, + writableStrategy + ); + + // --- Readable side (byte stream, BYOB capable) --- + const sourcePull = (): Promise => { + this.#setBackpressure(false); + return this.#backpressureChange.promise; + }; + const sourceCancel = (reason: unknown): void => { + this.#errorWritableAndUnblockWrite(reason); + }; + + const byteSource: Record = { + type: 'bytes', + start: (c: object) => { + this.#readableController = c; + }, + pull: sourcePull, + cancel: sourceCancel, + }; + if (expectedLength !== undefined) { + byteSource.expectedLength = expectedLength; + } + + this.#readable = new ReadableStream(byteSource, { + highWaterMark: 0, + }); + } + + get readable(): ReadableStreamType { + if (!(#readable in this)) throw new TypeError('Illegal invocation'); + return this.#readable; + } + + get writable(): WritableStreamType { + if (!(#writable in this)) throw new TypeError('Illegal invocation'); + return this.#writable; + } +} + +class FixedLengthStream extends IdentityTransformStream { + constructor( + expectedLength: bigint | number, + _writableStrategy?: QueuingStrategy + ) { + // Validation of expectedLength is handled by the + // ReadableByteStreamController (normalizeExpectedLength) at the + // readable-side construction. writableStrategy is accepted for + // signature parity but currently unused (ITS defaults apply). + super(kPrivateSymbol, expectedLength); + } +} + +ObjectDefineProperties(IdentityTransformStream.prototype, { + readable: { enumerable: true }, + writable: { enumerable: true }, +}); + +module.exports = { + IdentityTransformStream, + FixedLengthStream, +}; diff --git a/src/per_isolate/webstreams/native.ts b/src/per_isolate/webstreams/native.ts new file mode 100644 index 00000000000..0a77b780772 --- /dev/null +++ b/src/per_isolate/webstreams/native.ts @@ -0,0 +1,1413 @@ +'use strict'; + +// The NATIVE consumer backend: streams whose data is produced by a C++ +// (JSG/KJ) source rather than a JS underlying source feeding the shared +// queue. See native-stream-integration.md §10 for the fence conventions +// separating this backend from the queued one (queue.ts). +// +// THE C++/JS CONTRACT (phase 3; currently exercised via JS mocks — the +// real C++ source arrives in a later integration step and must conform): +// - A native underlying source is an ordinary object carrying the +// kNativeSource marker as an OWN property whose value is the marker +// symbol itself. Marker presence ⇒ native ⇒ BYTE CAPABLE (there is +// no separate byte flag). +// - The source implements the standard UnderlyingSource shape: +// pull(controller) and cancel(reason) as regular methods ARE the +// hooks, driven by the standard pull algorithm (pulling/pullAgain +// serialization). `start` is IGNORED (assumed no-op). `type` and +// `autoAllocateChunkSize` are forbidden. Additional hooks (tee) are +// additional regular methods — safe because the native source object +// is never exposed to user code. +// - pull(controller) discriminates the read mode via the controller: +// controller.byobRequest !== null ⇒ BYOB read — fill the request's +// view and call respond(bytesWritten)/respondWithNewView(view), +// honoring `atLeast` (the read's minimum, in bytes). +// byobRequest === null ⇒ default read — the source allocates its OWN +// buffer and calls controller.enqueue(view). +// - The source calls enqueue-or-respond AT MOST ONCE per pull +// invocation and never pushes outside pull. desiredSize is exposed +// but the native source never consults it. +// - PER-PULL CANCELLATION: pull receives an extension argument +// pull(controller, signal) — an AbortSignal that fires when the +// consumer abandons the read (releaseLock, cancel, tee). The source +// MUST check signal.aborted synchronously immediately before +// enqueue/respond; if aborted, it retains the bytes and redelivers +// on the next pull. The controller REFUSES delivery on an aborted +// signal as a backstop. This eliminates JS-side buffering for the +// release-mid-pull race: race buffering lives SOURCE-SIDE; the JS +// conduit is uniformly bufferless. +// FUTURE OPTIMIZATION: reuse a single AbortController per conduit +// (reset between pulls) rather than allocating one per pull. +// - MIN-READ / UNDER-DELIVERY CONTRACT (KJ tryRead semantics): the +// respond is the source's COMPLETE answer for the read. Any internal +// accumulation toward `atLeast` happens inside the native source; +// the conduit never re-pulls an unsatisfied read. Responding with +// fewer bytes than `atLeast` IMPLICITLY SIGNALS CLOSURE: the partial +// fill commits fused as { done: true, value: partialView }, the +// stream closes, and every subsequent read returns EOF. This result +// is deliberately INDISTINGUISHABLE from the standard's one fused +// corner — a pending min read committed in the closed state via +// close() + respond(0) (RespondInClosedState → +// CommitPullIntoDescriptor, which fulfills the read-into request +// with the partial view and done: true). The signaling MECHANISM +// differs (a queued JS source is pulled again and must close() +// explicitly; for a native source the under-delivery itself is the +// signal), but consumers observe the same result shapes on both +// backends. +// - Close is otherwise a fused close-commit: controller.close() may +// follow a respond() in the same pull turn — deliberately divergent +// from the queued backend's drain-then-sentinel model. +// - tee(): returns a PAIR of new native source objects, leaving the +// original source closed. The stream layer wraps each branch in a +// fresh ReadableStream; branches are fully independent (no composite +// cancel). +// - expectedLength (non-standard extension, optional): the TOTAL bytes +// the source promises to produce — a non-negative bigint or integer +// number (normalized to bigint), read once at construction. +// undefined = unknown (C++ uses chunked encoding). The conduit +// enforces the exact-total contract: overflow at delivery and +// underflow at source-initiated close (including min-read +// under-delivery landing short) are RangeError violations. +// Consumer-initiated cancel/tee/extraction are exempt. C++ reads the +// property directly off the extracted source object. +// +// NATIVE-BACKEND INVARIANTS (binding on changes to this file; the queued +// backend in queue.ts has a DIFFERENT set — do not port logic across the +// fence without checking both): +// - There is NO JS-side queue AND NO JS-SIDE OVERFLOW BUFFER: every +// read becomes demand on the unified request FIFO, satisfied +// directly by the source inside pull. Race buffering (the +// release-mid-pull case) lives SOURCE-SIDE via the abort signal +// contract — the source stashes and redelivers; the conduit never +// holds data. Nothing here may assume entries, cursors, positions, +// or the CLOSE_SENTINEL exist. +// - JSG owns the source lifetime: no WeakRef owner tracking and no +// FinalizationRegistry backstop (the queued backend needs both). +// - The controller façade is consumed ONLY by the native source; it is +// never handed to user code. +// - Stream-level state transitions cross the module fence through the +// injected NativeStreamHooks (readableStreamClose/Error on the far +// side) — never by touching stream internals from here. +// - A COMPLETION LATCH prevents busy-loop re-pulls: a pull that +// settles without delivering, closing, or erroring while requests +// are still pending errors the stream. Aborted pulls are exempt. +// +// Nothing in this module is ever exposed to user code, with one TEMPORARY +// exception: streams.ts re-exports kNativeSource so mock native sources +// can be constructed from tests before the real C++ integration exists. + +import type { + ByteStreamConsumer as ByteStreamConsumerType, + PendingRead, + PullIntoDescriptor, +} from './queue'; +import type { + PromiseWithResolvers as PromiseWithResolversType, + ReadableStreamReadResult, +} from './types'; + +const { + AbortController, + AbortControllerAbort, + AbortControllerSignalGet, + AbortSignalAbortedGet, + ArrayBufferPrototypeByteLengthGet, + ArrayPrototypePush, + ArrayPrototypeShift, + ArrayPrototypeSplice, + BigInt, + DataViewPrototypeGetBuffer, + DataViewPrototypeGetByteLength, + DataViewPrototypeGetByteOffset, + NumberIsNaN, + ObjectGetOwnPropertyDescriptor, + PromiseResolve, + PromiseReject, + PromiseWithResolvers, + PromisePrototypeThen, + RangeError, + TypeError, + TypedArrayPrototypeGetBuffer, + TypedArrayPrototypeGetByteLength, + TypedArrayPrototypeGetByteOffset, + TypedArrayPrototypeGetSymbolToStringTag, + Uint8Array, + uncurryThis, +} = primordials; + +const { isArrayBufferView, isUint8Array, markPromiseHandled } = utils; + +// --------------------------------------------------------------------------- +// The construction-time handshake + +// A native underlying source carries this symbol as an own property whose +// value is the symbol itself. The value is currently meaningless beyond +// the identity check (reserved for future capability data). +const kNativeSource: symbol = utils.getApiSymbol('kNativeSource'); + +// --- Writable-side markers (same conventions as the readable side) --- + +// A native underlying sink carries this symbol as an own property whose +// value is the symbol itself. Detection uses own-property descriptor +// read (inherited/accessor markers ignored). Unlike the source side, +// native sinks need no new backend machinery — the standard +// WritableStream drives them via start/write/close/abort. The marker +// exists for pipe dispatch (is the dest native?) and extraction (hand +// the sink to C++ for the native+native fast path). The one extension: +// pipeFrom(source, options), the hook for the native+native pipe. +const kNativeSink: symbol = utils.getApiSymbol('kNativeSink'); + +// Extraction marker for native-backed WritableStream instances. Mirrors +// kExtractNativeSource: own, non-enumerable property, shared extractor +// function, one-shot via locked precondition. Writable has no +// "disturbed" concept — locked is the only precondition. +const kExtractNativeSink: symbol = utils.getApiSymbol('kExtractNativeSink'); + +function isNativeUnderlyingSink(sink: object): boolean { + const desc = ObjectGetOwnPropertyDescriptor(sink, kNativeSink) as + | PropertyDescriptor + | undefined; + const value = desc?.value as unknown; + if (value === undefined) return false; + if (value !== kNativeSink) { + throw new TypeError('Invalid native stream sink marker'); + } + return true; +} + +// The JS-to-C++ extraction marker. Installed on native-backed +// ReadableStream instances as an own, non-enumerable, non-writable, +// non-configurable property whose value is a shared extractor function. +// The C++ TypeWrapper detects this property's presence to classify the +// stream: present -> extract the native source for a pure C++ data +// path; absent -> acquire a DrainingReader instead. The marker is KEPT +// after extraction (presence means "native-backed", not "extractable"); +// the extractor's locked/disturbed preconditions make it one-shot. +// Bootstrap phase: regular symbol (temporarily exposed via streams.ts). +// Final implementation: runtime-provided private API symbol. +const kExtractNativeSource: symbol = utils.getApiSymbol('kExtractNativeSource'); + +function isActualObject(value: unknown): value is object { + return value != null && typeof value === 'object'; +} + +// Returns true if `source` is native-marked; false for ordinary (queued) +// sources. Throws on a malformed marker — by the time the marker key is +// present at all, this is a contract violation on the native/mock side, +// never a user-input condition. +// +// Hardening: OWN-property read via descriptor — a (temporarily exposed) +// symbol planted on Object.prototype must not convert every plain source +// into a native one, and a hostile accessor at the marker is never +// invoked (we read desc.value; accessor-defined markers are ignored). +function isNativeUnderlyingSource(source: object): boolean { + const desc = ObjectGetOwnPropertyDescriptor(source, kNativeSource) as + | PropertyDescriptor + | undefined; + const value = desc?.value as unknown; + if (value === undefined) return false; + if (value !== kNativeSource) { + throw new TypeError('Invalid native stream source marker'); + } + return true; +} + +// Stream-level transitions injected by readable.ts at construction (the +// stream's private state lives across the module fence). Both are +// state-guarded on the far side, so calling them redundantly is safe. +export interface NativeStreamHooks { + closeStream: () => void; + errorStream: (reason: unknown) => void; +} + +// Normalizes the non-standard `expectedLength` extension property: the +// TOTAL number of bytes the source promises to produce over its +// lifetime (undefined = unknown; C++ then uses chunked encoding). +// Accepts a non-negative bigint or a non-negative integer number +// (normalized to bigint — totals can exceed MAX_SAFE_INTEGER). +// Shape violations are TypeErrors; range violations are RangeErrors. +// Duplicated for the queued byte controller in readable.ts. +function normalizeExpectedLength(value: unknown): bigint | undefined { + if (value === undefined) return undefined; + if (typeof value === 'bigint') { + if (value < 0n) { + throw new RangeError('expectedLength must be non-negative'); + } + return value; + } + if (typeof value === 'number') { + if (NumberIsNaN(value) || value % 1 !== 0) { + throw new TypeError('expectedLength must be an integer'); + } + if (value < 0) { + throw new RangeError('expectedLength must be non-negative'); + } + return BigInt(value); + } + throw new TypeError( + 'expectedLength must be a non-negative bigint or integer' + ); +} + +// --------------------------------------------------------------------------- +// The unified request FIFO +// +// Default reads and BYOB reads share ONE queue so results resolve in +// submission order. The head request's kind is what the controller façade +// reflects to the source: byobRequest !== null ⇔ the head is a BYOB read. + +interface NativeDefaultRequest { + kind: 'default'; + read: PendingRead; +} + +interface NativeByobRequestEntry { + kind: 'byob'; + desc: PullIntoDescriptor; +} + +type NativeRequest = NativeDefaultRequest | NativeByobRequestEntry; + +// Cross-class accessors (assigned in static blocks). +let getRequestDesc: ( + request: NativeReadableStreamBYOBRequest +) => PullIntoDescriptor; +let isNativeController: ( + value: unknown +) => value is NativeReadableStreamController; +let getControllerConduit: ( + controller: NativeReadableStreamController +) => NativePullConduit; + +// --------------------------------------------------------------------------- +// The BYOB request façade +// +// Wraps the head pull-into descriptor for the native source's consumption +// during pull. Mirrors the shape of the global ReadableStreamBYOBRequest +// (view/atLeast/respond/respondWithNewView) but is a distinct, module- +// private class: it is only ever handed to the native source, never to +// user code, so it needs no global registration or brand hardening beyond +// its private fields. Invalidated automatically once its descriptor is no +// longer the head request (view/atLeast report null; responds throw). + +class NativeReadableStreamBYOBRequest { + #conduit: NativePullConduit; + #desc: PullIntoDescriptor; + + static { + getRequestDesc = (request) => request.#desc; + } + + constructor(conduit: NativePullConduit, desc: PullIntoDescriptor) { + this.#conduit = conduit; + this.#desc = desc; + } + + get view(): Uint8Array | null { + if (!(#desc in this)) throw new TypeError('Illegal invocation'); + if (!this.#conduit.isHeadDesc(this.#desc)) return null; + const desc = this.#desc; + return new Uint8Array( + desc.buffer, + desc.byteOffset + desc.bytesFilled, + desc.byteLength - desc.bytesFilled + ); + } + + // Non-standard workerd extension (mirrors the global BYOBRequest): the + // minimum number of BYTES the read requires. Delivering fewer is the + // under-delivery EOF signal (see the min-read contract in the module + // header). The subtraction is defensive: a live descriptor always has + // bytesFilled === 0 under the once-per-read contract. + get atLeast(): number | null { + if (!(#desc in this)) throw new TypeError('Illegal invocation'); + if (!this.#conduit.isHeadDesc(this.#desc)) return null; + const desc = this.#desc; + const remaining = desc.minimumFill - desc.bytesFilled; + return remaining > 0 ? remaining : 0; + } + + respond(bytesWritten: number): void { + if (!(#desc in this)) throw new TypeError('Illegal invocation'); + this.#conduit.respondToDesc(this.#desc, bytesWritten); + } + + respondWithNewView(view: ArrayBufferView): void { + if (!(#desc in this)) throw new TypeError('Illegal invocation'); + this.#conduit.respondToDescWithNewView(this.#desc, view); + } +} + +// --------------------------------------------------------------------------- +// The controller façade +// +// Occupies the stream's #controller slot and is passed to the native +// source's pull(controller) per the standard pull algorithm. Consumed +// ONLY by the native source — never handed to user code — but the methods +// brand-check anyway, matching the file-wide convention. + +class NativeReadableStreamController { + #conduit: NativePullConduit; + + static { + isNativeController = ( + value: unknown + ): value is NativeReadableStreamController => { + return isActualObject(value) && #conduit in value; + }; + + getControllerConduit = (controller) => controller.#conduit; + } + + constructor(conduit: NativePullConduit) { + this.#conduit = conduit; + } + + // Mode discrimination for the native source: non-null ⇔ the request at + // the head of the FIFO is a BYOB read. + get byobRequest(): NativeReadableStreamBYOBRequest | null { + if (!(#conduit in this)) throw new TypeError('Illegal invocation'); + return this.#conduit.getByobRequest(); + } + + // Exposed for shape parity with the standard controller. The native + // source contractually never consults it: pacing is purely demand- + // driven (effective high-water mark of zero; strategy is ignored). + get desiredSize(): number | null { + if (!(#conduit in this)) throw new TypeError('Illegal invocation'); + return this.#conduit.desiredSize; + } + + enqueue(chunk: ArrayBufferView): void { + if (!(#conduit in this)) throw new TypeError('Illegal invocation'); + this.#conduit.enqueue(chunk); + } + + close(): void { + if (!(#conduit in this)) throw new TypeError('Illegal invocation'); + this.#conduit.closeFromSource(); + } + + error(reason?: unknown): void { + if (!(#conduit in this)) throw new TypeError('Illegal invocation'); + this.#conduit.errorFromSource(reason); + } +} + +// --------------------------------------------------------------------------- +// The native pull conduit (the consumer-fence implementation) + +type PullAlgorithm = ( + controller: NativeReadableStreamController, + signal: AbortSignal +) => Promise; +type CancelAlgorithm = (reason: unknown) => Promise; + +interface NativeAlgorithms { + pullAlgorithm: PullAlgorithm | undefined; + cancelAlgorithm: CancelAlgorithm | undefined; + teeAlgorithm: (() => unknown) | undefined; + // Normalized expectedLength (undefined = unknown). The conduit + // enforces the exact-total contract. + expectedLength: bigint | undefined; +} + +class NativePullConduit implements ByteStreamConsumerType { + #requests: NativeRequest[] = []; + // 'closed' covers source-close, stream cancel, and tee (the contract + // leaves a teed-away original source closed). + #status: 'active' | 'closed' | 'errored' = 'active'; + #storedError: unknown; + #pulling: boolean = false; + #pullAgain: boolean = false; + // Set by a successful enqueue/respond, cleared at pull start. Used for + // the once-per-pull delivery check AND the completion latch: a pull + // that settles without delivering, closing, or erroring while requests + // are still pending is a broken source — the latch errors the stream + // rather than busy-looping the re-pull. Aborted pulls are exempt. + #deliveredThisPull: boolean = false; + // The AbortController for the currently in-flight pull, or undefined + // when no pull is active. Used as the identity key for superseded- + // settle detection: the settle handler captures its own controller + // reference and compares it to this field — a mismatch means the + // pull was superseded (tee/extract). Aborted pulls keep their + // controller reference so that #pulling serialization is maintained + // until the aborted promise settles (signal.aborted distinguishes + // aborted from live settlements). + #pullAbortController: AbortController | undefined; + #pullAlgorithm: PullAlgorithm | undefined; + #cancelAlgorithm: CancelAlgorithm | undefined; + #teeAlgorithm: (() => unknown) | undefined; + #hooks: NativeStreamHooks; + // Set immediately after construction by createNativeReadableStreamParts. + #controller: NativeReadableStreamController | undefined; + #byobRequestCache: NativeReadableStreamBYOBRequest | null = null; + // Back-reference to the original source object, kept for extraction + // (the JS-to-C++ path returns this object so C++ can unwrap its + // backing class). Set once at construction and never cleared: after + // detachForExtraction the conduit is terminal (#status 'closed'), so + // retaining the reference is harmless and keeps the field non-optional. + #source: object; + // The exact-total byte contract (undefined = unknown/chunked). The + // source must deliver exactly this many bytes over its lifetime: + // overflow at delivery and underflow at source-initiated close are + // RangeError violations. Consumer-initiated cancel/tee/extraction are + // exempt (the source didn't break its promise; the consumer left). + #expectedLength: bigint | undefined; + #bytesDelivered: bigint = 0n; + + constructor( + source: object, + algorithms: NativeAlgorithms, + hooks: NativeStreamHooks + ) { + this.#source = source; + this.#expectedLength = algorithms.expectedLength; + this.#pullAlgorithm = algorithms.pullAlgorithm; + this.#cancelAlgorithm = algorithms.cancelAlgorithm; + this.#teeAlgorithm = algorithms.teeAlgorithm; + this.#hooks = hooks; + } + + attachController(controller: NativeReadableStreamController): void { + this.#controller = controller; + } + + // --- StreamConsumer surface ---------------------------------------------- + + // Native sources are always pull-based (async); synchronous reads are + // never available. + tryReadSync( + _reader: object + ): ReadableStreamReadResult | undefined { + return undefined; + } + + read(reader: object): Promise> { + if (this.#status === 'errored') { + return PromiseReject(this.#storedError) as Promise< + ReadableStreamReadResult + >; + } + if (this.#status === 'closed') { + return PromiseResolve({ done: true, value: undefined }) as Promise< + ReadableStreamReadResult + >; + } + const withResolvers = PromiseWithResolvers() as PromiseWithResolversType< + ReadableStreamReadResult + >; + ArrayPrototypePush(this.#requests, { + kind: 'default', + read: { + resolve: withResolvers.resolve, + reject: withResolvers.reject, + reader, + }, + } satisfies NativeDefaultRequest); + // The pull prompt arrives via controllerPullIfNeeded from the reader + // layer (spec PullSteps ordering), not here. + return withResolvers.promise; + } + + drain(_maxSize?: number): { chunks: Uint8Array[]; done: boolean } { + // Nothing is ever buffered on the JS side; the draining reader falls + // back to read() for the wait-for-data case. + return { chunks: [], done: this.#status === 'closed' }; + } + + cancelReadsForReader(reader: object, reason: unknown): void { + const kept: NativeRequest[] = []; + const requests = this.#requests; + for (let i = 0; i < requests.length; i++) { + const request = requests[i] as NativeRequest; + const owner = + request.kind === 'default' ? request.read.reader : request.desc.reader; + if (owner === reader) { + if (request.kind === 'default') { + request.read.reject(reason); + } else { + request.desc.reject(reason); + } + } else { + ArrayPrototypePush(kept, request); + } + } + this.#requests = kept; + this.#byobRequestCache = null; + // ABORT SITE: if the released reader's reads were the only demand and + // a pull is in flight, abort it — the source contractually holds any + // bytes and redelivers on the next pull. + if ( + this.#requests.length === 0 && + this.#pullAbortController !== undefined + ) { + this.#abortCurrentPull(reason); + } + } + + errorAllReads(reason: unknown): void { + const requests = this.#requests; + this.#requests = []; + this.#byobRequestCache = null; + for (let i = 0; i < requests.length; i++) { + const request = requests[i] as NativeRequest; + if (request.kind === 'default') { + request.read.reject(reason); + } else { + request.desc.reject(reason); + } + } + } + + resolveAllReadsAsDone(): void { + const requests = this.#requests; + this.#requests = []; + this.#byobRequestCache = null; + for (let i = 0; i < requests.length; i++) { + const request = requests[i] as NativeRequest; + if (request.kind === 'default') { + request.read.resolve({ done: true, value: undefined }); + } else { + request.desc.resolve({ done: true, value: undefined }); + } + } + } + + get hasPendingRead(): boolean { + return this.#requests.length > 0; + } + + fulfillFirstPendingRead(value: Uint8Array): void { + const pending = ArrayPrototypeShift(this.#requests) as NativeDefaultRequest; + pending.read.resolve({ value, done: false }); + } + + cancelStream( + reason: unknown, + decideSourceCancel: (isLastConsumer: boolean) => Promise + ): Promise { + // Sole consumer by construction (native tee produces independent + // streams, never shared consumers), so this conduit is always the + // last one. Settle outstanding reads as done BEFORE the source-cancel + // decision, per the StreamConsumer contract ordering. + this.resolveAllReadsAsDone(); + // ABORT SITE: abort-then-cancel ordering. The cancel reason flows + // through to signal.reason so the source can distinguish cancel from + // release if it cares. + this.#abortCurrentPull(reason); + if (this.#status === 'active') this.#status = 'closed'; + return decideSourceCancel(true); + } + + // --- ByteStreamConsumer surface ------------------------------------------ + + readBYOB( + desc: PullIntoDescriptor + ): Promise> { + if (this.#status === 'errored') { + desc.reject(this.#storedError); + return desc.promise; + } + if (this.#status === 'closed') { + // Closed-stream BYOB semantics: hand the (transferred) buffer back + // via a zero-length view, done: true. + desc.resolve({ + done: true, + value: new desc.viewCtor(desc.buffer, desc.byteOffset, 0), + }); + return desc.promise; + } + ArrayPrototypePush(this.#requests, { + kind: 'byob', + desc, + } satisfies NativeByobRequestEntry); + return desc.promise; + } + + get hasPendingPullInto(): boolean { + const requests = this.#requests; + for (let i = 0; i < requests.length; i++) { + if ((requests[i] as NativeRequest).kind === 'byob') return true; + } + return false; + } + + get hasPartiallyFulfilledRead(): boolean { + const head = this.#requests[0]; + return ( + head !== undefined && head.kind === 'byob' && head.desc.bytesFilled > 0 + ); + } + + get headPullInto(): PullIntoDescriptor | undefined { + const head = this.#requests[0]; + return head !== undefined && head.kind === 'byob' ? head.desc : undefined; + } + + get pendingPullIntoView(): Uint8Array | undefined { + const head = this.#requests[0]; + if (head === undefined || head.kind !== 'byob') return undefined; + const desc = head.desc; + return new Uint8Array( + desc.buffer, + desc.byteOffset + desc.bytesFilled, + desc.byteLength - desc.bytesFilled + ); + } + + respondBYOB(bytesWritten: number): void { + const head = this.#requests[0]; + if (head === undefined || head.kind !== 'byob') { + throw new TypeError('No pending BYOB read to respond to'); + } + this.respondToDesc(head.desc, bytesWritten); + } + + commitPullIntosOnClose(): void { + // The fused close-commit happens in closeFromSource(); once the + // conduit has left the active state there is nothing left to commit. + } + + drainNoneDescriptors(): void { + // Native conduit doesn't use releaseLock pull-into descriptors. + } + + shiftAutoAllocateDescriptor(): PullIntoDescriptor | undefined { + // Native conduit doesn't use autoAllocateChunkSize. + return undefined; + } + + // --- Pull pacing ----------------------------------------------------------- + + // Standard [[pulling]]/[[pullAgain]] serialization. Demand-driven: pull + // only while requests are pending (effective high-water mark of zero). + // Re-evaluated after each pull settles for the REMAINING requests; a + // single request never spans pulls (the respond is the complete answer + // for its read — see the min-read contract). + // + // PER-PULL CANCELLATION: each pull invocation gets a fresh + // AbortController whose signal is passed as pull(controller, signal). + // The source contractually checks signal.aborted immediately before + // enqueue/respond; the controller backstop refuses delivery on an + // aborted signal. Settle handlers compare their captured controller + // reference against #pullAbortController — a mismatch means the pull + // was aborted or superseded and its settlement is inert (the rejection + // from kj cancellation is swallowed, preventing a healthy stream from + // erroring on a cancelled read). + pullIfNeeded(): void { + if (this.#status !== 'active') return; + if (this.#requests.length === 0) return; + const pullAlgorithm = this.#pullAlgorithm; + if (pullAlgorithm === undefined) return; + if (this.#pulling) { + this.#pullAgain = true; + return; + } + const controller = this.#controller; + if (controller === undefined) return; + this.#pulling = true; + this.#deliveredThisPull = false; + const ac = new AbortController(); + this.#pullAbortController = ac; + const signal = AbortControllerSignalGet(ac); + markPromiseHandled( + PromisePrototypeThen( + pullAlgorithm(controller, signal), + () => { + // Superseded-settle check: tee/extract may have cleared the + // controller reference; this settlement belongs to a dead pull. + if (this.#pullAbortController !== ac) return; + this.#pulling = false; + this.#pullAbortController = undefined; + // Aborted-pull check: the pull's signal was aborted (reader + // release, cancel, tee, extract). The source contractually + // held its data; skip the completion latch and just schedule + // the next pull if demand exists. + if (AbortSignalAbortedGet(signal)) { + if (this.#status === 'active' && this.#requests.length > 0) { + this.pullIfNeeded(); + } + return; + } + // COMPLETION LATCH: a pull that settled without delivering, + // closing, or erroring while requests are still pending is a + // broken source. Error the stream to prevent a busy-loop + // re-pull against a source that will never deliver. + if ( + !this.#deliveredThisPull && + this.#status === 'active' && + this.#requests.length > 0 + ) { + this.errorFromSource( + new TypeError( + 'native source pull settled without delivering, closing, ' + + 'or erroring — the source appears broken' + ) + ); + return; + } + if ( + this.#pullAgain || + (this.#status === 'active' && this.#requests.length > 0) + ) { + this.#pullAgain = false; + this.pullIfNeeded(); + } + }, + (e: unknown) => { + if (this.#pullAbortController !== ac) return; // superseded + this.#pulling = false; + this.#pullAbortController = undefined; + // Aborted-pull rejection is expected (kj cancellation, etc.) + // — not a source error. Schedule the next pull if needed. + if (AbortSignalAbortedGet(signal)) { + if (this.#status === 'active' && this.#requests.length > 0) { + this.pullIfNeeded(); + } + return; + } + this.#pullAgain = false; + this.errorFromSource(e); + } + ) as Promise + ); + } + + // Aborts the in-flight pull's signal. #pulling and #pullAbortController + // are deliberately NOT cleared — the aborted pull's promise must settle + // before a new pull can start, maintaining the non-reentrant [[pulling]] + // / [[pullAgain]] serialization. The settle handler detects the abort + // via signal.aborted and skips the completion latch. + #abortCurrentPull(reason: unknown): void { + const ac = this.#pullAbortController; + if (ac === undefined) return; + this.#pullAgain = false; + this.#deliveredThisPull = false; + this.#byobRequestCache = null; + AbortControllerAbort(ac, reason); + } + + // --- The source-facing operations (via the controller façade) ------------- + + isHeadDesc(desc: PullIntoDescriptor): boolean { + const head = this.#requests[0]; + return ( + this.#status === 'active' && + head !== undefined && + head.kind === 'byob' && + head.desc === desc + ); + } + + getByobRequest(): NativeReadableStreamBYOBRequest | null { + const head = this.#requests[0]; + if ( + this.#status !== 'active' || + head === undefined || + head.kind !== 'byob' + ) { + return null; + } + const cached = this.#byobRequestCache; + if (cached !== null && getRequestDesc(cached) === head.desc) { + return cached; + } + const request = new NativeReadableStreamBYOBRequest(this, head.desc); + this.#byobRequestCache = request; + return request; + } + + get desiredSize(): number | null { + if (this.#status === 'errored') return null; + // 0 in both the closed and readable states: there is no queue, so the + // demand-driven model never reports positive headroom. + return 0; + } + + enqueue(chunk: ArrayBufferView): void { + // A pull may legally settle after close/cancel/error/tee; a late + // enqueue is then a no-op rather than a violation. + if (this.#status !== 'active') return; + if (!isArrayBufferView(chunk)) { + throw new TypeError('enqueue requires an ArrayBufferView'); + } + if (this.#deliveredThisPull) { + // Contract: at most one enqueue-or-respond per pull invocation. + // Structurally guaranteed on the native side; kept as a loud check + // for mock authors. + throw new TypeError( + 'the native source must deliver at most once per pull invocation' + ); + } + const head = this.#requests[0]; + if (head === undefined) { + // REFUSAL BACKSTOP: the demand was withdrawn (reader released / + // stream cancelled) and the pull's signal was aborted. A + // conforming source checks signal.aborted before delivery and + // stashes the bytes for the next pull. Refusing loudly here + // catches sources that skip the check. + throw new TypeError( + 'delivery refused: the pull was aborted (the source must check ' + + 'signal.aborted before enqueue/respond and stash data for ' + + 'redelivery)' + ); + } + if (head.kind !== 'byob') { + // Normalize to a Uint8Array over the same bytes (the conduit is + // byte-oriented). Metadata reads go through captured getters. + let view: Uint8Array; + if (isUint8Array(chunk)) { + view = chunk as Uint8Array; + } else { + const isDataView = + TypedArrayPrototypeGetSymbolToStringTag(chunk) === undefined; + const buffer = ( + isDataView + ? DataViewPrototypeGetBuffer(chunk) + : TypedArrayPrototypeGetBuffer(chunk) + ) as ArrayBuffer; + const byteOffset = ( + isDataView + ? DataViewPrototypeGetByteOffset(chunk) + : TypedArrayPrototypeGetByteOffset(chunk) + ) as number; + const byteLength = ( + isDataView + ? DataViewPrototypeGetByteLength(chunk) + : TypedArrayPrototypeGetByteLength(chunk) + ) as number; + view = new Uint8Array(buffer, byteOffset, byteLength); + } + // EXPECTED-LENGTH CONTRACT: overflow check before commit. + this.#accountDelivery(TypedArrayPrototypeGetByteLength(view) as number); + ArrayPrototypeSplice(this.#requests, 0, 1); + this.#deliveredThisPull = true; + head.read.resolve({ done: false, value: view }); + return; + } + // Head is a BYOB read: the source must use the byobRequest instead. + // (The fill could be emulated by copying, but the contract says the + // native side discriminates via byobRequest — keep the boundary firm.) + throw new TypeError( + 'enqueue while a BYOB read is pending; use byobRequest.respond()' + ); + } + + respondToDesc(desc: PullIntoDescriptor, bytesWritten: number): void { + // Source-initiated stragglers: silent discard (see enqueue comment). + if (this.#status !== 'active') return; + if (this.#deliveredThisPull) { + throw new TypeError( + 'the native source must deliver at most once per pull invocation' + ); + } + if (!this.isHeadDesc(desc)) { + if (this.#requests[0] === undefined) { + // REFUSAL BACKSTOP: same as the enqueue case — the pull was + // aborted, the source should have checked signal.aborted. + throw new TypeError( + 'delivery refused: the pull was aborted (the source must check ' + + 'signal.aborted before enqueue/respond and stash data for ' + + 'redelivery)' + ); + } + throw new TypeError('This BYOB request has been invalidated'); + } + const written = +bytesWritten; + if (NumberIsNaN(written) || written % 1 !== 0 || written < 0) { + throw new TypeError('bytesWritten must be a non-negative integer'); + } + if (written === 0) { + throw new TypeError( + 'respond(0) is not supported for native streams; an empty answer ' + + 'means end-of-stream — call close() instead' + ); + } + const filled = desc.bytesFilled + written; + if (filled > desc.byteLength) { + throw new RangeError('bytesWritten exceeds the remaining view size'); + } + if (filled % desc.elementSize !== 0) { + // Every respond commits (see below), so every respond must account + // a whole number of elements — there is no queue to carry a + // remainder. + throw new TypeError( + 'native sources must respond in whole-element multiples of the view' + ); + } + // EXPECTED-LENGTH CONTRACT: overflow check before commit. + this.#accountDelivery(written); + desc.bytesFilled = filled; + this.#byobRequestCache = null; + this.#deliveredThisPull = true; + ArrayPrototypeSplice(this.#requests, 0, 1); + const value = new desc.viewCtor( + desc.buffer, + desc.byteOffset, + filled / desc.elementSize + ); + if (filled >= desc.minimumFill) { + desc.resolve({ done: false, value }); + return; + } + // MIN-READ CONTRACT: the respond is the source's COMPLETE answer for + // this read. Delivering fewer bytes than the requested minimum + // (atLeast) signals end-of-stream: the partial fill commits FUSED + // with done: true, the stream closes, and every subsequent read + // returns EOF. The conduit never re-pulls an unsatisfied read — any + // accumulation toward the minimum happens inside the native source + // (tryRead-style). State transitions before promise resolutions. + // + // EXPECTED-LENGTH: under-delivery is a close signal, so the + // exact-total contract applies — landing short of expectedLength is + // underflow and errors the stream instead of closing it. (Landing + // exactly on it is a legitimate fused close.) + if ( + this.#expectedLength !== undefined && + this.#bytesDelivered < this.#expectedLength + ) { + const error = new RangeError( + 'native source closed (via min-read under-delivery) before ' + + 'producing its declared expectedLength' + ); + this.#status = 'errored'; + this.#storedError = error; + desc.reject(error); + this.errorAllReads(error); + this.#hooks.errorStream(error); + return; + } + this.#status = 'closed'; + desc.resolve({ done: true, value }); + this.#settleRemainingAsEof(); + this.#hooks.closeStream(); + } + + // The exact-total accounting: adds `byteLength` to the running total, + // throwing RangeError on overflow (delivering past the declared + // expectedLength). Called from both delivery paths before commit. + // On overflow, errors the stream explicitly before throwing — the + // throw alone only reaches the caller, which may be an external sink + // that only errors the writable side (leaving the readable hanging). + #accountDelivery(byteLength: number): void { + if (this.#expectedLength === undefined) return; + const delivered = this.#bytesDelivered + BigInt(byteLength); + if (delivered > this.#expectedLength) { + const e = new RangeError( + 'native source delivered more bytes than its declared expectedLength' + ); + this.errorFromSource(e); + throw e; + } + this.#bytesDelivered = delivered; + } + + get expectedLength(): bigint | undefined { + return this.#expectedLength; + } + + // Resolves every queued request as end-of-stream (default reads get + // {done, undefined}; BYOB reads get their buffer back via a zero-length + // view). Shared by the close paths. + #settleRemainingAsEof(): void { + const requests = this.#requests; + this.#requests = []; + this.#byobRequestCache = null; + for (let i = 0; i < requests.length; i++) { + const request = requests[i] as NativeRequest; + if (request.kind === 'default') { + request.read.resolve({ done: true, value: undefined }); + } else { + const desc = request.desc; + desc.resolve({ + done: true, + value: new desc.viewCtor(desc.buffer, desc.byteOffset, 0), + }); + } + } + } + + respondToDescWithNewView( + desc: PullIntoDescriptor, + view: ArrayBufferView + ): void { + // Source-initiated stragglers: silent discard. + if (this.#status !== 'active') return; + if (!this.isHeadDesc(desc)) { + if (this.#requests[0] === undefined) { + // REFUSAL BACKSTOP (same as enqueue/respond). + throw new TypeError( + 'delivery refused: the pull was aborted (the source must check ' + + 'signal.aborted before enqueue/respond and stash data for ' + + 'redelivery)' + ); + } + throw new TypeError('This BYOB request has been invalidated'); + } + if (!isArrayBufferView(view)) { + throw new TypeError('respondWithNewView requires an ArrayBufferView'); + } + const isDataView = + TypedArrayPrototypeGetSymbolToStringTag(view) === undefined; + const buffer = ( + isDataView + ? DataViewPrototypeGetBuffer(view) + : TypedArrayPrototypeGetBuffer(view) + ) as ArrayBuffer; + const byteOffset = ( + isDataView + ? DataViewPrototypeGetByteOffset(view) + : TypedArrayPrototypeGetByteOffset(view) + ) as number; + const byteLength = ( + isDataView + ? DataViewPrototypeGetByteLength(view) + : TypedArrayPrototypeGetByteLength(view) + ) as number; + if ( + ArrayBufferPrototypeByteLengthGet(buffer) !== + ArrayBufferPrototypeByteLengthGet(desc.buffer) + ) { + throw new RangeError( + "The view's buffer must have the same byteLength as the request" + ); + } + if (byteOffset !== desc.byteOffset + desc.bytesFilled) { + throw new RangeError( + 'The new view must start where the previous fill ended' + ); + } + if (desc.bytesFilled + byteLength > desc.byteLength) { + throw new RangeError('The new view exceeds the remaining view size'); + } + // Adopt the new backing buffer (the native source may have swapped + // buffers), then account the bytes like a plain respond. + desc.buffer = buffer; + this.respondToDesc(desc, byteLength); + } + + closeFromSource(): void { + if (this.#status !== 'active') return; // late settle tolerated + // EXPECTED-LENGTH CONTRACT: source-initiated close before reaching + // the declared total is underflow — an error, not a clean close. + // (Consumer-initiated cancel/tee/extraction are exempt; the source + // kept its promise, the consumer left.) + if ( + this.#expectedLength !== undefined && + this.#bytesDelivered < this.#expectedLength + ) { + this.errorFromSource( + new RangeError( + 'native source closed before producing its declared expectedLength' + ) + ); + return; + } + this.#status = 'closed'; + this.#byobRequestCache = null; + + // DEFENSIVE, currently unreachable: under the min-read contract every + // respond commits (satisfied → done: false; under-delivered → fused + // EOF), so a partially-filled descriptor cannot persist to close(). + // Retained because the contract describes close() fusing with a + // partial fill, should a future revision reintroduce partial + // responds. + const head = this.#requests[0]; + if ( + head !== undefined && + head.kind === 'byob' && + head.desc.bytesFilled > 0 + ) { + const desc = head.desc; + if (desc.bytesFilled % desc.elementSize !== 0) { + // Closed mid-element: unrecoverable (no queue to carry the + // remainder). Error everything instead. + const error = new TypeError( + 'native source closed mid-element on a BYOB read' + ); + this.#status = 'errored'; + this.#storedError = error; + this.errorAllReads(error); + this.#hooks.errorStream(error); + return; + } + ArrayPrototypeSplice(this.#requests, 0, 1); + desc.resolve({ + done: true, + value: new desc.viewCtor( + desc.buffer, + desc.byteOffset, + desc.bytesFilled / desc.elementSize + ), + }); + } + + this.#settleRemainingAsEof(); + this.#hooks.closeStream(); + } + + errorFromSource(reason: unknown): void { + if (this.#status !== 'active') return; // late settle tolerated + this.#status = 'errored'; + this.#storedError = reason; + this.errorAllReads(reason); + this.#hooks.errorStream(reason); + } + + // Spec PromiseCall over the extracted cancel method; resolved if the + // source declared none. Invoked through the controller dispatch chain + // (the stream layer's cancel policy callback). + cancelSource(reason: unknown): Promise { + if (this.#status === 'active') this.#status = 'closed'; + this.#byobRequestCache = null; + const cancelAlgorithm = this.#cancelAlgorithm; + if (cancelAlgorithm === undefined) { + return PromiseResolve() as Promise; + } + return cancelAlgorithm(reason); + } + + onReaderRelease(): void { + // The released reader's requests are rejected separately via + // cancelReadsForReader; here we just drop any byobRequest minted for + // a descriptor that is about to be rejected. + this.#byobRequestCache = null; + } + + // The native tee handoff: calls the source's tee hook, which returns a + // pair of new native source objects and leaves the original source + // closed. The stream layer constructs the branch streams and locks the + // parent; this conduit becomes inert. + teeSource(): [object, object] { + if (this.#status !== 'active') { + throw new TypeError('Cannot tee a closed or errored native stream'); + } + if (this.#requests.length > 0) { + // Unreachable through tee() (pending reads imply a locked stream, + // and tee() rejects locked streams upstream) — kept as a guard. + throw new TypeError('Cannot tee a native stream with pending reads'); + } + // ABORT SITE: moot by construction (an in-flight pull at tee time + // implies a prior release, which already aborted), but specified for + // completeness. + this.#abortCurrentPull(new TypeError('stream was teed')); + const teeAlgorithm = this.#teeAlgorithm; + if (teeAlgorithm === undefined) { + throw new TypeError('This native stream source does not support tee'); + } + const result = teeAlgorithm() as { 0: unknown; 1: unknown } | null; + const branch1 = isActualObject(result) + ? (result as { 0: unknown })[0] + : undefined; + const branch2 = isActualObject(result) + ? (result as { 1: unknown })[1] + : undefined; + if (!isActualObject(branch1) || !isActualObject(branch2)) { + throw new TypeError( + 'The native source tee hook must return a pair of source objects' + ); + } + // Per the contract the hook leaves the original source closed; mirror + // that here so late pull settles become no-ops. + this.#status = 'closed'; + this.#byobRequestCache = null; + return [branch1, branch2]; + } + + // JS-to-C++ extraction: returns the original native source object and + // terminates the conduit. The caller (readable.ts extractor) handles + // stream-level locking/disturbing; this method handles conduit-level + // cleanup only. Precondition: the caller validated locked/disturbed + // already, so no in-flight pull is possible (pull requires a prior + // read, which sets disturbed). The abort site is specified anyway. + detachForExtraction(): object { + const source = this.#source; + this.#abortCurrentPull(new TypeError('stream was extracted')); + this.#status = 'closed'; + this.#byobRequestCache = null; + return source; + } +} + +// --------------------------------------------------------------------------- +// Construction glue + +// Extraction follows the queued controllers' conventions: properties are +// read ONCE, alphabetically, validated, and invoked thereafter only via +// captured uncurryThis wrappers with the source as `this` (spec +// CreateAlgorithmFromUnderlyingMethod / PromiseCall). `start` is +// deliberately NOT read: native sources have no start algorithm (assumed +// no-op per the contract; its presence is simply ignored). +function createNativeReadableStreamParts( + source: object, + hooks: NativeStreamHooks +): { + controller: NativeReadableStreamController; + conduit: NativePullConduit; +} { + const { + autoAllocateChunkSize, + cancel: cancelFn, + expectedLength: rawExpectedLength, + pull: pullFn, + tee: teeFn, + type, + } = source as { + autoAllocateChunkSize?: unknown; + cancel?: unknown; + expectedLength?: unknown; + pull?: unknown; + tee?: unknown; + type?: unknown; + }; + + // Native sources must not look like queued sources: the reader layer's + // autoAllocate synthesis is queued-only and relies on this prohibition, + // and a declared type would suggest the object was built for the queued + // byte path. + if (autoAllocateChunkSize !== undefined) { + throw new TypeError( + 'Native stream sources must not declare autoAllocateChunkSize' + ); + } + if (type !== undefined) { + throw new TypeError('Native stream sources must not declare type'); + } + if (cancelFn !== undefined && typeof cancelFn !== 'function') { + throw new TypeError('underlyingSource.cancel must be a function'); + } + if (pullFn !== undefined && typeof pullFn !== 'function') { + throw new TypeError('underlyingSource.pull must be a function'); + } + if (teeFn !== undefined && typeof teeFn !== 'function') { + throw new TypeError('underlyingSource.tee must be a function'); + } + // Non-standard extension: the TOTAL bytes this source promises to + // produce. Read once here and cached; the conduit enforces the + // contract (overflow at delivery, underflow at source-initiated + // close). C++ reads it directly off the extracted source object. + const expectedLength = normalizeExpectedLength(rawExpectedLength); + + let pullAlgorithm: PullAlgorithm | undefined; + if (pullFn !== undefined) { + const callPull = uncurryThis(pullFn); + pullAlgorithm = ( + controller: NativeReadableStreamController, + signal: AbortSignal + ) => { + try { + return PromiseResolve( + callPull(source, controller, signal) + ) as Promise; + } catch (e) { + return PromiseReject(e) as Promise; + } + }; + } + let cancelAlgorithm: CancelAlgorithm | undefined; + if (cancelFn !== undefined) { + const callCancel = uncurryThis(cancelFn); + cancelAlgorithm = (reason: unknown) => { + try { + return PromiseResolve(callCancel(source, reason)) as Promise; + } catch (e) { + return PromiseReject(e) as Promise; + } + }; + } + let teeAlgorithm: (() => unknown) | undefined; + if (teeFn !== undefined) { + const callTee = uncurryThis(teeFn); + // Synchronous by contract: the branches exist upon return. + teeAlgorithm = () => callTee(source); + } + + const conduit = new NativePullConduit( + source, + { pullAlgorithm, cancelAlgorithm, teeAlgorithm, expectedLength }, + hooks + ); + const controller = new NativeReadableStreamController(conduit); + conduit.attachController(controller); + return { controller, conduit }; +} + +// --------------------------------------------------------------------------- +// Dispatch-chain behaviors (joined to the chained controller helpers by +// readable.ts; the chain lets live across the module fence). + +function nativeControllerPullIfNeeded( + controller: NativeReadableStreamController +): void { + getControllerConduit(controller).pullIfNeeded(); +} + +function nativeControllerCancelSteps( + controller: NativeReadableStreamController, + reason: unknown +): Promise { + return getControllerConduit(controller).cancelSource(reason); +} + +function nativeControllerMaybeCloseStream( + _controller: NativeReadableStreamController +): void { + // No deferred-close bookkeeping exists: close propagation is the fused + // close-commit on the read path. +} + +function nativeControllerOnReaderRelease( + controller: NativeReadableStreamController +): void { + getControllerConduit(controller).onReaderRelease(); +} + +function nativeControllerTeeSource( + controller: NativeReadableStreamController +): [object, object] { + return getControllerConduit(controller).teeSource(); +} + +function nativeControllerExtractSource( + controller: NativeReadableStreamController +): object { + return getControllerConduit(controller).detachForExtraction(); +} + +function nativeControllerExpectedLength( + controller: NativeReadableStreamController +): bigint | undefined { + return getControllerConduit(controller).expectedLength; +} + +// Internal namespace, never re-exported to users (kNativeSource and +// kExtractNativeSource excepted, TEMPORARILY, via streams.ts — see the +// module header). +const nativeStreamInternals = { + kNativeSource, + kExtractNativeSource, + kNativeSink, + kExtractNativeSink, + isNativeUnderlyingSource, + isNativeUnderlyingSink, + isNativeController, + createNativeReadableStreamParts, + nativeControllerPullIfNeeded, + nativeControllerCancelSteps, + nativeControllerMaybeCloseStream, + nativeControllerOnReaderRelease, + nativeControllerTeeSource, + nativeControllerExtractSource, + nativeControllerExpectedLength, +}; + +// Type-only exports (fully erased at runtime — the loader sees only the +// module.exports assignment below, matching the queue.ts pattern). +// NativeStreamInternals lets the require() site in readable.ts cast the +// untyped loader result back to the real shape, preserving the brand +// predicates' type-guard narrowing. +export type { NativeReadableStreamController, NativePullConduit }; +export type NativeStreamInternals = typeof nativeStreamInternals; + +module.exports = { nativeStreamInternals }; diff --git a/src/per_isolate/webstreams/queue.ts b/src/per_isolate/webstreams/queue.ts new file mode 100644 index 00000000000..eb1788bc13f --- /dev/null +++ b/src/per_isolate/webstreams/queue.ts @@ -0,0 +1,1317 @@ +'use strict'; + +// Single-queue / multi-cursor model backing the TypeScript Streams +// implementation: the QUEUED consumer backend. See the queue/cursor design +// doc for the full rationale, and native-stream-integration.md §10 for the +// fence conventions separating this backend from the native one. +// +// One StreamQueue per controller holds each chunk exactly once. Every +// consumer (the stream itself, and each tee branch) owns a QueueCursor — a +// logical position into the shared queue. Entries are reclaimed once every +// live cursor has advanced past them; the slowest cursor drives +// backpressure (desiredSize = highWaterMark - max(cursor remaining size)). +// +// QUEUED-BACKEND INVARIANTS (binding on changes to this file; the native +// backend in native.ts has a DIFFERENT set — do not port logic across the +// fence without checking both): +// - The CLOSE_SENTINEL is always the LAST slot; it is the ONLY close- +// propagation mechanism (drain-then-close per cursor; BYOB descriptors +// stay pending at the sentinel per the spec footgun — the native +// backend deliberately differs with its fused close-commit). +// - read() takes the fast path ONLY when no reads are pending (per-reader +// FIFO; entries and pending reads coexist under batched notification). +// - Byte entries are {buffer, byteOffset, byteLength} triples; queue +// internals never read view metadata through patchable getters. +// - Copy-on-read for shared byte entries: copy iff any OTHER live cursor +// still needs the entry (exact last-consumer test) — REQUIRED for +// soundness, not just spec parity. +// - Cursors hold weak owner refs; orphan pruning happens on every cursor +// iteration, with the FinalizationRegistry as the idle-queue backstop +// (the native backend needs NEITHER — JSG owns its source lifetime). +// - desiredSize reflects the SLOWEST live cursor; a pending read on any +// cursor overrides backpressure at the controller. +// +// Nothing in this module is ever exposed to user code: queues and cursors +// are held in #private fields of the stream classes. Method calls on these +// classes still follow the bootstrap primordials discipline (no bare +// prototype lookups on builtins, no for...of over Sets/arrays). + +import type { + PromiseWithResolvers as PromiseWithResolversType, + ReadableStreamReadResult, +} from './types'; + +const { + ArrayBufferPrototypeSlice, + ArrayPrototypePush, + ArrayPrototypeShift, + ArrayPrototypeSplice, + FinalizationRegistry, + FinalizationRegistryPrototypeRegister, + FinalizationRegistryPrototypeUnregister, + MathMin, + ObjectCreate, + PromiseResolve, + PromiseReject, + PromiseWithResolvers, + ReflectConstruct, + SafeSet, + Symbol, + TypeError, + TypedArrayPrototypeSet, + TypedArrayPrototypeSlice, + Uint8Array, + WeakRef, + WeakRefPrototypeDeref, +} = primordials; + +// Read-result objects are plain { value, done } objects with the default +// Object.prototype, matching spec behavior. Internal machinery is protected +// by captured primordials (PromisePrototypeThen, etc.), not by the result +// object's prototype chain. If the user patches Object.prototype.then, that +// only affects their own promise resolution — same trade-off as Node.js. +export function createReadResult( + value: T, + done: false +): { value: T; done: false }; +export function createReadResult( + value: T | undefined, + done: true +): { value: T | undefined; done: true }; +export function createReadResult( + value: T | undefined, + done: boolean +): { value: T | undefined; done: boolean } { + return { value, done }; +} + +// --------------------------------------------------------------------------- +// Entries and the close sentinel + +export interface QueueEntry { + value: T; + // As computed by the queuing strategy's size() (byteLength for byte + // streams). Drives remainingSize/desiredSize accounting. + size: number; +} + +// End-of-stream marker stored in the entries array after close(). Always the +// LAST slot; nothing may be enqueued after it. The sentinel is the ONLY +// close-propagation mechanism: a cursor observes the close exactly when its +// position reaches the sentinel, guaranteeing it drains all buffered data +// first. Every consuming path must handle it explicitly (it has no value and +// no size). +const CLOSE_SENTINEL: symbol = Symbol('closeSentinel'); + +export type QueueSlot = QueueEntry | symbol; + +function isQueueEntry( + slot: QueueSlot | undefined +): slot is QueueEntry { + return slot !== undefined && slot !== CLOSE_SENTINEL; +} + +// Byte queue entries are spec-shaped {buffer, byteOffset, byteLength} +// triples, NOT live ArrayBufferViews. The byte controller normalizes chunks +// into triples at the enqueue() trust boundary (after validation and buffer +// transfer), so queue internals never read view metadata through patchable +// %TypedArray%.prototype getters. +export interface ByteQueueEntry { + buffer: ArrayBuffer; + byteOffset: number; + byteLength: number; +} + +// --------------------------------------------------------------------------- +// Pending reads + +export interface PendingRead { + resolve: (result: ReadableStreamReadResult) => void; + reject: (reason: unknown) => void; + // The reader that submitted this read. Used for selective rejection when + // a reader's lock is released (the cursor itself outlives readers). + // Also keeps the reader (and through it, the owning stream) strongly + // reachable, which is what guarantees a cursor with pending reads is + // never orphaned. + reader: object; +} + +type ArrayBufferViewCtor = new ( + buffer: ArrayBuffer, + byteOffset: number, + length: number +) => ArrayBufferView; + +// Spec-shaped pull-into descriptor ([[pendingPullIntos]] item). We never +// hold the caller's live view: its backing buffer is transferred at +// read(view) time (and re-transferred on every respond()), so the result +// view is reconstructed from these primitives only at fulfillment. +export interface PullIntoDescriptor { + // CURRENT backing buffer — re-pointed by the controller after each + // transfer (read(view), respond(), respondWithNewView()). + buffer: ArrayBuffer; + // Spec "buffer byte length": the buffer's byteLength at creation time. + // Persists even when `buffer` has been transferred (detached → byteLength 0). + // Used by respondWithNewView() to validate the replacement buffer matches. + bufferByteLength: number; + byteOffset: number; // start of the caller's view within buffer + byteLength: number; // caller's view length in bytes + bytesFilled: number; // accumulates across enqueues/responds + minimumFill: number; // min × elementSize, in BYTES + elementSize: number; // BYTES_PER_ELEMENT (1 for DataView) + // Captured via safe view-type detection — NEVER view.constructor, which + // is user-controllable via an own property. + viewCtor: ArrayBufferViewCtor; + // 'default' = synthetic descriptor for a default read on a byte stream + // with autoAllocateChunkSize set. + // 'none' = descriptor whose reader was released (releaseLock); respond() + // enqueues the data rather than resolving a read promise. + readerType: 'byob' | 'default' | 'none'; + promise: Promise>; + resolve: (result: ReadableStreamReadResult) => void; + reject: (reason: unknown) => void; + reader: object; +} + +// --------------------------------------------------------------------------- +// The consumer interface (the FENCE between backends) +// +// The reader layer in readable.ts programs against this contract and MUST +// remain backend-blind: it never knows whether it is talking to the queued +// cursor machinery in this file or the native pull conduit (native.ts). +// All backend divergence lives behind this interface or at the five +// enumerated BACKEND-DISPATCH points (see native-stream-integration.md +// §10 "Fences and signposts"). + +export interface StreamConsumer { + // Submit a default read. Per-reader FIFO; reader identity enables + // selective rejection on lock release. + read(reader: object): Promise>; + // Attempt a synchronous read. Returns the result directly when data (or + // the close sentinel) is immediately available at the cursor, or + // undefined when no data is buffered / reads are already queued (caller + // must fall back to the async read() path). This avoids the microtask + // gap that async/await introduces on an already-resolved promise, + // preserving spec PullSteps timing for the drain-then-close check. + tryReadSync(reader: object): ReadableStreamReadResult | undefined; + // Bulk read for the draining reader / pipeTo. Synchronous: collects what + // is buffered (the draining reader handles the empty-then-wait case). + drain(maxSize?: number): { chunks: V[]; done: boolean }; + // Reject pending reads submitted by a specific reader (lock release). + cancelReadsForReader(reader: object, reason: unknown): void; + // Reject all pending reads (stream ERROR path only). + errorAllReads(reason: unknown): void; + // Resolve all pending reads as done (stream CANCEL path). + resolveAllReadsAsDone(): void; + readonly hasPendingRead: boolean; + // Fulfill the first pending read directly with a value, skipping the queue. + // Spec ReadableStreamFulfillReadRequest: shift + chunk steps. + fulfillFirstPendingRead(value: V): void; + // Stream-cancel teardown. The STREAM layer owns the source-cancel policy + // (tee composite hooks vs direct controller cancel) and passes it as + // `decideSourceCancel`, invoked with whether this consumer is the last + // one; the consumer owns the mechanics and the ordering guarantee (the + // decision runs BEFORE consumer removal, so a reason-carrying cancel + // wins any idempotency cache over GC-path hooks). + cancelStream( + reason: unknown, + decideSourceCancel: (isLastConsumer: boolean) => Promise + ): Promise; +} + +// Byte-capable consumers additionally support the BYOB machinery. Both the +// ByteStreamCursor (queued) and the NativePullConduit (native) satisfy +// this. +export interface ByteStreamConsumer extends StreamConsumer { + readBYOB( + desc: PullIntoDescriptor + ): Promise>; + readonly hasPendingPullInto: boolean; + readonly hasPartiallyFulfilledRead: boolean; + readonly headPullInto: PullIntoDescriptor | undefined; + readonly pendingPullIntoView: Uint8Array | undefined; + respondBYOB(bytesWritten: number): void; + commitPullIntosOnClose(): void; + // Spec: enqueue() drains 'none'-typed head descriptors BEFORE adding + // the new chunk. Called by the controller's enqueue() path. + drainNoneDescriptors(): void; + // Spec step 9.3: if the head descriptor is an auto-allocate + // (readerType 'default'), shift it out and return it so the controller + // can fulfill the read directly from the enqueued chunk. + shiftAutoAllocateDescriptor(): PullIntoDescriptor | undefined; +} + +// --------------------------------------------------------------------------- +// Orphan-detection backstop +// +// Lazy WeakRef pruning (see #forEachLiveCursor) only runs inside queue +// operations. An idle queue whose consumers were all GC'd — but which is +// pinned from the C++ side via the controller — would otherwise never +// observe "all cursors gone" and never cancel the underlying source. The +// registry provides the active signal; cleanup scheduling is controlled by +// workerd and deliberately coarse. + +interface CursorLike { + queue: { removeCursor(cursor: unknown): void }; +} + +const cursorCleanupRegistry = new FinalizationRegistry((cursor: CursorLike) => { + cursor.queue.removeCursor(cursor); +}); + +// --------------------------------------------------------------------------- +// StreamQueue + +// T is the entry value type (ByteQueueEntry for byte streams); V is the type +// delivered to read results (Uint8Array remainder views for byte streams). +class StreamQueue { + #entries: QueueSlot[] = []; + #headOffset: number = 0; // logical index of #entries[0] + #cursors: Set>; + #highWaterMark: number; + #state: 'readable' | 'closed' | 'errored' = 'readable'; + // Idempotent hook, wired to the controller: cancels the underlying source + // when the last consumer goes away. "cursorCount === 0" is a state, not an + // event — every path that can remove the last cursor funnels here. + #onAllCursorsGone: () => void; + #hadCursors: boolean = false; + #allGoneSignaled: boolean = false; + + constructor(highWaterMark: number, onAllCursorsGone: () => void) { + this.#cursors = new SafeSet(); + this.#highWaterMark = highWaterMark; + this.#onAllCursorsGone = onAllCursorsGone; + } + + // Iterate live cursors, pruning any whose owning stream has been GC'd. + // This is the ONLY way cursor iteration happens — raw iteration would + // skip orphan detection. SafeSet#forEach dispatches through the captured + // SetPrototypeForEach (deleting during forEach is safe per spec; for...of + // over a SafeSet is NOT pollution-safe and must not be used). + #forEachLiveCursor(fn: (cursor: QueueCursor) => void): void { + this.#cursors.forEach((cursor) => { + if (cursor.isOrphaned()) { + this.#cursors.delete(cursor); + unregisterCursorCleanup(cursor); + } else { + fn(cursor); + } + }); + this.#checkAllCursorsGone(); + } + + #checkAllCursorsGone(): void { + if ( + this.#hadCursors && + !this.#allGoneSignaled && + this.#cursors.size === 0 + ) { + this.#allGoneSignaled = true; + this.#onAllCursorsGone(); + } + } + + // The slowest cursor's backlog determines backpressure. Note that a + // pending read on any cursor overrides backpressure at the controller + // (shouldPull), so this is honest signaling, not a memory bound. + get desiredSize(): number { + let max = 0; + this.#forEachLiveCursor((cursor) => { + const remaining = cursor.remainingSize; + if (remaining > max) max = remaining; + }); + return this.#highWaterMark - max; + } + + get length(): number { + // Logical end position (one past the last slot). + return this.#headOffset + this.#entries.length; + } + + get cursorCount(): number { + // Prune orphans, then report. + this.#forEachLiveCursor(() => {}); + return this.#cursors.size; + } + + // The single live cursor, if there is exactly one. Used by the byte + // controller's byobRequest getter (zero-copy is only unambiguous with a + // single consumer). + get singleCursor(): QueueCursor | undefined { + if (this.cursorCount !== 1) return undefined; + let found: QueueCursor | undefined; + this.#forEachLiveCursor((cursor) => { + found = cursor; + }); + return found; + } + + anyCursorHasPendingRead(): boolean { + let any = false; + this.#forEachLiveCursor((cursor) => { + if (cursor.hasPendingRead) any = true; + }); + return any; + } + + // The exact "last consumer" test for copy-on-read: true if any OTHER + // live cursor has not yet advanced past the entry at logicalIndex. When + // false, the asking cursor is the entry's final consumer and may take + // the underlying buffer zero-copy. + hasOtherLiveCursorAtOrBefore( + cursor: QueueCursor, + logicalIndex: number + ): boolean { + let found = false; + this.#forEachLiveCursor((other) => { + if (other !== cursor && other.position <= logicalIndex) { + found = true; + } + }); + return found; + } + + // True if any live cursor satisfies the predicate. Used by the byte + // controller's close() validation across ALL consumers (tee branches + // included), not just the single-cursor case. + someLiveCursor(predicate: (cursor: QueueCursor) => boolean): boolean { + let found = false; + this.#forEachLiveCursor((cursor) => { + if (predicate(cursor)) found = true; + }); + return found; + } + + // Snapshot of the live owner streams (one per live cursor). Used for + // error propagation across tee branches — the queue itself stays + // policy-free; the controller decides what to do with the owners. + getLiveOwners(): object[] { + const owners: object[] = []; + this.#forEachLiveCursor((cursor) => { + const owner = cursor.ownerDeref(); + if (owner !== undefined) { + ArrayPrototypePush(owners, owner); + } + }); + return owners; + } + + // Access a slot by logical position. May return the CLOSE_SENTINEL — + // callers must check (isQueueEntry) before touching value/size. + getEntry(logicalIndex: number): QueueSlot | undefined { + const actual = logicalIndex - this.#headOffset; + if (actual < 0 || actual >= this.#entries.length) return undefined; + return this.#entries[actual]; + } + + enqueue(entry: QueueEntry, notify: boolean = true): void { + // The controller pre-checks canCloseOrEnqueue before calling size(). + // A reentrant close()/error() from inside size() may change the state + // between the pre-check and this push; the spec's EnqueueValueWithSize + // has no state guard, so we accept the enqueue unconditionally. + if (this.#state === 'closed') { + // Reentrant close() from inside size() already pushed the sentinel. + // The spec's close just sets closeRequested — the chunk is still + // readable. Insert the entry BEFORE the sentinel so cursors drain + // through it before reaching the close marker. + const entries = this.#entries; + // Sentinel position BEFORE insertion — cursors at or past this + // point already resolved {done: true} and must not be touched. + const sentinelPos = this.#headOffset + entries.length - 1; + const sentinel = entries[entries.length - 1]; + entries[entries.length - 1] = entry; + ArrayPrototypePush(entries, sentinel as QueueSlot); + // Only update cursors that haven't yet reached the sentinel. + // A cursor at sentinelPos already drained and resolved done — + // inflating its remainingSize or notifying it would corrupt + // desiredSize and break the drain-then-close terminality guarantee. + this.#forEachLiveCursor((cursor) => { + if (cursor.position < sentinelPos) { + cursor.addToTotalSize(entry.size); + if (notify) cursor.notify(); + } + }); + } else { + ArrayPrototypePush(this.#entries, entry); + if (this.#state === 'readable') { + this.#forEachLiveCursor((cursor) => { + // Increment the cursor's running total BEFORE notify(), which may + // immediately consume the entry (decrementing it back). The +=/-= + // order preserves spec-mandated IEEE 754 drift. + cursor.addToTotalSize(entry.size); + if (notify) cursor.notify(); + }); + } + } + } + + // Push the close sentinel as the final slot and notify. Cursors that have + // already drained to the sentinel position resolve their pending reads as + // done; slower cursors discover the close independently when they reach + // it. + close(): void { + if (this.#state !== 'readable') { + throw new TypeError('Cannot close a closed or errored queue'); + } + this.#state = 'closed'; + ArrayPrototypePush(this.#entries, CLOSE_SENTINEL); + this.#forEachLiveCursor((cursor) => { + cursor.notify(); + }); + } + + // Stream error: reject all pending reads on every cursor (byte cursors + // also reject pending pull-intos — partial fills are lost) and drop all + // buffered data. Cursors remain attached; reads submitted after the error + // are rejected at the reader/stream layer via the stored error. + error(reason: unknown): void { + this.#state = 'errored'; + // Capture the logical end BEFORE dropping entries so cursor positions + // remain meaningful (they all now point at/past the end). + const end = this.length; + ArrayPrototypeSplice(this.#entries, 0, this.#entries.length); + this.#headOffset = end; + this.#forEachLiveCursor((cursor) => { + cursor.errorAllReads(reason); + }); + } + + // Called by the QueueCursor constructor (cursors self-register). `owner` + // is the stream object — held strongly only by the FinalizationRegistry + // key and weakly by the cursor. When forking (tee), the new cursor's + // constructor receives the source cursor's position AND byteOffset so the + // branch resumes exactly where the original left off. + addCursor(cursor: QueueCursor, owner: object): void { + this.#hadCursors = true; + this.#allGoneSignaled = false; + this.#cursors.add(cursor); + registerCursorCleanup(owner, cursor); + } + + removeCursor(cursor: QueueCursor): void { + this.#cursors.delete(cursor); + unregisterCursorCleanup(cursor); + this.#gc(); + this.#checkAllCursorsGone(); + } + + // Called whenever any cursor advances: reclaim entries every live cursor + // has passed. Orphaned cursors are pruned during iteration, so their + // stale positions never block reclamation. + onCursorAdvanced(): void { + this.#gc(); + } + + #gc(): void { + if (this.#cursors.size === 0) return; + let minPos = Infinity; + this.#forEachLiveCursor((cursor) => { + if (cursor.position < minPos) minPos = cursor.position; + }); + if (minPos === Infinity) return; // all cursors were orphaned + const freedCount = minPos - this.#headOffset; + if (freedCount > 0) { + ArrayPrototypeSplice(this.#entries, 0, freedCount); + this.#headOffset = minPos; + } + } +} + +function registerCursorCleanup(owner: object, cursor: object): void { + FinalizationRegistryPrototypeRegister( + cursorCleanupRegistry, + owner, + cursor, + cursor + ); +} + +function unregisterCursorCleanup(cursor: object): void { + FinalizationRegistryPrototypeUnregister(cursorCleanupRegistry, cursor); +} + +// --------------------------------------------------------------------------- +// QueueCursor + +class QueueCursor implements StreamConsumer { + #queue: StreamQueue; + // Weak back-reference to the owning ReadableStream. A cursor whose owner + // has been GC'd is an orphan and gets pruned by the queue. Note pending + // reads pin the owner (pending.reader → reader → stream), so orphaned + // implies no pending reads from any live reader. + #owner: unknown; + #position: number; + #byteOffset: number; // partial consumption of the entry at #position + #pendingReads: PendingRead[] = []; + // Running total mirroring the spec's [[queueTotalSize]]. Incremented on + // enqueue, decremented on consume. Must use +=/-= (not recomputation) to + // preserve IEEE 754 double-precision drift that WPTs verify. + #queueTotalSize: number = 0; + + constructor( + queue: StreamQueue, + owner: object, + startPosition: number = queue.length, + byteOffset: number = 0, + initialTotalSize: number = 0 + ) { + this.#queue = queue; + this.#owner = new WeakRef(owner); + this.#position = startPosition; + this.#byteOffset = byteOffset; + this.#queueTotalSize = initialTotalSize; + queue.addCursor(this, owner); + } + + get queue(): StreamQueue { + return this.#queue; + } + + get position(): number { + return this.#position; + } + + get byteOffset(): number { + return this.#byteOffset; + } + + get hasPendingRead(): boolean { + return this.#pendingReads.length > 0; + } + + fulfillFirstPendingRead(value: V): void { + const pending = ArrayPrototypeShift(this.#pendingReads) as PendingRead; + pending.resolve(createReadResult(value, false)); + } + + isOrphaned(): boolean { + return WeakRefPrototypeDeref(this.#owner) === undefined; + } + + // The owning stream, if it is still alive. Used by the controller to + // propagate error transitions to every consumer stream (tee branches) + // without holding strong references to them. + ownerDeref(): object | undefined { + return WeakRefPrototypeDeref(this.#owner) as object | undefined; + } + + // Spec [[queueTotalSize]]: running total of unconsumed entry sizes. + // Uses += / -= to match IEEE 754 drift that WPTs verify. + get remainingSize(): number { + // Clamp to 0 per spec (ResetQueue, EnqueueValueWithSize clamping). + const total = this.#queueTotalSize; + return total < 0 ? 0 : total; + } + + // Called by StreamQueue.enqueue() to increment the running total. + addToTotalSize(size: number): void { + this.#queueTotalSize += size; + } + + // Produce the read-result value for `entry` (which is at the current + // position). Base implementation: the whole entry value. ByteStreamCursor + // overrides this to build a remainder view honoring #byteOffset. + protected readEntryValue(entry: QueueEntry): V { + return entry.value as unknown as V; + } + + // Advance past the entry at the current position (whole-entry consume). + protected advancePastEntry(): void { + const slot = this.#queue.getEntry(this.#position); + if (isQueueEntry(slot)) { + // Only the REMAINING portion of this entry contributes to the + // running total — bytes before #byteOffset were already debited + // (by setConsumed or via initialTotalSize at cursor construction). + this.#queueTotalSize -= slot.size - this.#byteOffset; + } + this.#position++; + this.#byteOffset = 0; + } + + // Used by byte fills for sub-entry consumption tracking. + protected setConsumed(position: number, byteOffset: number): void { + if (position === this.#position && byteOffset === this.#byteOffset) { + return; + } + // Adjust running total for consumed entries and byte offset changes. + // The entry at the CURRENT position may be partially consumed + // (#byteOffset > 0) — those bytes were already debited, so only + // the remainder counts. Subsequent entries are fully outstanding. + for (let i = this.#position; i < position; i++) { + const slot = this.#queue.getEntry(i); + if (isQueueEntry(slot)) { + this.#queueTotalSize -= + i === this.#position ? slot.size - this.#byteOffset : slot.size; + } + } + // When staying on the same entry (position unchanged), only the + // DELTA from old to new offset is freshly consumed. When advancing + // past entries, the new entry's offset is all-new consumption. + const alreadyDebited = position === this.#position ? this.#byteOffset : 0; + this.#queueTotalSize -= byteOffset - alreadyDebited; + this.#position = position; + this.#byteOffset = byteOffset; + this.#queue.onCursorAdvanced(); + } + + // Submit a read. If data is available AND no reads are already pending, + // fulfill immediately; otherwise defer. + // + // INVARIANT (per-reader FIFO): the fast path is taken only when + // #pendingReads is empty. Entries CAN coexist with pending reads (e.g. + // while notification is deferred during a batched enqueue), and a fresh + // read must never jump ahead of older ones. + // + // Ordering: spec PullSteps dequeues, then calls CallPullIfNeeded, then + // fulfills the read request. We preserve that order (advance → + // onCursorAdvanced, which may trigger a pull → resolve) so the relative + // microtask ordering of pull side-effects vs. read fulfillment matches + // ordering-sensitive WPT tests. + // Attempt a synchronous read. Returns the result directly when data (or + // the close sentinel) is immediately available at the cursor, or + // undefined when no data is buffered / reads are already queued. This + // lets the reader layer perform the drain-then-close check without an + // intervening microtask (spec PullSteps timing). + tryReadSync(_reader: object): ReadableStreamReadResult | undefined { + if (this.#pendingReads.length !== 0) return undefined; + const slot = this.#queue.getEntry(this.#position); + if (slot === CLOSE_SENTINEL) { + return createReadResult(undefined, true); + } + if (isQueueEntry(slot)) { + const value = this.readEntryValue(slot); + this.advancePastEntry(); + this.#queue.onCursorAdvanced(); + return createReadResult(value, false); + } + return undefined; + } + + read(reader: object): Promise> { + // Fast path: try the synchronous read first to avoid wrapping in a + // promise when data is already buffered. + const sync = this.tryReadSync(reader); + if (sync !== undefined) return PromiseResolve(sync); + // No data available (or reads already queued) — defer. + const { promise, resolve, reject } = + PromiseWithResolvers() as PromiseWithResolversType< + ReadableStreamReadResult + >; + ArrayPrototypePush(this.#pendingReads, { resolve, reject, reader }); + return promise; + } + + // Called by the queue when new data (or the close sentinel) is enqueued. + notify(): void { + while (this.#pendingReads.length > 0) { + const slot = this.#queue.getEntry(this.#position); + if (slot === undefined) break; + if (slot === CLOSE_SENTINEL) { + // End of stream: every remaining pending read resolves done. This + // deliberately uses the non-virtual helper — the byte cursor's + // pending pull-intos must NOT be auto-committed at the sentinel + // (see "Close semantics for byte cursors" in the design doc). + this.#resolvePendingReadsAsDone(); + break; + } + const entry = slot as QueueEntry; + const value = this.readEntryValue(entry); + this.advancePastEntry(); + // assert: pendingReads is non-empty (the while condition guarantees it) + const pending = ArrayPrototypeShift(this.#pendingReads) as PendingRead; + pending.resolve(createReadResult(value, false)); + } + this.#queue.onCursorAdvanced(); + } + + // Reject pending reads submitted by a specific reader (lock release). + cancelReadsForReader(reader: object, reason: unknown): void { + const remaining: PendingRead[] = []; + const pending = this.#pendingReads; + this.#pendingReads = remaining; + for (let i = 0; i < pending.length; i++) { + const read = pending[i] as PendingRead; + if (read.reader === reader) { + read.reject(reason); + } else { + ArrayPrototypePush(remaining, read); + } + } + } + + // Reject all pending reads (stream ERROR path only). + errorAllReads(reason: unknown): void { + this.#queueTotalSize = 0; + const pending = this.#pendingReads; + this.#pendingReads = []; + for (let i = 0; i < pending.length; i++) { + const read = pending[i] as PendingRead; + read.reject(reason); + } + } + + // Resolve all pending reads as done (stream CANCEL path). Per spec, + // cancel() RESOLVES pending reads with { done: true, value: undefined } — + // including BYOB reads, whose partial data is dropped. Rejection is + // reserved for error() and releaseLock(). + resolveAllReadsAsDone(): void { + this.#queueTotalSize = 0; + this.#resolvePendingReadsAsDone(); + } + + // Stream-cancel teardown (StreamConsumer interface). QUEUED INVARIANT: + // the source-cancel decision runs BEFORE removeCursor so a + // reason-carrying cancel wins the controller's idempotency cache over + // the all-cursors-gone GC hook's undefined-reason call. The stream layer + // owns the policy (tee composite hooks vs direct controller cancel) via + // `decideSourceCancel` — including binding the reason — while the + // last-consumer determination is queue knowledge, supplied to it here. + cancelStream( + _reason: unknown, + decideSourceCancel: (isLastConsumer: boolean) => Promise + ): Promise { + this.resolveAllReadsAsDone(); + const promise = decideSourceCancel(this.#queue.cursorCount === 1); + this.#queue.removeCursor(this); + return promise; + } + + #resolvePendingReadsAsDone(): void { + const pending = this.#pendingReads; + this.#pendingReads = []; + for (let i = 0; i < pending.length; i++) { + const read = pending[i] as PendingRead; + read.resolve(createReadResult(undefined, true)); + } + } + + // Bulk read for the draining reader (and pipeTo): consume all buffered + // entries from the current position, up to the soft limit `maxSize` + // (in strategy size units — bytes for byte streams). Returns + // synchronously; the draining reader wraps in a promise at its API + // boundary. Always takes at least one available entry so callers make + // progress even when maxSize is smaller than the next entry. + drain(maxSize: number = Infinity): { chunks: V[]; done: boolean } { + const chunks: V[] = []; + let total = 0; + let done = false; + while (total < maxSize) { + const slot = this.#queue.getEntry(this.#position); + if (slot === undefined) break; + if (slot === CLOSE_SENTINEL) { + done = true; + break; + } + const entry = slot as QueueEntry; + ArrayPrototypePush(chunks, this.readEntryValue(entry)); + total += entry.size; + this.advancePastEntry(); + } + if (chunks.length > 0) { + this.#queue.onCursorAdvanced(); + } + const result = ObjectCreate(null) as { chunks: V[]; done: boolean }; + result.chunks = chunks; + result.done = done; + return result; + } +} + +// --------------------------------------------------------------------------- +// ByteStreamCursor + +// Byte cursors add sub-entry granularity (partial entry consumption) and +// BYOB support via spec-shaped pull-into descriptors. Entry values are +// {buffer, byteOffset, byteLength} triples; read results are Uint8Array +// remainder views. +class ByteStreamCursor + extends QueueCursor + implements ByteStreamConsumer +{ + // Spec [[pendingPullIntos]] — a LIST. Multiple reads can be queued, and + // autoAllocateChunkSize creates synthetic descriptors for default reads. + #pendingPullIntos: PullIntoDescriptor[] = []; + + // Callback invoked when the cursor detects a fractional-element fill at + // the close sentinel — the stream must be errored with a TypeError. Set + // by the controller (the cursor layer cannot error the stream directly). + #errorStreamCallback: ((e: unknown) => void) | undefined; + + get hasPendingPullInto(): boolean { + return this.#pendingPullIntos.length > 0; + } + + override get hasPendingRead(): boolean { + if (super.hasPendingRead) return true; + // Descriptors with readerType 'none' are leftovers from releaseLock — + // they don't represent active reads and should NOT trigger pull(). + for (let i = 0; i < this.#pendingPullIntos.length; i++) { + if ( + (this.#pendingPullIntos[i] as PullIntoDescriptor).readerType !== 'none' + ) { + return true; + } + } + return false; + } + + get hasPartiallyFulfilledRead(): boolean { + const head = this.#pendingPullIntos[0]; + return head !== undefined && head.bytesFilled > 0; + } + + // The head pull-into descriptor, for the controller's respond() / + // respondWithNewView() paths (validation, buffer re-transfer and + // re-pointing happen there, at the trust boundary). + get headPullInto(): PullIntoDescriptor | undefined { + return this.#pendingPullIntos[0]; + } + + // View over the unfilled remainder of the head descriptor — what + // byobRequest.view exposes to the underlying source. + get pendingPullIntoView(): Uint8Array | undefined { + const head = this.#pendingPullIntos[0]; + if (head === undefined) return undefined; + return new Uint8Array( + head.buffer, + head.byteOffset + head.bytesFilled, + head.byteLength - head.bytesFilled + ); + } + + // Set the callback the controller uses to receive fractional-element- + // at-close errors (the cursor cannot error the stream directly). + set errorStreamCallback(cb: (e: unknown) => void) { + this.#errorStreamCallback = cb; + } + + // Default reads on a byte cursor return the REMAINDER of the entry at the + // cursor's position: a view over the original (transferred) entry buffer + // starting at the current byteOffset (whole entry when byteOffset is 0). + // + // COPY-ON-READ for shared entries: if any OTHER live cursor still needs + // this entry, hand out a copy — aliased mutable views across tee branches + // would let one consumer corrupt its sibling's data. The last consumer + // (and the single-cursor common case) takes the view zero-copy. This is + // the exact last-consumer test from the design doc; readEntryValue is + // always invoked for the entry at the CURRENT position, before advancing. + protected override readEntryValue( + entry: QueueEntry + ): Uint8Array { + const v = entry.value; + const view = new Uint8Array( + v.buffer, + v.byteOffset + this.byteOffset, + v.byteLength - this.byteOffset + ); + if (this.queue.hasOtherLiveCursorAtOrBefore(this, this.position)) { + return TypedArrayPrototypeSlice(view) as Uint8Array; + } + return view; + } + + // Called by the BYOB reader after validation, buffer transfer, and + // descriptor construction. Same FIFO invariant as read(): the fast path + // is taken only when nothing is already pending. + // + // PRECONDITION: the stream is readable. Unlike read(), the cursor does + // not resolve BYOB reads at the sentinel — a read submitted after close + // must be resolved by the READER layer with { done: true, value: + // zero-length view over the transferred buffer } before ever reaching + // here, and errored streams reject at the reader layer. A descriptor + // pushed here while the queue is already closed would pend until + // respond(0)/cancel/release, per the byte-cursor close matrix. + readBYOB( + desc: PullIntoDescriptor + ): Promise> { + if (this.#pendingPullIntos.length === 0) { + this.#fillFromQueue(desc); + if (desc.bytesFilled >= desc.minimumFill) { + return PromiseResolve(createReadResult(this.#convert(desc), false)); + } + // After filling, if the cursor is at the close sentinel and the + // descriptor has a fractional element fill, the remaining bytes can + // never complete an element — error the stream immediately. + if ( + desc.bytesFilled > 0 && + desc.bytesFilled % desc.elementSize !== 0 && + this.queue.getEntry(this.position) === CLOSE_SENTINEL + ) { + const e = new TypeError( + 'Insufficient bytes to fill elements in the given view' + ); + if (this.#errorStreamCallback !== undefined) { + this.#errorStreamCallback(e); + } + return PromiseReject(e); + } + } + ArrayPrototypePush(this.#pendingPullIntos, desc); + return desc.promise; + } + + // Keep filling head descriptors from newly queued data while they can be + // completed; then let the base class service default pending reads + // (including sentinel handling) and refresh backpressure. + override notify(): void { + // Two-phase processing per spec + // ReadableByteStreamControllerProcessPullIntoDescriptorsUsingQueue: + // fill ALL ready descriptors first, THEN resolve them. This ensures + // byobRequest is null by the time any resolve fires (which may trigger + // user-observable code via Object.prototype.then interception). + let filledPullIntos: + | Array<{ desc: PullIntoDescriptor; view: ArrayBufferView }> + | undefined; + while (this.#pendingPullIntos.length > 0) { + const slot = this.queue.getEntry(this.position); + if (slot === undefined) break; + if (slot === CLOSE_SENTINEL) { + // End of DATA. Check for fractional-element fill: if any BYOB + // descriptor has partially filled bytes that don't align to the + // element size, the remaining bytes can never complete an element + // — the stream must be errored with a TypeError (spec + // ReadableByteStreamControllerClose step 4). + if (this.#checkFractionalFillAtClose()) return; + // Synthetic descriptors for DEFAULT reads (autoAllocateChunkSize) + // follow default-read close semantics: ReadableStreamClose drains + // read requests with done, so they resolve { done: true } now. + this.#resolveDefaultPullIntosAsDone(); + // TRUE BYOB descriptors in single-cursor mode are NOT committed + // here — the source can call respond(0)-while-closed, which + // reaches commitPullIntosOnClose() via the controller. But in + // multi-cursor mode (tee branches), byobRequest is null and + // respond(0) is unreachable, so we must commit them now. This + // is the equivalent of the spec's per-branch controller running + // ReadableByteStreamControllerRespondInClosedState. + if (this.queue.singleCursor === undefined) { + this.commitPullIntosOnClose(); + } + break; + } + const head = this.#pendingPullIntos[0] as PullIntoDescriptor; + this.#fillFromQueue(head); + if (head.bytesFilled < head.minimumFill) break; // need more data + ArrayPrototypeShift(this.#pendingPullIntos); + const view = this.#convert(head); + if (filledPullIntos === undefined) filledPullIntos = []; + ArrayPrototypePush(filledPullIntos, { desc: head, view }); + } + // Phase 2: resolve all filled descriptors after the fill loop. + if (filledPullIntos !== undefined) { + for (let i = 0; i < filledPullIntos.length; i++) { + const filled = filledPullIntos[i] as { + desc: PullIntoDescriptor; + view: ArrayBufferView; + }; + filled.desc.resolve(createReadResult(filled.view, false)); + } + } + super.notify(); + } + + // Spec ReadableByteStreamControllerClose step 4: if any pending BYOB + // descriptor has a fractional element fill (bytesFilled > 0 but not + // element-aligned), the remaining bytes can never form a complete element + // — error the stream with TypeError and reject all pending reads. Returns + // true if the stream was errored (caller should bail out of notify). + #checkFractionalFillAtClose(): boolean { + for (let i = 0; i < this.#pendingPullIntos.length; i++) { + const desc = this.#pendingPullIntos[i] as PullIntoDescriptor; + if (desc.bytesFilled > 0 && desc.bytesFilled % desc.elementSize !== 0) { + const e = new TypeError( + 'Insufficient bytes to fill elements in the given view' + ); + if (this.#errorStreamCallback !== undefined) { + this.#errorStreamCallback(e); + } + // errorAllReads is called by the controller's error() path + // (via the stream error machinery), so we don't call it here. + return true; + } + } + return false; + } + + // Resolve synthetic default-read descriptors (autoAllocateChunkSize) as + // done at end-of-stream; keep true BYOB descriptors pending per the close + // matrix. In practice the list is homogeneous (single reader type at a + // time), so the filtering is defensive. + // + // The descriptor is KEPT in the list (not shifted) so that a subsequent + // respond(0) finds it and doesn't throw. commitPullIntosOnClose skips + // already-resolved descriptors. + #resolveDefaultPullIntosAsDone(): void { + for (let i = 0; i < this.#pendingPullIntos.length; i++) { + const desc = this.#pendingPullIntos[i] as PullIntoDescriptor; + if (desc.readerType === 'default') { + desc.resolve(createReadResult(undefined, true)); + desc.readerType = 'none'; // mark as consumed + } + } + } + + // byobRequest.respond(bytesWritten) — zero-copy path. The controller has + // already validated bytesWritten, re-transferred the buffer, and + // re-pointed head.buffer at the transferred copy. + respondBYOB(bytesWritten: number): void { + const head = this.#pendingPullIntos[0]; + if (head === undefined) { + throw new TypeError('No pending BYOB request to respond to'); + } + head.bytesFilled += bytesWritten; + + // Spec ReadableByteStreamControllerRespondInReadableState step 3: + // if the head descriptor's reader was released (readerType 'none'), + // the filled data is cloned into the queue rather than resolving a read + // promise. Subsequent descriptors are then filled from the queue. + // Shift BEFORE enqueue to avoid re-entrant notify filling the same head. + if (head.readerType === 'none') { + ArrayPrototypeShift(this.#pendingPullIntos); + if (head.bytesFilled > 0) { + // Clone (not transfer) the filled portion into a new queue entry. + // enqueue triggers notify() which fills subsequent descriptors. + this.queue.enqueue({ + value: { + buffer: ArrayBufferPrototypeSlice( + head.buffer, + head.byteOffset, + head.byteOffset + head.bytesFilled + ), + byteOffset: 0, + byteLength: head.bytesFilled, + }, + size: head.bytesFilled, + }); + } + return; + } + + if (head.bytesFilled < head.minimumFill) { + // Not enough yet — stays pending. byobRequest now exposes a view + // over the unfilled remainder of the (new) buffer; the source can + // keep writing or fall back to enqueue(). + return; + } + // Spec ReadableByteStreamControllerRespondInReadableState step 7–10: + // Remove from pending FIRST, then split remainder and enqueue. + // Order matters: enqueue triggers notify() on live cursors, and the + // head must already be gone to avoid re-entrant filling. + ArrayPrototypeShift(this.#pendingPullIntos); + const remainderSize = head.bytesFilled % head.elementSize; + if (remainderSize > 0) { + // The remainder bytes live at the END of the filled region. + const end = head.byteOffset + head.bytesFilled; + // Enqueue the remainder as a new queue entry (a slice of the + // transferred buffer). The spec uses CloneArrayBuffer here; we use + // slice since the buffer is already the transferred copy. + this.queue.enqueue({ + value: { + buffer: ArrayBufferPrototypeSlice( + head.buffer, + end - remainderSize, + end + ), + byteOffset: 0, + byteLength: remainderSize, + }, + size: remainderSize, + }); + // Truncate bytesFilled to an element-aligned boundary. + head.bytesFilled -= remainderSize; + } + head.resolve(createReadResult(this.#convert(head), false)); + // Spec ProcessPullIntosUsingQueue: data queued via the enqueue path may + // already satisfy subsequent descriptors — fill them now rather than + // waiting for the next enqueue. notify() runs exactly that loop (and + // its default-read/backpressure follow-ups are no-ops here). + this.notify(); + } + + // Commit all pending pull-into descriptors at end-of-stream: resolve with + // the filled-so-far view (possibly zero-length — the buffer is handed + // back) and done: true in a SINGLE result. Called by the controller from + // the respond(0)-while-closed path. A fractional-element fill never + // reaches this point: controller.close() throws for it. + commitPullIntosOnClose(): void { + const pending = this.#pendingPullIntos; + this.#pendingPullIntos = []; + for (let i = 0; i < pending.length; i++) { + const desc = pending[i] as PullIntoDescriptor; + // Skip already-resolved descriptors (auto-allocate close path set + // readerType to 'none' after resolving the read promise). + if (desc.readerType === 'none') continue; + // assert: desc.bytesFilled % desc.elementSize === 0 + desc.resolve(createReadResult(this.#convert(desc), true)); + } + } + + // Spec ReadableByteStreamControllerEnqueue step 8.5: if the head + // pending pull-into has readerType 'none' (leftover from releaseLock), + // transfer its buffer and enqueue any filled data before the new chunk + // is added. This ensures the released descriptor is processed eagerly. + drainNoneDescriptors(): void { + while (this.#pendingPullIntos.length > 0) { + const head = this.#pendingPullIntos[0] as PullIntoDescriptor; + if (head.readerType !== 'none') break; + ArrayPrototypeShift(this.#pendingPullIntos); + if (head.bytesFilled > 0) { + // Clone the filled portion into a new queue entry. + this.queue.enqueue({ + value: { + buffer: ArrayBufferPrototypeSlice( + head.buffer, + head.byteOffset, + head.byteOffset + head.bytesFilled + ), + byteOffset: 0, + byteLength: head.bytesFilled, + }, + size: head.bytesFilled, + }); + } + } + } + + // Spec ReadableByteStreamControllerEnqueue step 9.3: if the head + // descriptor is an auto-allocate (readerType 'default'), shift it out + // so the controller can fulfill the read directly from the enqueued + // chunk (bypassing the queue and the auto-allocate buffer). + shiftAutoAllocateDescriptor(): PullIntoDescriptor | undefined { + if (this.#pendingPullIntos.length === 0) return undefined; + const head = this.#pendingPullIntos[0] as PullIntoDescriptor; + if (head.readerType !== 'default') return undefined; + ArrayPrototypeShift(this.#pendingPullIntos); + return head; + } + + // Stream error: partial fills are lost. + override errorAllReads(reason: unknown): void { + const pending = this.#pendingPullIntos; + this.#pendingPullIntos = []; + for (let i = 0; i < pending.length; i++) { + const desc = pending[i] as PullIntoDescriptor; + desc.reject(reason); + } + super.errorAllReads(reason); + } + + // Stream cancel: pending BYOB reads resolve { done: true, value: + // undefined } — partial data and buffers are dropped (spec, + // WPT-verified). + override resolveAllReadsAsDone(): void { + const pending = this.#pendingPullIntos; + this.#pendingPullIntos = []; + for (let i = 0; i < pending.length; i++) { + const desc = pending[i] as PullIntoDescriptor; + desc.resolve(createReadResult(undefined, true)); + } + super.resolveAllReadsAsDone(); + } + + // Reject pull-intos submitted by a specific reader (lock release). + override cancelReadsForReader(reader: object, reason: unknown): void { + // Spec: on releaseLock, the reader's readIntoRequests are rejected, but + // the controller's pendingPullIntos STAY. Descriptors whose reader is + // being released have their readerType set to 'none' — respond() will + // then enqueue the filled data instead of resolving a read promise. + // The byobRequest is NOT invalidated (it still points at the head + // descriptor's buffer). + for (let i = 0; i < this.#pendingPullIntos.length; i++) { + const desc = this.#pendingPullIntos[i] as PullIntoDescriptor; + if (desc.reader === reader) { + desc.reject(reason); + desc.readerType = 'none'; + } + } + super.cancelReadsForReader(reader, reason); + } + + // The ONLY place result views are built. Construction count is in + // ELEMENTS (bytesFilled / elementSize) — passing bytes happens to work + // for Uint8Array and is wrong for every other view type. For DataView, + // elementSize is 1 and its length parameter is in bytes, so the same + // expression holds. + #convert(desc: PullIntoDescriptor): ArrayBufferView { + // assert: desc.bytesFilled % desc.elementSize === 0 + // assert: desc.bytesFilled <= desc.byteLength + return ReflectConstruct(desc.viewCtor, [ + desc.buffer, + desc.byteOffset, + desc.bytesFilled / desc.elementSize, + ]); + } + + // Total bytes available to this cursor in the queue (up to the sentinel), + // accounting for the partially consumed current entry. + #availableBytes(): number { + let available = 0; + for (let i = this.position; i < this.queue.length; i++) { + const slot = this.queue.getEntry(i); + if (!isQueueEntry(slot)) break; // sentinel is always last + available += slot.value.byteLength; + } + return available - this.byteOffset; + } + + // Fill `desc` from the queue starting at (position, byteOffset). Mirrors + // ReadableByteStreamControllerFillPullIntoDescriptorFromQueue: + // - if queued data reaches an element-aligned boundary >= minimumFill, + // copy only up to that aligned boundary (remainder bytes stay queued + // for the next read) — descriptor is then ready; + // - otherwise copy everything available (the descriptor may temporarily + // end mid-element) and stay pending. + // Stops at the CLOSE_SENTINEL. Advances position/byteOffset for consumed + // bytes (which triggers GC + backpressure refresh via setConsumed). + #fillFromQueue(desc: PullIntoDescriptor): void { + const available = this.#availableBytes(); + if (available <= 0) return; + + const maxBytesToCopy = MathMin( + available, + desc.byteLength - desc.bytesFilled + ); + const maxBytesFilled = desc.bytesFilled + maxBytesToCopy; + const maxAlignedBytes = + maxBytesFilled - (maxBytesFilled % desc.elementSize); + let remaining = + maxAlignedBytes >= desc.minimumFill + ? maxAlignedBytes - desc.bytesFilled + : maxBytesToCopy; + + let pos = this.position; + let off = this.byteOffset; + while (remaining > 0) { + // Guaranteed to be a data entry by the #availableBytes computation. + const entry = (this.queue.getEntry(pos) as QueueEntry) + .value; + const n = MathMin(entry.byteLength - off, remaining); + const dest = new Uint8Array( + desc.buffer, + desc.byteOffset + desc.bytesFilled, + n + ); + const src = new Uint8Array(entry.buffer, entry.byteOffset + off, n); + TypedArrayPrototypeSet(dest, src); + desc.bytesFilled += n; + remaining -= n; + off += n; + if (off === entry.byteLength) { + pos++; + off = 0; + } + } + this.setConsumed(pos, off); + } +} + +// Type-only exports (fully erased at runtime — the loader sees only the +// module.exports assignment below, matching the readable.ts pattern). +export type { StreamQueue, QueueCursor, ByteStreamCursor }; + +module.exports = { + CLOSE_SENTINEL, + createReadResult, + StreamQueue, + QueueCursor, + ByteStreamCursor, +}; diff --git a/src/per_isolate/webstreams/readable.ts b/src/per_isolate/webstreams/readable.ts new file mode 100644 index 00000000000..17cc9b44b2a --- /dev/null +++ b/src/per_isolate/webstreams/readable.ts @@ -0,0 +1,3668 @@ +'use strict'; + +import type { + PromiseWithResolvers as PromiseWithResolversType, + QueuingStrategy, + ReadableByteStreamController as ReadableByteStreamControllerType, + ReadableStream as ReadableStreamType, + ReadableStreamBYOBReaderReadOptions, + ReadableStreamBYOBReader as ReadableStreamBYOBReaderType, + ReadableStreamBYOBRequest as ReadableStreamBYOBRequestType, + ReadableStreamDefaultController as ReadableStreamDefaultControllerType, + ReadableStreamDefaultReader as ReadableStreamDefaultReaderType, + ReadableStreamReader as ReadableStreamReaderType, + ReadableStreamReadResult, + StreamPipeOptions, + TransformStream as TransformStreamType, + UnderlyingByteSource, + UnderlyingDefaultSource, + UnderlyingSource, + WritableStream as WritableStreamType, +} from './types'; +import type { + ByteQueueEntry, + ByteStreamConsumer as ByteStreamConsumerType, + ByteStreamCursor as ByteStreamCursorType, + PullIntoDescriptor, + QueueCursor as QueueCursorType, + StreamConsumer as StreamConsumerType, + StreamQueue as StreamQueueType, +} from './queue'; +import type { + NativeReadableStreamController as NativeReadableStreamControllerType, + NativeStreamInternals, +} from './native'; + +const { + AbortSignalAbortedGet, + AbortSignalReasonGet, + AggregateError, + ArrayBuffer, + ArrayBufferPrototypeByteLengthGet, + ArrayBufferPrototypeDetachedGet, + ArrayBufferPrototypeTransfer, + ArrayPrototypePush, + AsyncIteratorPrototype, + BigInt, + DataView, + DataViewPrototypeGetBuffer, + DataViewPrototypeGetByteLength, + DataViewPrototypeGetByteOffset, + EventTargetAddEventListener, + EventTargetRemoveEventListener, + NumberIsNaN, + ObjectCreate, + ObjectDefineProperty, + ObjectDefineProperties, + ObjectGetOwnPropertyDescriptor, + ObjectSetPrototypeOf, + PromiseResolve, + PromiseReject, + PromiseWithResolvers, + PromisePrototypeThen, + SafePromise, + RangeError, + SafeArrayIterator, + SafeWeakMap, + Symbol, + SymbolAsyncIterator, + SymbolIterator, + SymbolToStringTag, + TypeError, + TypedArrayCtorByName, + TypedArrayPrototypeGetBuffer, + TypedArrayPrototypeGetByteLength, + TypedArrayPrototypeGetByteOffset, + TypedArrayPrototypeGetLength, + TypedArrayPrototypeGetSymbolToStringTag, + Uint8Array, + uncurryThis, +} = primordials; + +// SafePromise.race is used by the pipe algorithm where species protection +// is needed for the race's internal .then() calls on its arguments. +const SafePromiseRace = SafePromise.race; + +const { isArrayBufferView, isPromise, markPromiseHandled } = utils; + +const { + StreamQueue, + QueueCursor, + ByteStreamCursor, + CLOSE_SENTINEL, + createReadResult, +} = require('webstreams/queue'); + +// Internal writable operations for the pipe (never re-exported to users). +const { + kExtractNativeSink, + internalsForPipe: writableInternals, +} = require('webstreams/writable'); + +// The native backend (leaf module — see the fence conventions in native.ts +// and queue.ts). The cast restores the real shape the untyped loader +// erases, so the brand predicates keep their type-guard narrowing. +const { nativeStreamInternals } = require('webstreams/native') as { + nativeStreamInternals: NativeStreamInternals; +}; +const { + kExtractNativeSource, + isNativeUnderlyingSource, + isNativeController, + createNativeReadableStreamParts, + nativeControllerPullIfNeeded, + nativeControllerCancelSteps, + nativeControllerMaybeCloseStream, + nativeControllerOnReaderRelease, + nativeControllerTeeSource, + nativeControllerExtractSource, + nativeControllerExpectedLength, +} = nativeStreamInternals; + +// Normalizes the non-standard `expectedLength` extension property on +// byte-stream underlying sources: the TOTAL bytes the source promises to +// produce (undefined = unknown; the C++ side then uses chunked +// encoding). Accepts a non-negative bigint or non-negative integer +// number (normalized to bigint — totals can exceed MAX_SAFE_INTEGER). +// Only meaningful for type:'bytes' sources; the default controller never +// reads it. Duplicated for the native backend in native.ts. +function normalizeExpectedLength(value: unknown): bigint | undefined { + if (value === undefined) return undefined; + if (typeof value === 'bigint') { + if (value < 0n) { + throw new RangeError('expectedLength must be non-negative'); + } + return value; + } + if (typeof value === 'number') { + if (NumberIsNaN(value) || value % 1 !== 0) { + throw new TypeError('expectedLength must be an integer'); + } + if (value < 0) { + throw new RangeError('expectedLength must be non-negative'); + } + return BigInt(value); + } + throw new TypeError( + 'expectedLength must be a non-negative bigint or integer' + ); +} + +// --- Composite tee-cancel reasons ------------------------------------------ +// DELIBERATE SPEC DIVERGENCE (recorded in the design doc's divergence +// table): when all tee branches have cancelled, the source's cancel +// algorithm receives a single flattened AggregateError carrying every +// branch's reason, rather than the spec's two-element array. Re-tee +// composites are flattened into one level: the WeakMap below brands OUR +// composite errors (unforgeably) and carries their flat reason list — a +// user-supplied AggregateError used as a cancel reason is never unpacked. +const teeCompositeReasons = new SafeWeakMap(); + +function appendCancelReason(flat: unknown[], reason: unknown): void { + // WeakMap.get on a non-object key returns undefined (no throw). + const nested = teeCompositeReasons.get(reason) as unknown[] | undefined; + if (nested !== undefined) { + for (let i = 0; i < nested.length; i++) { + ArrayPrototypePush(flat, nested[i]); + } + } else { + ArrayPrototypePush(flat, reason); + } +} + +function makeCompositeCancelReason( + reason1: unknown, + reason2: unknown +): AggregateError { + const flat: unknown[] = []; + appendCancelReason(flat, reason1); + appendCancelReason(flat, reason2); + // SafeArrayIterator: AggregateError's iterable conversion must not run + // through the patchable %ArrayIteratorPrototype%. + const error = new AggregateError( + new SafeArrayIterator(flat), + 'All readable stream tee branches were canceled' + ) as AggregateError; + teeCompositeReasons.set(error, flat); + return error; +} + +const kPrivateSymbol = Symbol('private'); + +function isActualObject(value: unknown) { + return value != null && typeof value === 'object'; +} + +function assertPrivateSymbol(symbol: symbol) { + if (symbol !== kPrivateSymbol) { + throw new TypeError('Illegal constructor'); + } +} + +let isReadableStreamLocked: (stream: ReadableStream) => boolean; +// Accessor to extract the encapsulated ReadableStreamReaderBase from any +// outer reader (DefaultReader, BYOBReader, DrainingReader). Assigned in +// each outer reader's static {} block. +let getReaderBase: (reader: object) => ReadableStreamReaderBase; +let isReaderBoundToStream: (reader: object) => boolean; +let acquireReadableStreamDefaultReader: ( + stream: ReadableStream +) => ReadableStreamDefaultReader; +let initializeReadableStreamGenericReader: ( + stream: ReadableStream, + reader: ReadableStreamReaderBase +) => void; +let cancelReadableStreamGenericReader: ( + reader: object, + reason?: unknown +) => Promise; +let acquireReadableStreamBYOBReader: ( + stream: ReadableStream +) => ReadableStreamBYOBReaderType; +let readableStreamCancel: ( + stream: ReadableStream, + reason?: unknown +) => Promise; +let readableStreamPipeThroughTo: ( + source: ReadableStream, + destination: WritableStreamType, + options?: StreamPipeOptions +) => Promise; +let readableStreamTee: ( + stream: ReadableStream +) => [ReadableStream, ReadableStream]; +let readableStreamReaderGenericCancel: ( + reader: object, + reason?: unknown +) => Promise; +let readableStreamReaderGenericRelease: (reader: object) => void; +let readableStreamDefaultReaderRead: ( + reader: ReadableStreamDefaultReaderType, + readRequest: ReadableStreamAsyncIteratorReadRequest +) => void; +let isReadableStream: (value: unknown) => boolean; +let isByteStreamController: (value: unknown) => boolean; + +// BACKEND-DISPATCH: the byte-CAPABLE gate (one of the five sanctioned +// dispatch points). True for any controller whose backend can satisfy +// BYOB reads: the queued byte controller, or ANY native controller — +// native sources are byte-capable by definition (the marker is +// sufficient). Distinct from isByteStreamController, which remains the +// QUEUED-byte brand check used by the queued-only paths (autoAllocate +// synthesis, tee's cursor fork). +function isByteCapableController(value: unknown): boolean { + return isByteStreamController(value) || isNativeController(value); +} + +let getReadableStreamController: ( + stream: ReadableStream +) => + | ReadableStreamDefaultControllerType + | ReadableByteStreamControllerType + | NativeReadableStreamControllerType + | undefined; +let getReadableStreamReader: ( + stream: ReadableStream +) => ReadableStreamReaderType | undefined; +// The stream OWNS its consumer (readers only borrow it while locked; it +// persists across reader attach/detach). Created during controller setup. +// The consumer is the FENCE between backends: a QueueCursor/ +// ByteStreamCursor (queued) today, a NativePullConduit (native) later. +// The reader layer must stay backend-blind — it programs against +// StreamConsumer only; backend-specific access (cursor position/queue for +// tee, the controllers' close checks) happens behind sanctioned casts at +// the enumerated dispatch points. +let getReadableStreamConsumer: ( + stream: ReadableStream +) => StreamConsumerType | undefined; +let setReadableStreamConsumer: ( + stream: ReadableStream, + consumer: StreamConsumerType | undefined +) => void; +// Controller internals exposed for the reader/stream layers. Each takes the +// full controller union; the implementations are CHAINED (see the +// BACKEND-DISPATCH note at the default controller's static block): queued +// default assigns first, queued byte wraps it, and the native backend's +// wrap is joined at the bottom of this module (its brand lives across the +// module fence in native.ts). +let controllerPullIfNeeded: ( + controller: + | ReadableStreamDefaultControllerType + | ReadableByteStreamControllerType + | NativeReadableStreamControllerType +) => void; +let controllerCancelSteps: ( + controller: + | ReadableStreamDefaultControllerType + | ReadableByteStreamControllerType + | NativeReadableStreamControllerType, + reason: unknown +) => Promise; +let controllerMaybeCloseStream: ( + controller: + | ReadableStreamDefaultControllerType + | ReadableByteStreamControllerType + | NativeReadableStreamControllerType +) => void; +// Invalidate any cached byobRequest when a reader releases its lock (the +// release rejects the pending pull-intos, so a cached request would point +// at a dead descriptor). No-op for default controllers. +let controllerOnReaderRelease: ( + controller: + | ReadableStreamDefaultControllerType + | ReadableByteStreamControllerType + | NativeReadableStreamControllerType +) => void; +let getReaderStream: (reader: object) => ReadableStream | undefined; + +// BACKEND-DISPATCH point #4: the shared extractor function installed +// on native-backed streams as kExtractNativeSource. Assigned in +// ReadableStream's static block (needs private-field access). +let extractNativeSource: (this: ReadableStream) => object; + +// The non-standard expectedLength pass-through for the DrainingReader +// (and the C++ bridge). Chained like the other controller helpers: +// default → undefined; queued byte → cached construction value; native → +// cached construction value (joined in ReadableStream's static block). +let getControllerExpectedLength: ( + controller: + | ReadableStreamDefaultControllerType + | ReadableByteStreamControllerType + | NativeReadableStreamControllerType +) => bigint | undefined; + +let getReadableStreamGetState: ( + stream: ReadableStream +) => 'readable' | 'closed' | 'errored'; +// @ts-expect-error +let getReadableStreamIsDisturbed: (stream: ReadableStream) => boolean; +let getReadableStreamStoredError: (stream: ReadableStream) => unknown; +let setReadableStreamState: ( + stream: ReadableStream, + state: 'readable' | 'closed' | 'errored' +) => void; +// Marks the stream disturbed (one-way — there is no un-disturb). +let setReadableStreamDisturbed: (stream: ReadableStream) => void; +let setReadableStreamStoredError: ( + stream: ReadableStream, + error: unknown +) => void; +let setReadableStreamReader: ( + stream: ReadableStream, + reader: ReadableStreamReaderType | undefined +) => void; +let getGenericReaderClosedPromise: (reader: object) => Promise; +let resolveGenericReaderPromise: (reader: object) => void; +let rejectGenericReaderPromise: (reader: object, reason?: unknown) => void; +// Tee-branch lifecycle: fire #onBranchSettled if set. Called from +// readableStreamClose/readableStreamError so the shared cancel-promise +// settles when the source closes/errors and at least one sibling was +// already cancelled. Assigned in ReadableStream's static block. +let notifyBranchSettled: (stream: ReadableStream) => void; +// Set the #onBranchSettled callback for tee branches. Assigned in +// ReadableStream's static block. +let setOnBranchSettled: ( + stream: ReadableStream, + cb: (() => void) | undefined +) => void; + +interface ReadableStreamIteratorState { + done: boolean; + current?: Promise> | undefined; +} + +class ReadableStreamAsyncIteratorReadRequest { + #reader: ReadableStreamDefaultReaderType; + #state: ReadableStreamIteratorState; + #promise: PromiseWithResolversType>; + constructor( + reader: ReadableStreamDefaultReaderType, + state: ReadableStreamIteratorState, + promise: PromiseWithResolversType> + ) { + this.#reader = reader; + this.#state = state; + this.#promise = promise; + } + + chunk(chunk: R) { + this.#state.current = undefined; + this.#promise.resolve(createReadResult(chunk, false)); + } + + close() { + this.#state.current = undefined; + this.#state.done = true; + readableStreamReaderGenericRelease(this.#reader); + this.#promise.resolve(createReadResult(undefined, true)); + } + + error(error: unknown) { + this.#state.current = undefined; + this.#state.done = true; + readableStreamReaderGenericRelease(this.#reader); + this.#promise.reject(error); + } +} + +// --- ReadableStream async iterator prototype (spec §4.2.4) ----------------- +// Shared prototype with next/return methods; per-instance state stored in a +// WeakMap. This matches the spec's ReadableStreamAsyncIteratorPrototype which +// sits between each iterator instance and %AsyncIteratorPrototype%. +interface IteratorInternalState { + reader: ReadableStreamDefaultReaderType; + preventCancel: boolean; + state: ReadableStreamIteratorState; + started: boolean; +} + +const iteratorStateMap = new SafeWeakMap() as WeakMap< + object, + IteratorInternalState +>; + +function getIteratorState(iter: object): IteratorInternalState { + const s = iteratorStateMap.get(iter); + if (s === undefined) { + throw new TypeError('Illegal invocation'); + } + return s; +} + +function iteratorNextSteps(s: IteratorInternalState) { + if (s.state.done) { + return PromiseResolve(createReadResult(undefined, true)); + } + if (!isReaderBoundToStream(s.reader)) { + throw new TypeError('The reader is not bound to a ReadableStream'); + } + const promise = PromiseWithResolvers(); + readableStreamDefaultReaderRead( + s.reader, + new ReadableStreamAsyncIteratorReadRequest(s.reader, s.state, promise) + ); + return promise.promise; +} + +async function iteratorReturnSteps(s: IteratorInternalState, value: unknown) { + if (s.state.done) { + return createReadResult(value, true); + } + s.state.done = true; + + if (!isReaderBoundToStream(s.reader)) { + throw new TypeError('The reader is not bound to a ReadableStream'); + } + + if (!s.preventCancel) { + const result = readableStreamReaderGenericCancel(s.reader, value); + readableStreamReaderGenericRelease(s.reader); + await result; + return createReadResult(value, true); + } + + readableStreamReaderGenericRelease(s.reader); + return createReadResult(value, true); +} + +const ReadableStreamAsyncIteratorPrototype = ObjectSetPrototypeOf( + { + next(this: object) { + const s = getIteratorState(this); + if (!s.started) { + s.state.current = PromiseResolve(); + s.started = true; + } + s.state.current = + s.state.current !== undefined + ? PromisePrototypeThen( + s.state.current, + () => iteratorNextSteps(s), + () => iteratorNextSteps(s) + ) + : iteratorNextSteps(s); + return s.state.current; + }, + + return(this: object, error: unknown) { + const s = getIteratorState(this); + s.started = true; + s.state.current = + s.state.current !== undefined + ? PromisePrototypeThen( + s.state.current, + () => iteratorReturnSteps(s, error), + () => iteratorReturnSteps(s, error) + ) + : iteratorReturnSteps(s, error); + return s.state.current; + }, + + [SymbolAsyncIterator](this: object) { + return this; + }, + }, + AsyncIteratorPrototype +); +// ---- end async iterator prototype ------------------------------------------ + +class ReadableStreamReaderBase { + #stream?: ReadableStream | undefined; + // @ts-expect-error + #closedPromise: Promise | PromiseWithResolversType; + // NOTE: pending reads live on the stream's cursor (PendingRead entries + // keyed by reader identity), not on the reader — the cursor outlives + // reader attach/detach cycles. + + static { + initializeReadableStreamGenericReader = ( + stream: ReadableStream, + base: ReadableStreamReaderBase + ) => { + base.#stream = stream; + + switch (getReadableStreamGetState(stream)) { + case 'readable': { + base.#closedPromise = PromiseWithResolvers(); + markPromiseHandled( + (base.#closedPromise as PromiseWithResolversType).promise + ); + break; + } + case 'closed': { + base.#closedPromise = PromiseResolve(); + break; + } + case 'errored': { + const stored = getReadableStreamStoredError(stream); + // Because we need to mark the promise as handled, and because + // Promise.reject does not give us the ability to do so before + // the promise is reported as rejected, we need to create the + // promise with PromiseWithResolvers and reject it manually. + const promise: PromiseWithResolversType = + PromiseWithResolvers(); + markPromiseHandled(promise.promise); + promise.reject(stored); + base.#closedPromise = promise.promise; + break; + } + } + }; + + getGenericReaderClosedPromise = (reader: object) => { + const base = getReaderBase(reader); + const promise = base.#closedPromise; + if (isPromise(promise)) { + return promise as Promise; + } + if (!isPromise((promise as PromiseWithResolversType).promise)) { + throw new TypeError('invalid reader state'); + } + return (promise as PromiseWithResolversType).promise; + }; + + resolveGenericReaderPromise = (reader: object) => { + const base = getReaderBase(reader); + const promise = base.#closedPromise as PromiseWithResolversType; + const maybeResolve = promise.resolve; + if (typeof maybeResolve === 'function') { + maybeResolve(); + base.#closedPromise = promise.promise; + } + }; + + rejectGenericReaderPromise = (reader: object, reason?: unknown) => { + const base = getReaderBase(reader); + const promise = base.#closedPromise as PromiseWithResolversType; + const maybeReject = promise.reject; + if (typeof maybeReject === 'function') { + maybeReject(reason); + base.#closedPromise = promise.promise; + } + }; + + isReaderBoundToStream = (reader: object) => { + const base = getReaderBase(reader); + return base.#stream !== undefined; + }; + + cancelReadableStreamGenericReader = (reader: object, reason?: unknown) => { + const base = getReaderBase(reader); + const stream = base.#stream; + if (stream === undefined) { + return PromiseReject( + new TypeError('This reader has been released') + ) as Promise; + } + return readableStreamCancel(stream, reason); + }; + + readableStreamReaderGenericRelease = (reader: object) => { + const base = getReaderBase(reader); + const stream = base.#stream; + if (stream === undefined) return; + const releaseError = new TypeError('This reader has been released'); + // Per spec: a still-pending closedPromise is rejected; a settled one + // is replaced with a fresh rejected promise. Both are marked handled. + if (getReadableStreamGetState(stream) === 'readable') { + rejectGenericReaderPromise(reader, releaseError); + } else { + const replacement: PromiseWithResolversType = + PromiseWithResolvers(); + markPromiseHandled(replacement.promise); + replacement.reject(releaseError); + base.#closedPromise = replacement.promise; + } + // Reject THIS reader's pending reads. The consumer itself persists — + // it is stream-owned, and a future reader resumes where it left off. + const consumer = getReadableStreamConsumer(stream); + if (consumer !== undefined) { + consumer.cancelReadsForReader(reader, releaseError); + } + // Notify the controller that the reader was released. For byte + // controllers, the pull-into descriptors STAY (readerType → 'none'); + // the byobRequest is NOT invalidated per spec. + const controller = getReadableStreamController(stream); + if (controller !== undefined) { + controllerOnReaderRelease(controller); + } + setReadableStreamReader(stream, undefined); + base.#stream = undefined; + }; + + readableStreamReaderGenericCancel = (reader: object, reason?: unknown) => { + return cancelReadableStreamGenericReader(reader, reason); + }; + + getReaderStream = (reader: object) => { + const base = getReaderBase(reader); + return base.#stream; + }; + } +} + +// The default-read core, shared by ReadableStreamDefaultReader.read() and +// the async-iterator read path. Internal callers MUST use this rather than +// the public read() — reader prototypes end up user-reachable, so internal +// dispatch through them would be interceptable. +// +// BACKEND-BLIND: this function programs against StreamConsumer and must +// never branch on the backend. (The autoAllocate check below is the ONE +// sanctioned queued-byte-specific check — see the marker.) +function defaultReaderReadInternal( + reader: object, + stream: ReadableStream +): Promise> { + setReadableStreamDisturbed(stream); + const state = getReadableStreamGetState(stream); + if (state === 'closed') + return PromiseResolve(createReadResult(undefined, true)) as Promise< + ReadableStreamReadResult + >; + if (state === 'errored') + return PromiseReject(getReadableStreamStoredError(stream)) as Promise< + ReadableStreamReadResult + >; + const consumer = getReadableStreamConsumer(stream); + if (consumer === undefined) { + // The consumer was detached (cancelled/tee'd-away) — nothing to read. + return PromiseResolve(createReadResult(undefined, true)) as Promise< + ReadableStreamReadResult + >; + } + const controller = getReadableStreamController(stream); + + // --- Synchronous fast path (spec PullSteps) --- + // When data is immediately available the spec dequeues, performs the + // drain-then-close check, and fulfills the read request all in one + // synchronous call. An async/await on an already-resolved promise + // would insert a microtask gap between the dequeue and the close, + // causing reader.closed to resolve one tick too late. tryReadSync + // returns the result directly (no promise wrapping) so the close + // check runs in the same synchronous call. + // + // The autoAllocateChunkSize path is always async (BYOB machinery), + // so it skips this fast path. + let useAsyncPath = false; + if (controller !== undefined && isByteStreamController(controller)) { + const autoAllocateChunkSize = getByteControllerAutoAllocateChunkSize( + controller as ReadableByteStreamController + ); + if (autoAllocateChunkSize !== undefined) { + useAsyncPath = true; + } + } + if (!useAsyncPath) { + const syncResult = consumer.tryReadSync(reader) as + | ReadableStreamReadResult + | undefined; + if (syncResult !== undefined) { + // Spec PullSteps ordering: pull trigger, then drain-then-close, + // then fulfill the read request — all synchronous. + if (controller !== undefined) controllerPullIfNeeded(controller); + if (controller !== undefined) controllerMaybeCloseStream(controller); + if (syncResult.done) { + readableStreamClose(stream); + } + return PromiseResolve(syncResult); + } + } + + // --- Async fallback --- + // Data is not immediately available (pending reads queued, or native + // source, or autoAllocateChunkSize BYOB path). Fall through to the + // promise-based read and handle completion asynchronously. + return defaultReaderReadInternalAsync( + reader, + stream, + consumer, + controller, + useAsyncPath + ); +} + +// Async continuation of defaultReaderReadInternal for cases where +// the data is not synchronously available. +async function defaultReaderReadInternalAsync( + reader: object, + stream: ReadableStream, + consumer: StreamConsumerType, + controller: ReadableStreamDefaultControllerType | undefined, + isByteAutoAllocate: boolean +): Promise> { + let promise: Promise>; + if (isByteAutoAllocate) { + // QUEUED-BYTE-SPECIFIC (sanctioned exception to backend-blindness): + // autoAllocateChunkSize exists only on the queued byte controller — + // native sources are forbidden from declaring it (they allocate their + // own buffers for default reads), so a native stream correctly falls + // through to the plain consumer.read() below. + const autoAllocateChunkSize = getByteControllerAutoAllocateChunkSize( + controller as ReadableByteStreamController + ); + const withResolvers = PromiseWithResolvers() as PromiseWithResolversType< + ReadableStreamReadResult + >; + const descriptor: PullIntoDescriptor = { + buffer: new ArrayBuffer(autoAllocateChunkSize as number), + bufferByteLength: autoAllocateChunkSize as number, + byteOffset: 0, + byteLength: autoAllocateChunkSize as number, + bytesFilled: 0, + minimumFill: 1, + elementSize: 1, + viewCtor: Uint8Array, + readerType: 'default', + promise: withResolvers.promise, + resolve: withResolvers.resolve, + reject: withResolvers.reject, + reader, + }; + const byteConsumer = consumer as unknown as ByteStreamConsumerType; + promise = byteConsumer.readBYOB(descriptor); + } else { + promise = consumer.read(reader); + } + + // A read request is a pull trigger (spec PullSteps ordering: the pull's + // synchronous side-effects run before the read result is delivered). + if (controller !== undefined) controllerPullIfNeeded(controller); + const result = (await promise) as ReadableStreamReadResult; + // Drain-then-close (spec HandleQueueDrain): if this read consumed the + // last data with close requested, the controller's primary stream + // transitions now — checked on every read completion, not just done + // results, because the read that drains the final chunk resolves with + // done: false. + if (controller !== undefined) controllerMaybeCloseStream(controller); + if (result.done) { + // This stream's cursor reached the sentinel. Close THIS stream (tee + // branches close independently of the controller's primary stream). + readableStreamClose(stream); + } + return result; +} + +class ReadableStreamDefaultReader< + R, +> implements ReadableStreamDefaultReaderType { + #base: ReadableStreamReaderBase; + + static { + // Wire getReaderBase for the first reader type. The BYOB and draining + // readers chain onto this in their own static blocks. + const defaultGetBase = (reader: object) => { + return (reader as ReadableStreamDefaultReader).#base; + }; + getReaderBase = defaultGetBase; + + readableStreamDefaultReaderRead = ( + reader: ReadableStreamDefaultReaderType, + readRequest: ReadableStreamAsyncIteratorReadRequest + ) => { + const stream = getReaderStream(reader); + if (stream === undefined) { + readRequest.error(new TypeError('This reader has been released')); + return; + } + PromisePrototypeThen( + defaultReaderReadInternal(reader, stream), + (result: ReadableStreamReadResult) => { + if (result.done) { + readRequest.close(); + } else { + readRequest.chunk(result.value as R); + } + }, + (e: unknown) => { + readRequest.error(e); + } + ); + }; + } + + constructor(stream: ReadableStream) { + this.#base = new ReadableStreamReaderBase(); + if (isReadableStreamLocked(stream)) { + throw new TypeError('Cannot get a reader for a stream that is locked'); + } + setReadableStreamReader( + stream, + this as unknown as ReadableStreamReaderType + ); + initializeReadableStreamGenericReader(stream, this.#base); + } + + get closed(): Promise { + try { + return getGenericReaderClosedPromise(this); + } catch (e) { + return PromiseReject(e) as Promise; + } + } + + cancel(reason: unknown = undefined): Promise { + try { + return cancelReadableStreamGenericReader(this, reason); + } catch (e) { + return PromiseReject(e) as Promise; + } + } + + read(): Promise> { + try { + const stream = getReaderStream(this); + if (stream === undefined) { + return PromiseReject( + new TypeError('This reader has been released') + ) as Promise>; + } + return defaultReaderReadInternal(this, stream); + } catch (e) { + return PromiseReject(e) as Promise>; + } + } + + releaseLock(): void { + if (!isReaderBoundToStream(this)) return; + readableStreamReaderGenericRelease(this); + } + + [SymbolToStringTag] = 'ReadableStreamDefaultReader'; +} + +class ReadableStreamBYOBReader implements ReadableStreamBYOBReaderType { + #base: ReadableStreamReaderBase; + + static { + const prev = getReaderBase; + getReaderBase = (reader: object) => { + if (#base in reader) { + return reader.#base as unknown as ReadableStreamReaderBase; + } + return prev(reader); + }; + } + + constructor(stream: ReadableStream) { + this.#base = new ReadableStreamReaderBase(); + if (isReadableStreamLocked(stream)) { + throw new TypeError('Cannot get a reader for a stream that is locked'); + } + const controller = getReadableStreamController(stream); + // The byte-CAPABLE gate (see isByteCapableController): queued byte + // controllers and ALL native controllers pass (native sources are + // byte-capable by definition). Using the queued-only brand here would + // wrongly reject getReader({mode:'byob'}) on a native stream. + if (!isByteCapableController(controller)) { + throw new TypeError( + 'BYOB reader can only be used on a stream with a byte source' + ); + } + setReadableStreamReader( + stream, + this as unknown as ReadableStreamReaderType + ); + initializeReadableStreamGenericReader(stream, this.#base); + } + + get closed(): Promise { + try { + return getGenericReaderClosedPromise(this); + } catch (e) { + return PromiseReject(e) as Promise; + } + } + + cancel(reason: unknown = undefined): Promise { + try { + return cancelReadableStreamGenericReader(this, reason); + } catch (e) { + return PromiseReject(e) as Promise; + } + } + + async read( + view: T, + options: ReadableStreamBYOBReaderReadOptions = {} + ): Promise> { + // --- View validation (spec read(view, options) steps 1-3) --- + if (!isArrayBufferView(view)) { + throw new TypeError('view must be an ArrayBufferView'); + } + const info = getViewInfo(view); + if (info.byteLength === 0) { + throw new TypeError('view must have a non-zero byteLength'); + } + if (ArrayBufferPrototypeByteLengthGet(info.buffer) === 0) { + throw new TypeError( + "view's backing buffer must have a non-zero byteLength" + ); + } + if (!isActualObject(options)) { + throw new TypeError('options must be an object'); + } + // --- min validation (steps 4-6). min is in ELEMENTS for typed arrays + // (bytes for DataView, where info.length === byteLength). NO clamping: + // out-of-range is an error, not a request to do less. + let min = 1; + if (options.min !== undefined) { + min = +options.min; + if (NumberIsNaN(min) || min % 1 !== 0 || min < 0) { + throw new TypeError('options.min must be a non-negative integer'); + } + if (min === 0) { + throw new TypeError('options.min must be greater than 0'); + } + if (min > info.length) { + throw new RangeError( + 'options.min must not exceed the length of the view' + ); + } + } + const stream = getReaderStream(this); + if (stream === undefined) { + throw new TypeError('This reader has been released'); + } + setReadableStreamDisturbed(stream); + if (getReadableStreamGetState(stream) === 'errored') { + throw getReadableStreamStoredError(stream); + } + // Transfer the buffer regardless of state (spec step). + const transferred = ArrayBufferPrototypeTransfer(info.buffer); + if (getReadableStreamGetState(stream) === 'closed') { + // The stream may be closed with no consumer remaining (e.g. after + // cancel clears it). Without a consumer, no pull-into can be + // registered. Return { value: undefined, done: true } — matching + // browser behavior and the C++ implementation. + const consumer = getReadableStreamConsumer(stream); + if (consumer === undefined) { + return createReadResult(undefined as unknown as T, true); + } + // Normal close with a live consumer: return a zero-length view over + // the transferred buffer (spec ReadableByteStreamControllerPullInto + // step 2 "closed" branch). + const emptyView = new info.viewCtor(transferred, info.byteOffset, 0); + return createReadResult(emptyView as T, true); + } + // BACKEND-BLIND: the byte-consumer interface covers both backends; the + // cast is justified by the byte-capable reader gate in the constructor. + const consumer = getReadableStreamConsumer( + stream + ) as unknown as ByteStreamConsumerType; + const withResolvers = PromiseWithResolvers() as PromiseWithResolversType< + ReadableStreamReadResult + >; + const descriptor: PullIntoDescriptor = { + buffer: transferred, + bufferByteLength: ArrayBufferPrototypeByteLengthGet(transferred), + byteOffset: info.byteOffset, + byteLength: info.byteLength, + bytesFilled: 0, + minimumFill: min * info.elementSize, + elementSize: info.elementSize, + viewCtor: info.viewCtor, + readerType: 'byob', + promise: withResolvers.promise, + resolve: withResolvers.resolve, + reject: withResolvers.reject, + reader: this, + }; + const promise = consumer.readBYOB(descriptor); + const controller = getReadableStreamController(stream); + if (controller !== undefined) controllerPullIfNeeded(controller); + const result = await promise; + // Drain-then-close (spec HandleQueueDrain): a BYOB fill that consumed + // the last queued bytes with close requested must transition the + // stream — without this, the NEXT read(view) would pend forever (BYOB + // descriptors are not auto-committed at the sentinel). + if (controller !== undefined) controllerMaybeCloseStream(controller); + return result as unknown as ReadableStreamReadResult; + } + + async readAtLeast( + minElements: number, + view: T + ): Promise> { + return this.read(view, { min: minElements }); + } + + releaseLock(): void { + if (!isReaderBoundToStream(this)) return; + readableStreamReaderGenericRelease(this); + } + + [SymbolToStringTag] = 'ReadableStreamBYOBReader'; +} + +// Stream-level state transitions shared by the controllers and (in later +// phases) the read paths. Defined as plain functions — they only use the +// static-block-exported accessors, which are assigned at class-definition +// time, strictly before any of this can run. +function readableStreamClose(stream: ReadableStream): void { + if (getReadableStreamGetState(stream) !== 'readable') return; + setReadableStreamState(stream, 'closed'); + const reader = getReadableStreamReader(stream); + if (reader !== undefined) { + resolveGenericReaderPromise(reader); + } + notifyBranchSettled(stream); +} + +function readableStreamError(stream: ReadableStream, e: unknown): void { + if (getReadableStreamGetState(stream) !== 'readable') return; + setReadableStreamState(stream, 'errored'); + setReadableStreamStoredError(stream, e); + const reader = getReadableStreamReader(stream); + if (reader !== undefined) { + rejectGenericReaderPromise(reader, e); + } + notifyBranchSettled(stream); +} + +// Metadata snapshot of an ArrayBufferView, captured at a trust boundary. +// All reads go through captured prototype getters — view.buffer et al. are +// patchable accessors, and view.constructor is user-controllable. +interface ViewInfo { + buffer: ArrayBuffer; + byteOffset: number; + byteLength: number; + elementSize: number; // BYTES_PER_ELEMENT; 1 for DataView + length: number; // element count; byteLength for DataView + // The view's REAL constructor, resolved from the internal type slot via + // the primordials name→ctor map (DataView capture for DataViews). + viewCtor: new ( + buffer: ArrayBuffer, + byteOffset: number, + length: number + ) => ArrayBufferView; +} + +// PRECONDITION: caller has verified isArrayBufferView(view). Every +// ArrayBufferView is either a typed array (identified via the internal +// [[TypedArrayName]] slot) or a DataView; the DataView getters brand-check +// the receiver, so nothing dishonest survives this function. +function getViewInfo(view: ArrayBufferView): ViewInfo { + const name = TypedArrayPrototypeGetSymbolToStringTag(view); + if (name !== undefined) { + const viewCtor = TypedArrayCtorByName[name]; + return { + buffer: TypedArrayPrototypeGetBuffer(view), + byteOffset: TypedArrayPrototypeGetByteOffset(view), + byteLength: TypedArrayPrototypeGetByteLength(view), + // BYTES_PER_ELEMENT is a non-writable, non-configurable own property + // of the captured constructor — safe to read. + elementSize: viewCtor.BYTES_PER_ELEMENT, + length: TypedArrayPrototypeGetLength(view), + viewCtor, + }; + } + const dataView = view as DataView; + const byteLength = DataViewPrototypeGetByteLength(dataView); + return { + buffer: DataViewPrototypeGetBuffer(dataView), + byteOffset: DataViewPrototypeGetByteOffset(dataView), + byteLength, + elementSize: 1, + length: byteLength, + viewCtor: DataView, + }; +} + +// Validate and normalize a user-provided chunk for a byte stream at the +// enqueue()/respondWithNewView() trust boundary: snapshot metadata, reject +// zero-length views and zero-length (or detached — detached buffers report +// byteLength 0) buffers, and transfer the backing buffer. The returned +// triple references the TRANSFERRED buffer; the caller's view is detached. +function validateAndTransferView(view: ArrayBufferView): ByteQueueEntry { + if (!isArrayBufferView(view)) { + throw new TypeError('chunk must be an ArrayBufferView'); + } + const info = getViewInfo(view); + if (info.byteLength === 0) { + throw new TypeError('chunk must have a non-zero byteLength'); + } + if (ArrayBufferPrototypeByteLengthGet(info.buffer) === 0) { + throw new TypeError( + "chunk's backing buffer must have a non-zero byteLength" + ); + } + return { + buffer: ArrayBufferPrototypeTransfer(info.buffer), + byteOffset: info.byteOffset, + byteLength: info.byteLength, + }; +} + +class ReadableStreamDefaultController< + R = unknown, +> implements ReadableStreamDefaultControllerType { + #stream: ReadableStream; + #queue: StreamQueueType; + // Algorithms are cleared (closures dropped) on close-complete, error, and + // cancel, per spec ClearAlgorithms. + #sizeAlgorithm: ((chunk: R) => number) | undefined; + #pullAlgorithm: (() => Promise) | undefined; + #cancelAlgorithm: ((reason: unknown) => Promise) | undefined; + #started: boolean = false; + #pulling: boolean = false; + #pullAgain: boolean = false; + #closeRequested: boolean = false; + #cancelPromise: Promise | undefined; + + static { + // BACKEND-DISPATCH: the chained controller helpers (one of the five + // sanctioned dispatch points). Each backend wraps the previous + // implementation behind its own brand check — private-brand `in` for + // the queued controllers, NEVER instanceof: these classes end up on + // the global, so Symbol.hasInstance on them is user-reachable. The + // byte controller chains in its static block below; the native + // controller's wrap is joined at the bottom of this module (its brand + // predicate crosses the module fence from native.ts). + controllerPullIfNeeded = (controller) => { + if (#queue in controller) { + (controller as ReadableStreamDefaultController).#callPullIfNeeded(); + } + // The byte controller wires its own branch in the byte pass. + }; + + // expectedLength is byte-stream-only: the default controller never + // reads it from the source (silently ignored if declared) and always + // reports undefined. + getControllerExpectedLength = () => undefined; + + controllerCancelSteps = (controller, reason) => { + if (#queue in controller) { + return (controller as ReadableStreamDefaultController).#cancelSteps( + reason + ); + } + return PromiseResolve(); + }; + + controllerMaybeCloseStream = (controller) => { + if (#queue in controller) { + (controller as ReadableStreamDefaultController).#maybeCloseStream(); + } + }; + + // Default controllers have no byobRequest to invalidate; the byte + // controller's static block wraps this with the real implementation. + controllerOnReaderRelease = (_controller) => {}; + } + + constructor( + privateSymbol: symbol, + stream: ReadableStream, + underlyingSource: UnderlyingDefaultSource, + sizeAlgorithm: (chunk: R) => number, + highWaterMark: number + ) { + assertPrivateSymbol(privateSymbol); + this.#stream = stream; + + // --- Underlying source method extraction --- + // Methods are read ONCE here (spec: dictionary conversion / + // CreateAlgorithmFromUnderlyingMethod) and invoked via captured + // uncurryThis wrappers with the source object as `this`. Property reads + // are in alphabetical order to match WebIDL dictionary conversion. + const cancelFn = underlyingSource.cancel; + if (cancelFn !== undefined && typeof cancelFn !== 'function') { + throw new TypeError('underlyingSource.cancel must be a function'); + } + const pullFn = underlyingSource.pull; + if (pullFn !== undefined && typeof pullFn !== 'function') { + throw new TypeError('underlyingSource.pull must be a function'); + } + const startFn = underlyingSource.start; + if (startFn !== undefined && typeof startFn !== 'function') { + throw new TypeError('underlyingSource.start must be a function'); + } + + if (cancelFn !== undefined) { + const callCancel = uncurryThis(cancelFn); + // Spec PromiseCall: sync throws become rejections; result is always + // a promise. + this.#cancelAlgorithm = (reason: unknown) => { + try { + return PromiseResolve( + callCancel(underlyingSource, reason) + ) as Promise; + } catch (e) { + return PromiseReject(e) as Promise; + } + }; + } + if (pullFn !== undefined) { + const callPull = uncurryThis(pullFn); + this.#pullAlgorithm = () => { + try { + return PromiseResolve( + callPull(underlyingSource, this) + ) as Promise; + } catch (e) { + return PromiseReject(e) as Promise; + } + }; + } + + // Strategy already extracted by caller (WebIDL conversion order). + this.#sizeAlgorithm = sizeAlgorithm; + + // --- Queue + the stream's own cursor --- + this.#queue = new StreamQueue(highWaterMark, () => { + // Last consumer went away (GC-driven cursor cleanup after all + // branch streams became unreachable). Silently stop the source + // by clearing algorithms — do NOT invoke the user's cancel + // callback, because GC timing is nondeterministic and must not + // produce user-observable side effects (the cancel callback + // could push to event arrays, resolve promises, etc.). + this.#clearAlgorithms(); + }) as StreamQueueType; + setReadableStreamConsumer( + stream, + new QueueCursor(this.#queue, stream) as QueueCursorType + ); + + // --- Start --- + // Per spec, start is invoked synchronously and a sync throw propagates + // out of the ReadableStream constructor. + const startResult: unknown = + startFn === undefined + ? undefined + : uncurryThis(startFn)(underlyingSource, this); + PromisePrototypeThen( + PromiseResolve(startResult), + () => { + this.#started = true; + this.#callPullIfNeeded(); + }, + (e: unknown) => { + this.error(e); + } + ); + } + + get desiredSize(): number | null { + if (!(#queue in this)) throw new TypeError('Illegal invocation'); + // null when ERRORED, 0 when CLOSED, computed while readable — including + // while close-requested-but-still-draining (spec GetDesiredSize). + switch (getReadableStreamGetState(this.#stream)) { + case 'errored': + return null; + case 'closed': + return 0; + default: + return this.#queue.desiredSize; + } + } + + enqueue(chunk: R = undefined as R): void { + if (!(#queue in this)) throw new TypeError('Illegal invocation'); + if (!this.#canCloseOrEnqueue()) { + throw new TypeError( + 'Cannot enqueue a chunk into a stream that is closed or closing' + ); + } + // Spec step 3: if the stream has a default reader with pending read + // requests, fulfill the first one directly — no size() call, no queue. + // This must be checked BEFORE size() to avoid reentrant reads from + // inside size() being fulfilled eagerly by the queue notification. + const cursor = getReadableStreamConsumer(this.#stream) as + | QueueCursorType + | undefined; + if ( + cursor !== undefined && + cursor.hasPendingRead && + isReadableStreamLocked(this.#stream) + ) { + cursor.fulfillFirstPendingRead(chunk); + this.#callPullIfNeeded(); + return; + } + // Snapshot pending-read state before size(). Reads added reentrantly + // from inside size() must NOT be auto-filled by this enqueue's queue + // notification — the spec's EnqueueValueWithSize just stores the + // chunk; the reentrant reads wait for the next enqueue to fulfill + // them via the direct path (step 3 above). + const hadPendingRead = cursor !== undefined && cursor.hasPendingRead; + let size: number; + try { + // assert: sizeAlgorithm is set (canCloseOrEnqueue guard above + // rejects after clearAlgorithms) + size = +(this.#sizeAlgorithm as (chunk: R) => number)(chunk as R); + // Spec EnqueueValueWithSize: NaN, negative, and +Infinity sizes are + // RangeErrors. + if (NumberIsNaN(size) || size < 0 || size === Infinity) { + throw new RangeError('Invalid chunk size'); + } + } catch (e) { + // A throwing size() (or invalid size) errors the stream AND + // propagates to the caller (spec enqueue error steps). + this.error(e); + throw e; + } + // Suppress cursor notification when reads were added reentrantly + // during size(). The chunk goes into the queue (updating the size + // accounting) but pending reads are left unfilled until the next + // enqueue triggers the direct-fulfillment path (step 3). + const notify = hadPendingRead || !cursor?.hasPendingRead; + this.#queue.enqueue({ value: chunk as R, size }, notify); + this.#callPullIfNeeded(); + } + + close(): void { + if (!(#queue in this)) throw new TypeError('Illegal invocation'); + if (!this.#canCloseOrEnqueue()) { + throw new TypeError( + 'Cannot close a stream that is already closed or closing' + ); + } + this.#closeRequested = true; + this.#queue.close(); // pushes the sentinel; notifies all cursors + // If the stream's own cursor has already drained to the sentinel, the + // stream closes immediately; otherwise the read paths complete the + // transition when the cursor reaches the sentinel (drain-then-close). + this.#maybeCloseStream(); + } + + error(reason: unknown = undefined): void { + if (!(#queue in this)) throw new TypeError('Illegal invocation'); + if (getReadableStreamGetState(this.#stream) !== 'readable') return; + // Propagate to every live consumer stream (tee branches) via the + // cursors' weak owner refs — no strong retention of branches. The + // primary stream is handled explicitly: a tee'd-away parent has no + // cursor. readableStreamError is state-guarded, so overlap is fine. + const owners = this.#queue.getLiveOwners(); + this.#queue.error(reason); // rejects pending reads, drops buffered data + this.#clearAlgorithms(); + for (let i = 0; i < owners.length; i++) { + readableStreamError(owners[i] as ReadableStream, reason); + } + readableStreamError(this.#stream, reason); + } + + #canCloseOrEnqueue(): boolean { + // #cancelPromise doubles as the "source cancelled" flag: after + // CancelSteps (explicit cancel or the all-cursors-gone hook) the + // algorithms are cleared and enqueue/close must be rejected even thoughs + // the original stream object may still report state 'readable' (the + // GC-driven path has no stream left to transition). + return ( + !this.#closeRequested && + this.#cancelPromise === undefined && + getReadableStreamGetState(this.#stream) === 'readable' + ); + } + + #maybeCloseStream(): void { + if (!this.#closeRequested) return; + // Check ALL live consumer streams (tee branches + the parent). + // A stream whose cursor has reached the close sentinel transitions + // to 'closed'; streams with buffered data before the sentinel stay + // 'readable' until drained (drain-then-close on subsequent reads). + let anyOpen = false; + const owners = this.#queue.getLiveOwners(); + for (let i = 0; i < owners.length; i++) { + const owner = owners[i] as ReadableStream; + const cursor = getReadableStreamConsumer(owner) as + | QueueCursorType + | undefined; + if (cursor === undefined) continue; + if (this.#queue.getEntry(cursor.position) === CLOSE_SENTINEL) { + readableStreamClose(owner); + } else { + anyOpen = true; + } + } + // Also check the parent stream itself (pre-tee path: only one consumer). + const parentCursor = getReadableStreamConsumer(this.#stream) as + | QueueCursorType + | undefined; + if (parentCursor !== undefined) { + if (this.#queue.getEntry(parentCursor.position) === CLOSE_SENTINEL) { + readableStreamClose(this.#stream); + } else { + anyOpen = true; + } + } + if (!anyOpen) { + this.#clearAlgorithms(); + } + } + + #shouldCallPull(): boolean { + if (!this.#started) return false; + if (!this.#canCloseOrEnqueue()) return false; + // The pending-read clause is what keeps a fast consumer from starving + // when the queue is at the high water mark: a consumer that reads + // faster than the HWM drains must still trigger pulls. + return this.#queue.desiredSize > 0 || this.#queue.anyCursorHasPendingRead(); + } + + #callPullIfNeeded(): void { + if (!this.#shouldCallPull()) return; + if (this.#pulling) { + this.#pullAgain = true; + return; + } + this.#pulling = true; + const pullAlgorithm = this.#pullAlgorithm; + const result = + pullAlgorithm === undefined ? PromiseResolve() : pullAlgorithm(); + PromisePrototypeThen( + result, + () => { + this.#pulling = false; + if (this.#pullAgain) { + this.#pullAgain = false; + this.#callPullIfNeeded(); + } + }, + (e: unknown) => { + this.error(e); + } + ); + } + + // Cancel the underlying source (spec CancelSteps). Idempotent and cached: + // this can be reached from both an explicit stream/branch cancel and the + // all-cursors-gone hook. + #cancelSteps(reason: unknown): Promise { + if (this.#cancelPromise !== undefined) return this.#cancelPromise; + const cancelAlgorithm = this.#cancelAlgorithm; + this.#clearAlgorithms(); + this.#cancelPromise = + cancelAlgorithm === undefined + ? (PromiseResolve() as Promise) + : cancelAlgorithm(reason); + return this.#cancelPromise; + } + + #clearAlgorithms(): void { + this.#pullAlgorithm = undefined; + this.#cancelAlgorithm = undefined; + this.#sizeAlgorithm = undefined; + } + + [SymbolToStringTag] = 'ReadableStreamDefaultController'; +} + +// Cross-class accessors for the BYOB request/controller pairing, assigned +// in the respective static blocks. The request methods delegate to the byte +// controller (assigned later in module evaluation — only ever called at +// runtime, after all classes are defined). +let initializeByobRequest: ( + request: ReadableStreamBYOBRequest, + controller: ReadableByteStreamController, + view: Uint8Array, + atLeast: number +) => void; +let invalidateByobRequestObject: (request: ReadableStreamBYOBRequest) => void; +let byteControllerRespond: ( + controller: ReadableByteStreamController, + bytesWritten: number +) => void; +let byteControllerRespondWithNewView: ( + controller: ReadableByteStreamController, + view: ArrayBufferView +) => void; +let getByteControllerAutoAllocateChunkSize: ( + controller: ReadableByteStreamController +) => number | undefined; + +class ReadableStreamBYOBRequest implements ReadableStreamBYOBRequestType { + // All null once invalidated. Per spec, EVERY respond()/ + // respondWithNewView()/enqueue() invalidates the outstanding request; + // the next byobRequest access mints a fresh one over the remainder. + #controller: ReadableByteStreamController | null = null; + #view: Uint8Array | null = null; + #atLeast: number | null = null; + + static { + initializeByobRequest = (request, controller, view, atLeast) => { + request.#controller = controller; + request.#view = view; + request.#atLeast = atLeast; + }; + + invalidateByobRequestObject = (request) => { + request.#controller = null; + request.#view = null; + request.#atLeast = null; + }; + } + + constructor(privateSymbol: symbol) { + assertPrivateSymbol(privateSymbol); + } + + get view(): Uint8Array | null { + if (!(#view in this)) throw new TypeError('Illegal invocation'); + return this.#view; + } + + // Non-standard workerd extension (compat parity with the existing + // implementation's getAtLeast): the minimum number of BYTES (never + // elements) the source must still deliver before the outstanding read + // is satisfied — i.e., the head descriptor's remaining minimum + // (minimumFill − bytesFilled), captured at mint. The capture stays + // fresh because every fill path (respond/respondWithNewView/enqueue) + // invalidates this request and the next access mints a new one. For a + // min-less read(view) this equals the view's element size, matching the + // old implementation's max(elementSize, atLeast) floor. null once + // invalidated, mirroring the old kj::Maybe behavior. + get atLeast(): number | null { + if (!(#view in this)) throw new TypeError('Illegal invocation'); + return this.#atLeast; + } + + respond(bytesWritten: number): void { + if (!(#view in this)) throw new TypeError('Illegal invocation'); + if (this.#controller === null) { + throw new TypeError('This BYOB request has been invalidated'); + } + byteControllerRespond(this.#controller, bytesWritten); + } + + respondWithNewView(view: ArrayBufferView): void { + if (!(#view in this)) throw new TypeError('Illegal invocation'); + if (this.#controller === null) { + throw new TypeError('This BYOB request has been invalidated'); + } + if (!isArrayBufferView(view)) { + throw new TypeError('view must be an ArrayBufferView'); + } + // Spec step 2: detached buffers are TypeError, not RangeError. + const info = getViewInfo(view); + if (ArrayBufferPrototypeDetachedGet(info.buffer)) { + throw new TypeError("The view's buffer has been detached"); + } + byteControllerRespondWithNewView(this.#controller, view); + } + + [SymbolToStringTag] = 'ReadableStreamBYOBRequest'; +} + +class ReadableByteStreamController implements ReadableByteStreamControllerType { + #stream: ReadableStream; + #queue: StreamQueueType; + #pullAlgorithm: (() => Promise) | undefined; + #cancelAlgorithm: ((reason: unknown) => Promise) | undefined; + #autoAllocateChunkSize: number | undefined; + // The non-standard exact-total byte contract (undefined = unknown). + // The source must deliver exactly this many bytes over its lifetime + // (via enqueue AND byobRequest responds combined): overflow at + // delivery and underflow at close() are RangeError violations. + // Consumer-initiated cancel is exempt. + #expectedLength: bigint | undefined; + #bytesDelivered: bigint = 0n; + #started: boolean = false; + #pulling: boolean = false; + #pullAgain: boolean = false; + #closeRequested: boolean = false; + #cancelPromise: Promise | undefined; + #byobRequest: ReadableStreamBYOBRequest | null = null; + + static { + isByteStreamController = (value: unknown) => { + return isActualObject(value) && #queue in value; + }; + + // Chain the controller dispatch helpers. The default controller's + // static block (which runs earlier in module evaluation) assigned the + // initial implementations; we wrap them. Note the two classes' #queue + // private names are distinct brands, so the `in` checks discriminate + // correctly. + const prevPullIfNeeded = controllerPullIfNeeded; + controllerPullIfNeeded = (controller) => { + if (#queue in controller) { + controller.#callPullIfNeeded(); + } else { + prevPullIfNeeded(controller); + } + }; + + const prevCancelSteps = controllerCancelSteps; + controllerCancelSteps = (controller, reason) => { + if (#queue in controller) { + return controller.#cancelSteps(reason); + } + return prevCancelSteps(controller, reason); + }; + + const prevMaybeCloseStream = controllerMaybeCloseStream; + controllerMaybeCloseStream = (controller) => { + if (#queue in controller) { + controller.#maybeCloseStream(); + } else { + prevMaybeCloseStream(controller); + } + }; + + byteControllerRespond = (controller, bytesWritten) => { + controller.#respond(bytesWritten); + }; + + byteControllerRespondWithNewView = (controller, view) => { + controller.#respondWithNewView(view); + }; + + getByteControllerAutoAllocateChunkSize = (controller) => { + return controller.#autoAllocateChunkSize; + }; + + const prevOnReaderRelease = controllerOnReaderRelease; + controllerOnReaderRelease = (controller) => { + if (#queue in controller) { + // Spec: releaseLock does NOT invalidate the byobRequest. The + // pull-into descriptor stays in pendingPullIntos with readerType + // set to 'none'; a future respond() will enqueue the data into the + // queue for the next reader instead of resolving a read promise. + } else { + prevOnReaderRelease(controller); + } + }; + + const prevExpectedLength = getControllerExpectedLength; + getControllerExpectedLength = (controller) => { + if (#queue in controller) { + return (controller as ReadableByteStreamController).#expectedLength; + } + return prevExpectedLength(controller); + }; + } + + constructor( + privateSymbol: symbol, + stream: ReadableStream, + underlyingSource: UnderlyingByteSource, + highWaterMark: number + ) { + assertPrivateSymbol(privateSymbol); + this.#stream = stream; + + // --- Underlying source extraction (alphabetical property reads) --- + const autoAllocateChunkSize = underlyingSource.autoAllocateChunkSize; + if (autoAllocateChunkSize !== undefined) { + const size = +autoAllocateChunkSize; + // Approximates WebIDL [EnforceRange] unsigned long long plus the + // spec's explicit zero check. + if (NumberIsNaN(size) || size <= 0 || size % 1 !== 0) { + throw new TypeError('autoAllocateChunkSize must be a positive integer'); + } + this.#autoAllocateChunkSize = size; + } + const cancelFn = underlyingSource.cancel; + if (cancelFn !== undefined && typeof cancelFn !== 'function') { + throw new TypeError('underlyingSource.cancel must be a function'); + } + // Non-standard extension (byte streams only): the TOTAL bytes this + // source promises to produce. Read once and cached; this controller + // enforces the contract at enqueue/respond (overflow) and close() + // (underflow). Exposed to the C++ bridge via the DrainingReader. + this.#expectedLength = normalizeExpectedLength( + underlyingSource.expectedLength + ); + const pullFn = underlyingSource.pull; + if (pullFn !== undefined && typeof pullFn !== 'function') { + throw new TypeError('underlyingSource.pull must be a function'); + } + const startFn = underlyingSource.start; + if (startFn !== undefined && typeof startFn !== 'function') { + throw new TypeError('underlyingSource.start must be a function'); + } + + if (cancelFn !== undefined) { + const callCancel = uncurryThis(cancelFn); + this.#cancelAlgorithm = (reason: unknown) => { + try { + return PromiseResolve( + callCancel(underlyingSource, reason) + ) as Promise; + } catch (e) { + return PromiseReject(e) as Promise; + } + }; + } + if (pullFn !== undefined) { + const callPull = uncurryThis(pullFn); + this.#pullAlgorithm = () => { + try { + return PromiseResolve( + callPull(underlyingSource, this) + ) as Promise; + } catch (e) { + return PromiseReject(e) as Promise; + } + }; + } + + // Strategy already extracted by caller (WebIDL conversion order). + + // --- Queue + the stream's own (byte) cursor --- + this.#queue = new StreamQueue(highWaterMark, () => { + // Last consumer went away (GC-driven cursor cleanup). Silently + // stop the source — see the default controller's hook for the + // rationale (GC must not produce user-observable side effects). + this.#clearAlgorithms(); + }) as StreamQueueType; + const cursor = new ByteStreamCursor(this.#queue, stream); + // Wire up the fractional-element-at-close error callback so the + // cursor can error the stream when a BYOB read lands at the close + // sentinel with a non-element-aligned partial fill. + cursor.errorStreamCallback = (e: unknown) => { + this.error(e); + }; + setReadableStreamConsumer(stream, cursor); + + // --- Start (sync throw propagates out of the ReadableStream ctor) --- + const startResult: unknown = + startFn === undefined + ? undefined + : uncurryThis(startFn)(underlyingSource, this); + PromisePrototypeThen( + PromiseResolve(startResult), + () => { + this.#started = true; + this.#callPullIfNeeded(); + }, + (e: unknown) => { + this.error(e); + } + ); + } + + get desiredSize(): number | null { + if (!(#queue in this)) throw new TypeError('Illegal invocation'); + switch (getReadableStreamGetState(this.#stream)) { + case 'errored': + return null; + case 'closed': + return 0; + default: + return this.#queue.desiredSize; + } + } + + get byobRequest(): ReadableStreamBYOBRequestType | null { + if (!(#queue in this)) throw new TypeError('Illegal invocation'); + if (this.#byobRequest === null) { + // Zero-copy is only unambiguous with exactly one consumer, and only + // when it has a head pull-into descriptor (from a BYOB read, or + // auto-allocated for a default read when autoAllocateChunkSize is + // set). Otherwise the source must fall back to enqueue(). The + // request object is cached for identity until invalidated. + const cursor = this.#queue.singleCursor as + | ByteStreamCursorType + | undefined; + if (cursor === undefined) return null; + const view = cursor.pendingPullIntoView; + const head = cursor.headPullInto; + if (view === undefined || head === undefined) return null; + const request = new ReadableStreamBYOBRequest(kPrivateSymbol); + // atLeast = the head descriptor's remaining minimum, in bytes. The + // head is unfulfilled by construction (a fulfilled descriptor is + // shifted before any request could be minted), so this is >= 1. + initializeByobRequest( + request, + this, + view, + head.minimumFill - head.bytesFilled + ); + this.#byobRequest = request; + } + return this.#byobRequest; + } + + enqueue(chunk: ArrayBufferView): void { + if (!(#queue in this)) throw new TypeError('Illegal invocation'); + if (!this.#canCloseOrEnqueue()) { + throw new TypeError( + 'Cannot enqueue a chunk into a stream that is closed or closing' + ); + } + // Trust boundary: snapshot metadata via captured getters, validate, + // transfer the backing buffer, normalize to a {buffer, byteOffset, + // byteLength} triple. Queue internals never touch the user's view. + const entry = validateAndTransferView(chunk); + // EXPECTED-LENGTH CONTRACT: overflow check before the entry reaches + // the queue. (The buffer was already transferred above — a refused + // overflow chunk's buffer is detached. Contract violators lose the + // buffer; the stream errors via the pull-rejection path anyway.) + this.#accountDelivery(entry.byteLength); + // Per spec, enqueue invalidates the outstanding byobRequest (a fresh + // one over the updated remainder is minted on next access). + this.#invalidateByobRequest(); + const drainCursor = this.#queue.singleCursor as + | ByteStreamCursorType + | undefined; + if (drainCursor !== undefined) { + const head = drainCursor.headPullInto; + if (head !== undefined) { + // Spec step 8.4: transfer the head descriptor's buffer so that + // old captured views are detached. + head.buffer = ArrayBufferPrototypeTransfer(head.buffer); + } + // Spec step 8.5: if the head pending pull-into has readerType 'none' + // (leftover from releaseLock), drain it before adding the new chunk. + drainCursor.drainNoneDescriptors(); + // Spec step 9.3: if the head descriptor is an auto-allocate + // (readerType 'default'), discard it and fulfill the pending default + // read directly from the enqueued chunk. The auto-allocate buffer is + // abandoned — the result uses the chunk's (smaller) buffer. + const autoDesc = drainCursor.shiftAutoAllocateDescriptor(); + if (autoDesc !== undefined) { + const view = new Uint8Array( + entry.buffer, + entry.byteOffset, + entry.byteLength + ); + autoDesc.resolve(createReadResult(view, false)); + this.#callPullIfNeeded(); + return; + } + } + this.#queue.enqueue({ value: entry, size: entry.byteLength }); + // The cursors' notify() (run by queue.enqueue) services pending + // pull-intos and default reads alike. + this.#callPullIfNeeded(); + } + + close(): void { + if (!(#queue in this)) throw new TypeError('Illegal invocation'); + if (!this.#canCloseOrEnqueue()) { + throw new TypeError( + 'Cannot close a stream that is already closed or closing' + ); + } + // Spec: closing with a fractional-element partial fill in a head + // descriptor is a TypeError that also errors the stream — the bytes to + // complete the element can never arrive. Checked across ALL live + // cursors: tee branches each track their own partial fills through the + // enqueue-path min-read machinery. + const hasFractionalFill = this.#queue.someLiveCursor((cursor) => { + const head: PullIntoDescriptor | undefined = ( + cursor as unknown as ByteStreamCursorType + ).headPullInto; + return head !== undefined && head.bytesFilled % head.elementSize !== 0; + }); + if (hasFractionalFill) { + const e = new TypeError( + 'Insufficient bytes to fill elements in the given view' + ); + this.error(e); + throw e; + } + // EXPECTED-LENGTH CONTRACT: closing before delivering the declared + // total is underflow — error the stream and throw (mirrors the + // fractional-fill violation above). Data still buffered in the queue + // counts as delivered: the source produced it. + if ( + this.#expectedLength !== undefined && + this.#bytesDelivered < this.#expectedLength + ) { + const e = new RangeError( + 'byte source closed before producing its declared expectedLength' + ); + this.error(e); + throw e; + } + this.#closeRequested = true; + this.#queue.close(); + this.#maybeCloseStream(); + } + + error(reason: unknown = undefined): void { + if (!(#queue in this)) throw new TypeError('Illegal invocation'); + if (getReadableStreamGetState(this.#stream) !== 'readable') return; + this.#invalidateByobRequest(); + // Branch propagation — see the default controller's error() for why. + const owners = this.#queue.getLiveOwners(); + this.#queue.error(reason); + this.#clearAlgorithms(); + for (let i = 0; i < owners.length; i++) { + readableStreamError(owners[i] as ReadableStream, reason); + } + readableStreamError(this.#stream, reason); + } + + // byobRequest.respond(bytesWritten) — the zero-copy path. + #respond(bytesWritten: number): void { + bytesWritten = +bytesWritten; + if ( + NumberIsNaN(bytesWritten) || + bytesWritten < 0 || + bytesWritten % 1 !== 0 + ) { + throw new TypeError('bytesWritten must be a non-negative integer'); + } + const cursor = this.#queue.singleCursor as ByteStreamCursorType | undefined; + const head = cursor === undefined ? undefined : cursor.headPullInto; + if (cursor === undefined || head === undefined) { + throw new TypeError('No pending BYOB request'); + } + const state = getReadableStreamGetState(this.#stream); + if (state === 'closed') { + if (bytesWritten !== 0) { + throw new TypeError( + 'bytesWritten must be zero after the stream is closed' + ); + } + } else { + if (bytesWritten === 0) { + throw new TypeError( + 'bytesWritten must be non-zero while the stream is readable' + ); + } + if (head.bytesFilled + bytesWritten > head.byteLength) { + throw new RangeError( + 'bytesWritten exceeds the remaining space in the view' + ); + } + } + // Spec: the descriptor's buffer is re-transferred on every respond; + // the result view and any remainder view target the NEW buffer. + head.buffer = ArrayBufferPrototypeTransfer(head.buffer); + this.#invalidateByobRequest(); + if (state === 'closed') { + // respond(0)-while-closed: commit all pending descriptors with + // { done: true, value: filled-so-far view }. + cursor.commitPullIntosOnClose(); + } else { + // EXPECTED-LENGTH CONTRACT: respond() bytes count toward the total + // (the second ingress path alongside enqueue). + this.#accountDelivery(bytesWritten); + cursor.respondBYOB(bytesWritten); + // The respond may have drained the queue with close requested. + this.#maybeCloseStream(); + this.#callPullIfNeeded(); + } + } + + // Exact-total accounting (see #expectedLength). On overflow (more bytes + // than declared), errors the readable side and throws — the throw + // propagates to the enqueue/respond caller. The readable must be + // errored explicitly here because when the caller is an external sink + // (e.g. IdentityTransformStream.sinkWrite), the throw only errors the + // writable side; without this.error() the readable would hang forever. + #accountDelivery(byteLength: number): void { + if (this.#expectedLength === undefined) { + // When there is no expectedLength, we don't need to perform any accounting. + return; + } + const delivered = this.#bytesDelivered + BigInt(byteLength); + if (delivered > this.#expectedLength) { + const e = new RangeError( + 'byte source delivered more bytes than its declared expectedLength' + ); + this.error(e); + throw e; + } + this.#bytesDelivered = delivered; + } + + // byobRequest.respondWithNewView(view). + #respondWithNewView(view: ArrayBufferView): void { + const cursor = this.#queue.singleCursor as ByteStreamCursorType | undefined; + const head = cursor === undefined ? undefined : cursor.headPullInto; + if (cursor === undefined || head === undefined) { + throw new TypeError('No pending BYOB request'); + } + const info = getViewInfo(view); + const state = getReadableStreamGetState(this.#stream); + if (state === 'closed') { + if (info.byteLength !== 0) { + throw new TypeError( + 'The view must be zero-length after the stream is closed' + ); + } + } else if (info.byteLength === 0) { + throw new TypeError( + 'The view must be non-zero-length while the stream is readable' + ); + } + if (head.byteOffset + head.bytesFilled !== info.byteOffset) { + throw new RangeError( + 'The view byteOffset must match the bytes already filled' + ); + } + // Spec step: "If firstDescriptor's buffer byte length ≠ + // view.[[ViewedArrayBuffer]].[[ArrayBufferByteLength]], throw a + // RangeError." We compare against the stored bufferByteLength (not + // head.buffer.byteLength, which may be 0 if the buffer was transferred). + if ( + head.bufferByteLength !== ArrayBufferPrototypeByteLengthGet(info.buffer) + ) { + throw new RangeError( + "The view's buffer must have the same byteLength as the request" + ); + } + if (head.bytesFilled + info.byteLength > head.byteLength) { + throw new RangeError('The view exceeds the remaining space'); + } + head.buffer = ArrayBufferPrototypeTransfer(info.buffer); + this.#invalidateByobRequest(); + if (state === 'closed') { + cursor.commitPullIntosOnClose(); + } else { + // EXPECTED-LENGTH CONTRACT: counts toward the total like respond(). + this.#accountDelivery(info.byteLength); + cursor.respondBYOB(info.byteLength); + // The respond may have drained the queue with close requested. + this.#maybeCloseStream(); + this.#callPullIfNeeded(); + } + } + + #invalidateByobRequest(): void { + if (this.#byobRequest !== null) { + invalidateByobRequestObject(this.#byobRequest); + this.#byobRequest = null; + } + } + + #canCloseOrEnqueue(): boolean { + return ( + !this.#closeRequested && + this.#cancelPromise === undefined && + getReadableStreamGetState(this.#stream) === 'readable' + ); + } + + #maybeCloseStream(): void { + if (!this.#closeRequested) return; + // Check ALL live consumer streams (tee branches + the parent). + // Mirror of the default controller's logic — see that for comments. + let anyOpen = false; + const owners = this.#queue.getLiveOwners(); + for (let i = 0; i < owners.length; i++) { + const owner = owners[i] as ReadableStream; + const cursor = getReadableStreamConsumer(owner) as + | QueueCursorType + | undefined; + if (cursor === undefined) continue; + if (this.#queue.getEntry(cursor.position) === CLOSE_SENTINEL) { + readableStreamClose(owner); + } else { + anyOpen = true; + } + } + const parentCursor = getReadableStreamConsumer(this.#stream) as + | QueueCursorType + | undefined; + if (parentCursor !== undefined) { + if (this.#queue.getEntry(parentCursor.position) === CLOSE_SENTINEL) { + readableStreamClose(this.#stream); + } else { + anyOpen = true; + } + } + if (!anyOpen) { + this.#clearAlgorithms(); + } + } + + #shouldCallPull(): boolean { + if (!this.#started) return false; + if (!this.#canCloseOrEnqueue()) return false; + return this.#queue.desiredSize > 0 || this.#queue.anyCursorHasPendingRead(); + } + + #callPullIfNeeded(): void { + if (!this.#shouldCallPull()) return; + if (this.#pulling) { + this.#pullAgain = true; + return; + } + this.#pulling = true; + const pullAlgorithm = this.#pullAlgorithm; + const result = + pullAlgorithm === undefined ? PromiseResolve() : pullAlgorithm(); + PromisePrototypeThen( + result, + () => { + this.#pulling = false; + if (this.#pullAgain) { + this.#pullAgain = false; + this.#callPullIfNeeded(); + } + }, + (e: unknown) => { + this.error(e); + } + ); + } + + #cancelSteps(reason: unknown): Promise { + if (this.#cancelPromise !== undefined) return this.#cancelPromise; + this.#invalidateByobRequest(); + const cancelAlgorithm = this.#cancelAlgorithm; + this.#clearAlgorithms(); + this.#cancelPromise = + cancelAlgorithm === undefined + ? (PromiseResolve() as Promise) + : cancelAlgorithm(reason); + return this.#cancelPromise; + } + + #clearAlgorithms(): void { + this.#pullAlgorithm = undefined; + this.#cancelAlgorithm = undefined; + } + + [SymbolToStringTag] = 'ReadableByteStreamController'; +} + +function setupReadableByteStreamControllerFromUnderlyingSource( + stream: ReadableStream, + underlyingSource: UnderlyingSource, + highWaterMark: number +): ReadableByteStreamControllerType { + return new ReadableByteStreamController( + kPrivateSymbol, + stream as unknown as ReadableStream, + underlyingSource as UnderlyingByteSource, + highWaterMark + ); +} + +function setupReadableStreamDefaultControllerFromUnderlyingSource( + stream: ReadableStream, + underlyingSource: UnderlyingSource, + sizeAlgorithm: (chunk: R) => number, + highWaterMark: number +): ReadableStreamDefaultControllerType { + return new ReadableStreamDefaultController( + kPrivateSymbol, + stream, + underlyingSource as UnderlyingDefaultSource, + sizeAlgorithm, + highWaterMark + ) as ReadableStreamDefaultControllerType; +} + +export interface DrainingReadResult { + chunks: R[]; // bulk chunks drained from the queue, in order + done: boolean; // true if the close sentinel was reached +} + +// Null-prototype factory for DrainingReadResult — same rationale as +// createReadResult (see queue.ts): prevents Object.prototype.then +// interception when the result flows through promise resolution. +function createDrainResult( + chunks: R[], + done: boolean +): DrainingReadResult { + const result = ObjectCreate(null) as DrainingReadResult; + result.chunks = chunks; + result.done = done; + return result; +} + +// The draining-read core: collect everything buffered at the cursor in one +// shot; when nothing is buffered, fall back to a single pending read and +// then sweep whatever arrived alongside it. After draining, the controller +// is pumped so a synchronous source can immediately refill. +async function drainingReaderReadInternal( + reader: object, + stream: ReadableStream, + maxSize?: number +): Promise> { + setReadableStreamDisturbed(stream); + const state = getReadableStreamGetState(stream); + if (state === 'closed') return createDrainResult([], true); + if (state === 'errored') throw getReadableStreamStoredError(stream); + // BACKEND-BLIND: drains whatever the consumer has buffered (a native + // consumer never has a backlog beyond its overflow slot — the + // empty-then-wait fallback below covers it naturally). + const consumer = getReadableStreamConsumer(stream); + if (consumer === undefined) return createDrainResult([], true); + const controller = getReadableStreamController(stream); + + let result = consumer.drain(maxSize) as DrainingReadResult; + if (result.chunks.length === 0 && !result.done) { + // Nothing buffered — wait for one chunk through the normal pending-read + // machinery (FIFO with everything else), then sweep the rest. + const promise = consumer.read(reader); + if (controller !== undefined) controllerPullIfNeeded(controller); + const single = await promise; + if (single.done) { + result = createDrainResult([], true); + } else { + const chunks: R[] = [single.value as R]; + const more = consumer.drain(maxSize) as DrainingReadResult; + for (let i = 0; i < more.chunks.length; i++) { + ArrayPrototypePush(chunks, more.chunks[i] as R); + } + result = createDrainResult(chunks, more.done); + } + } + // Pump: draining freed queue space, so the source may pull again + // immediately (synchronous sources refill before we return). + if (controller !== undefined) controllerPullIfNeeded(controller); + if (result.done) { + if (controller !== undefined) controllerMaybeCloseStream(controller); + readableStreamClose(stream); + } + return result; +} + +// Bulk reader for pipeTo and the C++ bridge (design doc "Draining Reads"). +// Lock-based exclusivity: while held, no default/BYOB reader can interfere +// with the cursor. Internal-only for now (not exposed via getReader(); +// see Open Question 3). +class ReadableStreamDrainingReader { + #base: ReadableStreamReaderBase; + + static { + const prev = getReaderBase; + getReaderBase = (reader: object) => { + if (#base in reader) { + return reader.#base as unknown as ReadableStreamReaderBase; + } + return prev(reader); + }; + } + + constructor(stream: ReadableStream) { + this.#base = new ReadableStreamReaderBase(); + if (isReadableStreamLocked(stream)) { + throw new TypeError('Cannot get a reader for a stream that is locked'); + } + setReadableStreamReader( + stream, + this as unknown as ReadableStreamReaderType + ); + initializeReadableStreamGenericReader(stream, this.#base); + } + + get closed(): Promise { + try { + return getGenericReaderClosedPromise(this); + } catch (e) { + return PromiseReject(e) as Promise; + } + } + + // The non-standard expectedLength pass-through for the C++ bridge: the + // TOTAL bytes the underlying source declared it will produce + // (undefined = unknown → chunked encoding). A construction-time value, + // cached on the controller; backend-blind via the chained helper + // (byte/native report their cached value; default streams report + // undefined). Returns undefined after release. + get expectedLength(): bigint | undefined { + const stream = getReaderStream(this); + if (stream === undefined) return undefined; + const controller = getReadableStreamController(stream); + if (controller === undefined) return undefined; + return getControllerExpectedLength(controller); + } + + cancel(reason?: unknown): Promise { + try { + return cancelReadableStreamGenericReader(this, reason); + } catch (e) { + return PromiseReject(e) as Promise; + } + } + + // Drains all currently buffered chunks (up to the soft limit maxSize, in + // strategy size units — bytes for byte streams). Always makes progress: + // waits for at least one chunk when nothing is buffered. + async read( + options: { maxSize?: number } = {} + ): Promise> { + const stream = getReaderStream(this); + if (stream === undefined) { + throw new TypeError('This reader has been released'); + } + let maxSize: number | undefined; + if (options.maxSize !== undefined) { + maxSize = +options.maxSize; + if (NumberIsNaN(maxSize) || maxSize <= 0) { + throw new TypeError('options.maxSize must be a positive number'); + } + } + return drainingReaderReadInternal(this, stream, maxSize); + } + + releaseLock(): void { + if (!isReaderBoundToStream(this)) return; + readableStreamReaderGenericRelease(this); + } + + [SymbolToStringTag] = 'ReadableStreamDrainingReader'; +} + +// The pipe (spec ReadableStreamPipeTo). Internal operations only on both +// ends — locks are held for the duration. Backpressure is honored by +// awaiting the writer's ready promise before every read; individual writes +// are intentionally not awaited (the close path queues behind them, and +// write failures surface through the destination's closed promise). +// +// The shutdown actions (abort/cancel/close) are started immediately rather +// than after explicitly waiting for in-flight sink operations: the +// writable state machine already serializes abort/close against in-flight +// writes via its pendingAbortRequest / close-marker machinery. +function pipeToInternal( + source: ReadableStream, + destination: WritableStreamType, + options: StreamPipeOptions = {} +): Promise { + // Spec-mandated read order (§4.9.1): preventAbort, preventCancel, + // preventClose, signal. WPT piping/throwing-options.any.js verifies + // that getter side-effects occur in exactly this sequence. + const preventAbort = !!options.preventAbort; + const preventCancel = !!options.preventCancel; + const preventClose = !!options.preventClose; + const signal = options.signal; + if (signal !== undefined) { + // Brand check: the captured getter throws for non-AbortSignal objects. + try { + AbortSignalAbortedGet(signal); + } catch { + throw new TypeError('options.signal must be an AbortSignal'); + } + } + + // Lock both ends. The pipe uses the draining reader: every pump + // iteration moves ALL buffered chunks to the destination in one step + // (per-chunk promise overhead only when the source is empty). + const reader = new ReadableStreamDrainingReader(source); + const writer = writableInternals.acquireWriter(destination); + setReadableStreamDisturbed(source); + + const { promise, resolve, reject } = + PromiseWithResolvers() as PromiseWithResolversType; + + let shuttingDown = false; + let abortAlgorithm: (() => void) | undefined; + + const finalize = (error?: { reason: unknown }): void => { + writableInternals.writerRelease(writer); + readableStreamReaderGenericRelease(reader); + if (signal !== undefined && abortAlgorithm !== undefined) { + EventTargetRemoveEventListener(signal, 'abort', abortAlgorithm); + } + if (error !== undefined) { + reject(error.reason); + } else { + resolve(); + } + }; + + // Shared "last write" promise: the pump updates this before each batch + // of writes so that shutdownWithAction can wait for them to be acknowledged. + let lastWriteSettled: Promise = PromiseResolve( + undefined + ) as Promise; + + const shutdownWithAction = ( + action: (() => Promise) | undefined, + error?: { reason: unknown } + ): void => { + if (shuttingDown) return; + shuttingDown = true; + + const runAction = (): void => { + if (action === undefined) { + finalize(error); + return; + } + PromisePrototypeThen( + action(), + () => finalize(error), + (actionError: unknown) => finalize({ reason: actionError }) + ); + }; + + // Spec: "If dest.[[state]] is 'writable' and + // WritableStreamCloseQueuedOrInFlight(dest) is false, wait until + // every chunk that has been written has been acknowledged." + const destState = writableInternals.getState(destination); + if ( + destState === 'writable' && + !writableInternals.closeQueuedOrInFlight(destination) + ) { + PromisePrototypeThen(lastWriteSettled, runAction, runAction); + } else { + runAction(); + } + }; + + const onSourceErrored = (e: unknown): void => { + shutdownWithAction( + preventAbort + ? undefined + : () => writableInternals.writableStreamAbort(destination, e), + { reason: e } + ); + }; + + const onDestErrored = (e: unknown): void => { + shutdownWithAction( + preventCancel ? undefined : () => readableStreamCancel(source, e), + { reason: e } + ); + }; + + const onDestClosedEarly = (): void => { + const e = new TypeError('Destination closed before the pipe completed'); + shutdownWithAction( + preventCancel ? undefined : () => readableStreamCancel(source, e), + { reason: e } + ); + }; + + const onSourceDone = (): void => { + shutdownWithAction( + preventClose + ? undefined + : () => writableInternals.writerCloseWithErrorPropagation(writer) + ); + }; + + if (signal !== undefined) { + abortAlgorithm = () => { + const abortReason = AbortSignalReasonGet(signal); + const actions: (() => Promise)[] = []; + if (!preventAbort) { + ArrayPrototypePush(actions, () => + // Spec step 14.1.3: only abort if dest is still writable. + writableInternals.getState(destination) === 'writable' + ? writableInternals.writableStreamAbort(destination, abortReason) + : (PromiseResolve(undefined) as Promise) + ); + } + if (!preventCancel) { + ArrayPrototypePush(actions, () => + // Spec step 14.1.4: only cancel if source is still readable. + getReadableStreamGetState(source) === 'readable' + ? readableStreamCancel(source, abortReason) + : (PromiseResolve(undefined) as Promise) + ); + } + shutdownWithAction( + actions.length === 0 + ? undefined + : () => { + // Settle when all actions settle; reject with the first + // failure (spec: shutdown actions run in parallel). + let remaining = actions.length; + let failed: { reason: unknown } | undefined; + const all = + PromiseWithResolvers() as PromiseWithResolversType; + for (let i = 0; i < actions.length; i++) { + const action = actions[i] as () => Promise; + PromisePrototypeThen( + action(), + () => { + if (--remaining === 0) { + if (failed !== undefined) { + all.reject(failed.reason); + } else { + all.resolve(); + } + } + }, + (e: unknown) => { + failed ??= { reason: e }; + if (--remaining === 0) all.reject(failed.reason); + } + ); + } + return all.promise; + }, + { reason: abortReason } + ); + }; + if (AbortSignalAbortedGet(signal)) { + abortAlgorithm(); + } else { + EventTargetAddEventListener(signal, 'abort', abortAlgorithm); + } + } + + // ---- Synchronous pre-checks (spec conditions 1-4) ---- + // The spec's "in parallel" conditions must be "applied in order": + // 1. Source errored → abort dest + // 2. Dest errored → cancel source + // 3. Source closed → close dest + // 4. Dest close-queued/closed → TypeError + cancel source + // When streams are already in terminal states at pipe start, we must + // honour this priority synchronously — the async pump loop and + // microtask-scheduled reactive handlers cannot guarantee spec ordering. + if (!shuttingDown) { + const srcState = getReadableStreamGetState(source); + if (srcState === 'errored') { + // Condition 1: source already errored → abort dest with source error + onSourceErrored(getReadableStreamStoredError(source)); + } else { + const destState = writableInternals.getState(destination); + if (destState === 'errored') { + // Condition 2: dest already errored → cancel source with dest error + onDestErrored(writableInternals.getStoredError(destination)); + } else if (srcState === 'closed') { + // Condition 3: source already closed → close dest + onSourceDone(); + } else if ( + writableInternals.closeQueuedOrInFlight(destination) || + destState === 'closed' + ) { + // Condition 4: dest already closing/closed → TypeError + onDestClosedEarly(); + } + } + } + + // ---- Reactive shutdown triggers (spec: "in parallel") ---- + // The pump loop must not block indefinitely on backpressure or reads + // when the other end closes, errors, or the signal fires. We use a + // shared "poke" promise that is resolved whenever the pump should + // re-evaluate its state (source close/error, dest error). + let pokeResolve: (() => void) | undefined; + let pokePromise: Promise | undefined; + + const resetPoke = (): void => { + const pwr = PromiseWithResolvers() as PromiseWithResolversType; + pokePromise = pwr.promise; + pokeResolve = pwr.resolve; + }; + resetPoke(); + + const poke = (): void => { + const r = pokeResolve; + if (r !== undefined) { + pokeResolve = undefined; + r(); + } + }; + + // Forward close/error propagation from the source. + PromisePrototypeThen( + getGenericReaderClosedPromise(reader), + () => { + if (!shuttingDown) poke(); // source closed — wake the pump + }, + (e: unknown) => { + if (!shuttingDown) { + onSourceErrored(e); + poke(); + } + } + ); + + // Backward error propagation: destination errors cancel the source. + PromisePrototypeThen( + writableInternals.getWriterClosedPromise(writer), + undefined, + (e: unknown) => { + if (!shuttingDown) { + onDestErrored(e); + poke(); + } + } + ); + + const pump = async (): Promise => { + while (!shuttingDown) { + // Wait for backpressure to drop, but also react to shutdown triggers. + // SafePromise.race avoids prototype-then interception. + try { + await SafePromiseRace([ + writableInternals.getWriterReadyPromise(writer), + pokePromise, + ]); + } catch (e) { + if (!shuttingDown) onDestErrored(e); + return; + } + if (shuttingDown) return; + resetPoke(); // arm the next poke for the next iteration + + // Spec conditions must be checked in priority order (1-4): + // source error > dest error > source close > dest close-early. + const srcState = getReadableStreamGetState(source); + if (srcState === 'errored') { + // Condition 1: source errored → abort dest + if (!shuttingDown) + onSourceErrored(getReadableStreamStoredError(source)); + return; + } + const destState = writableInternals.getState(destination); + if (destState === 'errored') { + // Condition 2: dest errored → cancel source + if (!shuttingDown) + onDestErrored(writableInternals.getStoredError(destination)); + return; + } + if (srcState === 'closed') { + // Condition 3: source closed → close dest + onSourceDone(); + return; + } + if ( + writableInternals.closeQueuedOrInFlight(destination) || + destState === 'closed' + ) { + // Condition 4: dest closing/closed → TypeError + onDestClosedEarly(); + return; + } + + let result: DrainingReadResult; + try { + result = await drainingReaderReadInternal(reader, source); + } catch (e) { + if (!shuttingDown) onSourceErrored(e); + return; + } + if (shuttingDown) return; + resetPoke(); + + // Move the whole batch; the destination FIFO preserves order and the + // close marker (if done) queues behind these writes. + const chunks = result.chunks; + for (let i = 0; i < chunks.length; i++) { + const writePromise = writableInternals.writerWrite( + writer, + chunks[i] as R + ); + markPromiseHandled(writePromise); + // Track the last write so shutdownWithAction can wait for it. + // The writable serializes writes, so when this settles all + // preceding writes have already settled. + lastWriteSettled = PromisePrototypeThen( + writePromise, + () => {}, + () => {} + ) as Promise; + } + if (result.done) { + onSourceDone(); + return; + } + } + }; + markPromiseHandled(pump()); + + return promise; +} + +class ReadableStream { + #controller?: + | ReadableStreamDefaultControllerType + | ReadableByteStreamControllerType + | NativeReadableStreamControllerType + | undefined; + #reader?: + | ReadableStreamDefaultReaderType + | ReadableStreamBYOBReaderType + | undefined; + // The stream's own CONSUMER — the backend fence (see queue.ts and + // native-stream-integration.md §10). A QueueCursor/ByteStreamCursor for + // queued (JS-backed) streams; a NativePullConduit for native streams. + // The stream owns it (readers only borrow it while locked); created + // during controller setup, removed on cancel/tee. + #consumer?: StreamConsumerType | undefined; + // Tee branches get a composite-cancel hook (spec ReadableStreamTee): the + // source is cancelled with [reason1, reason2] only when ALL branches have + // cancelled, and every branch's cancel() promise settles together at that + // point. undefined for non-branch streams. + #onCancel?: ((reason: unknown) => Promise) | undefined; + // Tee-branch lifecycle: called when this branch's stream closes or errors + // so the shared cancel-promise can settle if any sibling was already + // cancelled. Mirrors the spec's "resolve cancelPromise" in the done/error + // paths of ReadableStreamDefaultTee's pull loop. + #onBranchSettled?: (() => void) | undefined; + #disturbed: boolean = false; + #state: 'readable' | 'closed' | 'errored' = 'readable'; + #storedError?: unknown; + + static { + isReadableStream = (value: unknown) => { + return isActualObject(value) && #state in value; + }; + + isReadableStreamLocked = (stream: ReadableStream) => { + // The spec definition (a reader is attached) works unchanged for this + // implementation: tee() keeps the parent locked permanently via a + // real, never-exposed internal reader, and the pipe/draining paths + // hold ordinary reader locks. + return stream.#reader !== undefined; + }; + + readableStreamCancel = (stream: ReadableStream, reason?: unknown) => { + stream.#disturbed = true; + const state = stream.#state; + if (state === 'closed') { + return PromiseResolve() as Promise; + } + if (state === 'errored') { + return PromiseReject(stream.#storedError) as Promise; + } + // Close FIRST (spec ReadableStreamCancel): transitions state and + // resolves the attached reader's closedPromise. + readableStreamClose(stream); + const consumer = stream.#consumer; + const onCancel = stream.#onCancel; + const controller = stream.#controller; + let cancelPromise: Promise = PromiseResolve() as Promise; + // BACKEND-BLIND: the STREAM layer owns the source-cancel POLICY + // (tee composite hook vs direct controller cancel) and hands it to + // the consumer, which owns the teardown MECHANICS (resolve reads as + // done, ordering vs removal, last-consumer determination). See + // StreamConsumer.cancelStream in queue.ts. + const decideSourceCancel = (isLastConsumer: boolean): Promise => { + if (onCancel !== undefined) { + // Tee branch: the composite-cancel hook collects this branch's + // vote+reason; the source is cancelled only once every sibling + // has cancelled (spec ReadableStreamTee). The returned promise + // stays PENDING until then. + return onCancel(reason); + } + if (isLastConsumer && controller !== undefined) { + // Only the LAST consumer's cancel reaches the underlying source. + return controllerCancelSteps(controller, reason); + } + return PromiseResolve() as Promise; + }; + if (consumer !== undefined) { + cancelPromise = consumer.cancelStream(reason, decideSourceCancel); + stream.#consumer = undefined; + } else if (onCancel !== undefined) { + cancelPromise = onCancel(reason); + } + // Per spec the returned promise fulfills with undefined. + return PromisePrototypeThen( + cancelPromise, + () => undefined + ) as Promise; + }; + + acquireReadableStreamDefaultReader = (stream: ReadableStream) => { + return new ReadableStreamDefaultReader(stream); + }; + + acquireReadableStreamBYOBReader = (stream: ReadableStream) => { + return new ReadableStreamBYOBReader( + stream as unknown as ReadableStream + ); + }; + + readableStreamPipeThroughTo = ( + source: ReadableStream, + destination: WritableStreamType, + options?: StreamPipeOptions + ) => { + return pipeToInternal(source, destination, options); + }; + + // BACKEND-DISPATCH: tee is one of the five sanctioned dispatch points + // (native-stream-integration.md §10). The native branch runs first: + // the source's tee hook produces a PAIR of new native source objects + // (leaving the original source closed), each wrapped in a fresh + // ReadableStream via ordinary construction. Branches are fully + // independent — no shared consumer and no composite-cancel wiring + // (deliberate divergence from the queued model below: branch cancels + // go to each branch's OWN source). The parent becomes the same inert + // locked shell as in the queued model. + readableStreamTee = (stream: ReadableStream) => { + const controller = stream.#controller; + if (isNativeController(controller)) { + if (stream.#state !== 'readable') { + // Closed/errored native parents produce two branches in the + // same state, mirroring the queued behavior below — without + // touching the (closed) source. + const b1 = new ReadableStream(kPrivateSymbol as never); + const b2 = new ReadableStream(kPrivateSymbol as never); + b1.#state = stream.#state; + b2.#state = stream.#state; + b1.#storedError = stream.#storedError; + b2.#storedError = stream.#storedError; + return [b1, b2] as [ReadableStream, ReadableStream]; + } + // Sources from the tee hook are full native sources; ordinary + // construction validates and wires each branch. + const [source1, source2] = nativeControllerTeeSource(controller); + const branch1 = new ReadableStream(source1 as UnderlyingSource); + const branch2 = new ReadableStream(source2 as UnderlyingSource); + stream.#consumer = undefined; + if (!isReadableStreamLocked(stream)) { + acquireReadableStreamDefaultReader(stream); + } + return [branch1, branch2] as [ReadableStream, ReadableStream]; + } + const state = stream.#state; + + // Branch streams are shells wired to the SHARED controller — same + // queue, per-branch cursors. + const branch1 = new ReadableStream(kPrivateSymbol as never); + const branch2 = new ReadableStream(kPrivateSymbol as never); + branch1.#controller = controller; + branch2.#controller = controller; + + // QUEUED INVARIANT: this branch is queued-backend territory — the + // consumer is necessarily a QueueCursor (position/byteOffset/queue + // are cursor-only concepts); sanctioned cast. + const cursor = stream.#consumer as QueueCursorType | undefined; + + if (state === 'errored') { + branch1.#state = 'errored'; + branch2.#state = 'errored'; + branch1.#storedError = stream.#storedError; + branch2.#storedError = stream.#storedError; + } else if (state === 'closed') { + branch1.#state = 'closed'; + branch2.#state = 'closed'; + } else if (cursor !== undefined) { + const queue = cursor.queue; + const isBytes = + controller !== undefined && isByteStreamController(controller); + // Each branch forks at the original cursor's position AND + // byteOffset — partial entry consumption survives the fork. Byte + // streams need ByteStreamCursor (BYOB reads on branches). + // + // ORDER MATTERS: add the branch cursors BEFORE removing the + // original — removing the sole cursor first would fire the + // all-cursors-gone hook and cancel the underlying source mid-tee. + const totalSize = cursor.remainingSize; + branch1.#consumer = isBytes + ? new ByteStreamCursor( + queue, + branch1, + cursor.position, + cursor.byteOffset, + totalSize + ) + : new QueueCursor( + queue, + branch1, + cursor.position, + cursor.byteOffset, + totalSize + ); + branch2.#consumer = isBytes + ? new ByteStreamCursor( + queue, + branch2, + cursor.position, + cursor.byteOffset, + totalSize + ) + : new QueueCursor( + queue, + branch2, + cursor.position, + cursor.byteOffset, + totalSize + ); + queue.removeCursor(cursor); + stream.#consumer = undefined; + } + + // Composite cancel (spec ReadableStreamTee): each branch's cancel() + // records its reason and returns a SHARED promise that settles only + // once both branches have cancelled — at which point the source is + // cancelled with [reason1, reason2]. Re-tees relay through the + // parent branch's own hook so nesting composes. + const parentOnCancel = stream.#onCancel; + let canceled1 = false; + let canceled2 = false; + let canceling1 = false; + let canceling2 = false; + let reason1: unknown; + let reason2: unknown; + const cancelHolder = + PromiseWithResolvers() as PromiseWithResolversType; + const performSourceCancel = (composite: unknown): Promise => { + if (parentOnCancel !== undefined) { + // Re-tee: relay through the parent branch's hook. The composite + // is branded, so the parent-level composition FLATTENS it — the + // final AggregateError carries every leaf reason at one level. + return parentOnCancel(composite); + } + if (controller === undefined) { + return PromiseResolve() as Promise; + } + return controllerCancelSteps(controller, composite); + }; + const makeBranchCancel = (which: 1 | 2) => { + return (reason: unknown): Promise => { + if (which === 1) { + canceling1 = true; + canceled1 = true; + reason1 = reason; + } else { + canceling2 = true; + canceled2 = true; + reason2 = reason; + } + if (canceled1 && canceled2) { + // Resolving with the source-cancel promise adopts it: both + // branches' pending cancel() promises settle with its outcome. + cancelHolder.resolve( + performSourceCancel(makeCompositeCancelReason(reason1, reason2)) + ); + } + // Clear the canceling guard after the synchronous cancel path + // completes. The guard prevents onBranchSettled from resolving + // cancelHolder during the cancel itself (readableStreamCancel + // calls readableStreamClose before onCancel). After this point, + // if the sibling branch closes/errors from the source, the + // cancelHolder should resolve (spec step 14.b.v / 14.c.ii). + if (which === 1) { + canceling1 = false; + } else { + canceling2 = false; + } + return cancelHolder.promise; + }; + }; + branch1.#onCancel = makeBranchCancel(1); + branch2.#onCancel = makeBranchCancel(2); + + // Spec ReadableStreamDefaultTee: when the underlying reader signals + // done (or error), the cancel promise is resolved if either branch + // was already cancelled (steps 14.b.v / 14.c.ii). In our shared- + // queue model the equivalent is: a branch stream transitions to + // closed/errored via readableStreamClose/readableStreamError while + // the other branch was already cancelled. + // + // Guard: only resolve when the SOURCE closes/errors a branch whose + // sibling was already cancelled. Do NOT resolve when a branch is + // being cancelled itself — readableStreamCancel calls + // readableStreamClose before onCancel runs, and onCancel handles + // cancelHolder via performSourceCancel. + const onBranchSettled = (): void => { + if ((canceled1 && !canceling1) || (canceled2 && !canceling2)) { + cancelHolder.resolve(); + } + }; + setOnBranchSettled(branch1, onBranchSettled); + setOnBranchSettled(branch2, onBranchSettled); + + // Per spec, tee() locks the original permanently. Acquire a real + // reader (never exposed, so it can never be released) — the original + // becomes an inert locked shell that retains only enough state to + // report locked === true. + if (!isReadableStreamLocked(stream)) { + acquireReadableStreamDefaultReader(stream); + } + + return [branch1, branch2] as [ReadableStream, ReadableStream]; + }; + + getReadableStreamGetState = (stream: ReadableStream) => { + return stream.#state; + }; + + getReadableStreamIsDisturbed = (stream: ReadableStream) => { + return stream.#disturbed; + }; + + getReadableStreamStoredError = (stream: ReadableStream) => { + return stream.#storedError; + }; + + setReadableStreamState = ( + stream: ReadableStream, + state: 'readable' | 'closed' | 'errored' + ) => { + stream.#state = state; + }; + + setReadableStreamDisturbed = (stream: ReadableStream) => { + stream.#disturbed = true; + }; + + setReadableStreamStoredError = ( + stream: ReadableStream, + error: unknown + ) => { + stream.#storedError = error; + }; + + setReadableStreamReader = ( + stream: ReadableStream, + reader: ReadableStreamReaderType | undefined + ) => { + stream.#reader = reader; + }; + + getReadableStreamController = (stream: ReadableStream) => { + return stream.#controller; + }; + + getReadableStreamReader = (stream: ReadableStream) => { + return stream.#reader; + }; + + getReadableStreamConsumer = (stream: ReadableStream) => { + return stream.#consumer; + }; + + setReadableStreamConsumer = ( + stream: ReadableStream, + consumer: StreamConsumerType | undefined + ) => { + stream.#consumer = consumer; + }; + + notifyBranchSettled = (stream: ReadableStream) => { + const cb = stream.#onBranchSettled; + if (cb !== undefined) cb(); + }; + + setOnBranchSettled = ( + stream: ReadableStream, + cb: (() => void) | undefined + ) => { + stream.#onBranchSettled = cb; + }; + + // BACKEND-DISPATCH: the native side of the chained-controller-helpers + // dispatch point (whose canonical marker sits at the default + // controller's static block). The queued controllers wrap the chain + // inside their own static blocks because their brands are private + // names; the native controller's brand lives across the module fence + // in native.ts, so its predicate and behaviors arrive as exported + // internals and the wrap happens here instead. This static block is + // the right host: it runs after both queued controller classes (file + // order — their static blocks assigned the implementations being + // wrapped) and before any stream can exist, and a static block (unlike + // straight-line module code) reads the chain lets without tripping + // definite-assignment analysis. + const prevPullIfNeeded = controllerPullIfNeeded; + controllerPullIfNeeded = (controller) => { + if (isNativeController(controller)) { + nativeControllerPullIfNeeded(controller); + } else { + prevPullIfNeeded(controller); + } + }; + + const prevCancelSteps = controllerCancelSteps; + controllerCancelSteps = (controller, reason) => { + if (isNativeController(controller)) { + return nativeControllerCancelSteps(controller, reason); + } + return prevCancelSteps(controller, reason); + }; + + const prevMaybeCloseStream = controllerMaybeCloseStream; + controllerMaybeCloseStream = (controller) => { + if (isNativeController(controller)) { + nativeControllerMaybeCloseStream(controller); + } else { + prevMaybeCloseStream(controller); + } + }; + + const prevOnReaderRelease = controllerOnReaderRelease; + controllerOnReaderRelease = (controller) => { + if (isNativeController(controller)) { + nativeControllerOnReaderRelease(controller); + } else { + prevOnReaderRelease(controller); + } + }; + + const prevGetExpectedLength = getControllerExpectedLength; + getControllerExpectedLength = (controller) => { + if (isNativeController(controller)) { + return nativeControllerExpectedLength(controller); + } + return prevGetExpectedLength(controller); + }; + + // BACKEND-DISPATCH point #4: JS-to-C++ extraction + // (prepareReadableStreamForCpp in the design doc). A single shared + // function installed as the value of kExtractNativeSource on every + // native-backed stream. The TypeWrapper detects the property's + // presence (own-property get, no JS code execution) and calls it + // to extract the native underlying source; absent means "queued, + // use DrainingReader". One-shot: subsequent calls throw. + extractNativeSource = function (this: ReadableStream): object { + if (!isReadableStream(this)) { + throw new TypeError('Illegal invocation'); + } + if (isReadableStreamLocked(this)) { + throw new TypeError( + 'Cannot extract a native source from a locked stream' + ); + } + if (this.#disturbed) { + throw new TypeError( + 'Cannot extract a native source from a disturbed stream' + ); + } + const controller = this.#controller; + if (!isNativeController(controller)) { + throw new TypeError('This stream is not backed by a native source'); + } + // Atomic: validate -> extract -> lock+disturb. No TOCTOU gap. + const source = nativeControllerExtractSource(controller); + this.#consumer = undefined; + this.#disturbed = true; + // Permanent lock via an internal reader (never exposed, never + // released) — same pattern as tee's parent locking. + acquireReadableStreamDefaultReader(this); + return source; + }; + } + + constructor( + underlyingSource: UnderlyingSource = {}, + strategy: QueuingStrategy = {} + ) { + // Internal shell creation (tee branches): skip controller setup + // entirely — the tee wiring attaches the SHARED controller and a + // forked cursor afterwards. The private symbol is unreachable from + // user code. + if ((underlyingSource as unknown) === kPrivateSymbol) { + return; + } + + // BACKEND-DISPATCH: stream construction (one of the five sanctioned + // dispatch points): native vs queued byte vs queued default. + // + // The native check runs FIRST: native sources are forbidden from + // declaring `type` (enforced during native extraction), so a + // native-marked source must never fall through to the queued + // branches. The strategy argument is PERMANENTLY ignored on the + // native branch: native pacing is purely demand-driven (the source + // contractually ignores desiredSize). + if (isNativeUnderlyingSource(underlyingSource)) { + const { controller, conduit } = createNativeReadableStreamParts( + underlyingSource, + { + // Stream-level transitions for the far side of the module + // fence. Both helpers are state-guarded, so redundant calls + // (e.g. the reader layer's done-result close racing the + // conduit's hook) are harmless. + closeStream: () => readableStreamClose(this), + errorStream: (reason: unknown) => readableStreamError(this, reason), + } + ); + this.#controller = controller; + // Sanctioned cast: the conduit is byte-oriented (Uint8Array), but + // the stream's R is caller-chosen; same shape as the queued cursor + // attachments in the controller constructors. + this.#consumer = conduit as unknown as StreamConsumerType; + // BACKEND-DISPATCH point #4: install the JS-to-C++ extraction + // marker. Own, non-enumerable, non-writable, non-configurable. + // The value is the shared extractor function (this-bound at call + // time). Bootstrap phase: a regular symbol; final: private API + // symbol (invisible to JS entirely). + ObjectDefineProperty(this, kExtractNativeSource, { + __proto__: null, + value: extractNativeSource, + } as PropertyDescriptor); + return; + } + + // --- WebIDL strategy dictionary conversion (BEFORE source reads) --- + // Per WebIDL, dictionary-typed arguments are converted (property reads + // happen) at the IDL layer before the constructor body runs. The + // `strategy` parameter is QueuingStrategy (a dictionary); the + // `underlyingSource` parameter is plain `object` (no dictionary + // conversion). We simulate this by reading strategy properties first. + const sizeFn = strategy.size; + let sizeAlgorithm: (chunk: R) => number; + if (sizeFn === undefined) { + sizeAlgorithm = () => 1; + } else if (typeof sizeFn !== 'function') { + throw new TypeError('strategy.size must be a function'); + } else { + const callSize = uncurryThis(sizeFn); + sizeAlgorithm = (chunk: R) => callSize(undefined, chunk); + } + const rawHWM = strategy.highWaterMark; + + // WebIDL: the 'type' member is a string enum — perform ToString() on + // non-undefined values so that objects with toString()/valueOf() work + // (per spec: "Let type be ? Get(underlyingSourceDict, "type"). + // If type is not undefined, set type to ? ToString(type).") + const rawType = underlyingSource.type; + const type = rawType === undefined ? undefined : `${rawType}`; + + if (type === 'bytes') { + // Byte streams: size() is forbidden, highWaterMark defaults to 0. + if (sizeFn !== undefined) { + throw new RangeError( + 'The strategy for a byte stream cannot have a size function' + ); + } + let highWaterMark = 0; + if (rawHWM !== undefined) { + highWaterMark = +rawHWM; + if (NumberIsNaN(highWaterMark) || highWaterMark < 0) { + throw new RangeError('Invalid highWaterMark'); + } + } + this.#controller = setupReadableByteStreamControllerFromUnderlyingSource( + this, + underlyingSource, + highWaterMark + ); + } else { + // The spec says, "Assert: underlyingSourceDict["type"] does not exist" + // but we're not going to be that strict about it. We'll assert only + // that its value is `undefined`. + if (type !== undefined) { + throw new TypeError(`Invalid underlying source type: ${type}`); + } + // Default streams: highWaterMark defaults to 1. + let highWaterMark = 1; + if (rawHWM !== undefined) { + highWaterMark = +rawHWM; + if (NumberIsNaN(highWaterMark) || highWaterMark < 0) { + throw new RangeError('Invalid highWaterMark'); + } + } + this.#controller = + setupReadableStreamDefaultControllerFromUnderlyingSource( + this, + underlyingSource, + sizeAlgorithm, + highWaterMark + ); + } + } + + get locked(): boolean { + if (!isReadableStream(this)) { + throw new TypeError('Illegal invocation'); + } + return isReadableStreamLocked(this); + } + + cancel(reason: unknown = undefined): Promise { + try { + if (!isReadableStream(this)) { + throw new TypeError('Illegal invocation'); + } + if (isReadableStreamLocked(this)) { + throw new TypeError('Cannot cancel a stream that is locked'); + } + return readableStreamCancel(this, reason) as Promise; + } catch (e) { + return PromiseReject(e) as Promise; + } + } + + getReader( + options: { mode?: 'byob' } | null = {} + ): ReadableStreamReaderType { + if (!isReadableStream(this)) { + throw new TypeError('Illegal invocation'); + } + // WebIDL dictionary conversion: null and undefined become {}, + // objects have their properties read, primitives are rejected. + if (options != null && !isActualObject(options)) { + throw new TypeError('Reader options must be an object'); + } + + // WebIDL: the 'mode' member is a string enum — perform ToString() on + // non-undefined values so that objects with toString()/valueOf() work. + const rawMode = isActualObject(options) ? options.mode : undefined; + const mode = rawMode === undefined ? undefined : `${rawMode}`; + if (mode === undefined) { + return acquireReadableStreamDefaultReader(this); + } + if (mode !== 'byob') { + throw new TypeError(`Invalid reader mode: ${mode}`); + } + return acquireReadableStreamBYOBReader(this); + } + + pipeThrough( + transform: TransformStreamType, + // WebIDL: optional dictionary — null/undefined both become {}. + // Default = {} preserves Function.length = 1 (IDL harness check). + options: StreamPipeOptions = {} + ): ReadableStreamType { + if (!isReadableStream(this)) { + throw new TypeError('Illegal invocation'); + } + if (isReadableStreamLocked(this)) { + throw new TypeError('Cannot pipe a stream that is locked'); + } + // WebIDL dictionary conversion: alphabetical member order means + // "readable" is read (and brand-checked) BEFORE "writable". The + // WPT pipe-through.any.js tests verify that a bad readable stops + // the writable getter from ever being called. + const readable = transform.readable; + if (!isReadableStream(readable)) { + throw new TypeError( + "Failed to execute 'pipeThrough': readable is not a ReadableStream" + ); + } + const writable = transform.writable; + if (writable.locked) { + throw new TypeError('Cannot pipe to a locked writable stream'); + } + // WebIDL: null coerces to {} for optional dictionaries. + if (options === null) options = {} as StreamPipeOptions; + if (!isActualObject(options)) { + throw new TypeError('Pipe options must be an object'); + } + const promise = readableStreamPipeThroughTo(this, writable, options); + markPromiseHandled(promise); + return readable; + } + + pipeTo( + destination: WritableStream, + // WebIDL: optional dictionary — null/undefined both become {}. + // Default = {} preserves Function.length = 1 (IDL harness check). + options: StreamPipeOptions = {} + ): Promise { + try { + if (!isReadableStream(this)) { + throw new TypeError('Illegal invocation'); + } + if (isReadableStreamLocked(this)) { + throw new TypeError('Cannot pipe a stream that is locked'); + } + if (destination.locked) { + throw new TypeError('Cannot pipe to a locked writable stream'); + } + // WebIDL: null coerces to {} for optional dictionaries. + if (options === null) options = {} as StreamPipeOptions; + if (!isActualObject(options)) { + throw new TypeError('Pipe options must be an object'); + } + // PIPE DISPATCH: if both source and dest are native-backed, take + // the fast path — extract both and let the sink's pipeFrom hook + // arrange the pipe entirely at the C++ layer. Both markers are + // own-property reads (non-destructive); extraction happens only + // after both are confirmed. If both are native, pipeFrom MUST + // exist (invariant — its absence is a contract violation, not a + // fallback trigger). + const sourceExtractor = ObjectGetOwnPropertyDescriptor( + this, + kExtractNativeSource + )?.value as ((this: ReadableStream) => object) | undefined; + const sinkExtractor = ObjectGetOwnPropertyDescriptor( + destination, + kExtractNativeSink + )?.value as ((this: object) => object) | undefined; + if (sourceExtractor !== undefined && sinkExtractor !== undefined) { + // Captured-call discipline: Function.prototype.call is patchable, + // so re-bind both extractors and the hook through uncurryThis + // (captured Reflect machinery) instead of .call(). + const nativeSource = uncurryThis(sourceExtractor)(this); + const nativeSink = uncurryThis(sinkExtractor)(destination) as Record< + string, + unknown + >; + const pipeFrom = nativeSink.pipeFrom as + | ((source: object, opts: StreamPipeOptions) => Promise) + | undefined; + if (pipeFrom === undefined) { + throw new TypeError( + 'Native sink is missing the required pipeFrom hook' + ); + } + return uncurryThis(pipeFrom)(nativeSink, nativeSource, options); + } + return readableStreamPipeThroughTo(this, destination, options); + } catch (e) { + return PromiseReject(e) as Promise; + } + } + + tee(): [ReadableStream, ReadableStream] { + if (!isReadableStream(this)) { + throw new TypeError('Illegal invocation'); + } + if (isReadableStreamLocked(this)) { + throw new TypeError('Cannot tee a stream that is locked'); + } + return readableStreamTee(this); + } + + static from( + iterable: Iterable | AsyncIterable | R + ): ReadableStream { + // We are intentionally a bit more lax in what we accept here. + // The spec says AsyncIterable. We allow Iterable as well. If + // a String or ArrayBufferView is passed, we will treat it as + // a single chunk, rather than an iterable of chunks. + if (typeof iterable === 'string' || isArrayBufferView(iterable)) { + // INTENTIONAL SPEC DIVERGENCE: The spec treats strings as + // iterables and iterates them code-point-by-code-point. We + // deliberately treat strings (and ArrayBufferViews) as single + // chunks instead — iterating a string through a stream one + // code point at a time is both surprising to users and has + // terrible performance. This causes the WPT test + // "ReadableStream.from accepts a string" to fail. + // The ArrayBufferView case avoids traversing the patchable + // %ArrayIteratorPrototype%. + const chunk = iterable as unknown as R; + return new ReadableStream({ + pull(controller: ReadableStreamDefaultControllerType) { + controller.enqueue(chunk); + controller.close(); + }, + }); + } + + // It can't be an iterable if it's not an actual object. + if (isActualObject(iterable)) { + // Check @@asyncIterator first, but only if the value is non-null. + // Per spec, a null/undefined @@asyncIterator is ignored and we + // fall through to @@iterator (WPT: "from ignores a null @@asyncIterator"). + const asyncMethod = + SymbolAsyncIterator in iterable + ? (iterable as AsyncIterable)[primordials.SymbolAsyncIterator] + : undefined; + if (asyncMethod != null) { + const asyncIterator: AsyncIterator = + uncurryThis(asyncMethod)(iterable); + if (!isActualObject(asyncIterator)) { + throw new TypeError('The iterator method must return an object'); + } + // HWM 0: the iterator's next() must only be called in response + // to a consumer read(), never eagerly (WPT: "calls next() after + // first read()"). + return new ReadableStream( + { + async pull(controller: ReadableStreamDefaultControllerType) { + // If the pull method throws, the stream will error. + const next = await asyncIterator.next(); + if (!isActualObject(next)) { + throw new TypeError('The result of next() must be an object'); + } + if (next.done) { + return controller.close(); + } + controller.enqueue(next.value); + }, + async cancel(reason?: unknown) { + const returnMethod = asyncIterator.return; + // Per spec, iterators without a return() method cancel + // silently. But if return exists and is not callable, + // cancel must reject with TypeError. + if (returnMethod === undefined) return; + if (typeof returnMethod !== 'function') { + throw new TypeError('Iterator return() is not a function'); + } + const ret = await uncurryThis(returnMethod)( + asyncIterator, + reason + ); + if (!isActualObject(ret)) { + throw new TypeError('The return method must return an object'); + } + }, + }, + { highWaterMark: 0 } + ); + } + + if (SymbolIterator in iterable) { + const method = (iterable as Iterable)[primordials.SymbolIterator]; + const syncIterator: Iterator = uncurryThis(method)(iterable); + if (!isActualObject(syncIterator)) { + throw new TypeError('The iterator method must return an object'); + } + // HWM 0: same rationale as the async path above — next() must + // be deferred until a consumer read() arrives. + return new ReadableStream( + { + // The spec uses GetIterator(asyncIterable, async) which + // wraps the sync iterator in an async-from-sync wrapper. + // That wrapper awaits each value via PromiseResolve, so + // e.g. an iterable of promises yields the resolved values, + // not the Promise objects. + async pull(controller: ReadableStreamDefaultControllerType) { + // If the pull method throws, the stream will error. + const next = syncIterator.next(); + if (!isActualObject(next)) { + throw new TypeError('The result of next() must be an object'); + } + // Await the value: the async-from-sync iterator wrapper + // resolves each value through PromiseResolve, which + // awaits thenables (including Promises). + const value = await next.value; + if (next.done) { + controller.close(); + return; + } + controller.enqueue(value as R); + }, + async cancel(reason?: unknown) { + const returnMethod = syncIterator.return; + if (returnMethod === undefined) return; + if (typeof returnMethod !== 'function') { + throw new TypeError('Iterator return() is not a function'); + } + const ret = uncurryThis(returnMethod)(syncIterator, reason); + if (!isActualObject(ret)) { + throw new TypeError('The return method must return an object'); + } + }, + }, + { highWaterMark: 0 } + ); + } + } + + throw new TypeError('The argument must be sync or async iterable'); + } + + values(options: { preventCancel?: boolean } = {}): AsyncIterableIterator { + if (!isReadableStream(this)) { + throw new TypeError('Illegal invocation'); + } + if (!isActualObject(options)) { + throw new TypeError('Options must be an object'); + } + const { preventCancel } = options; + + if (isReadableStreamLocked(this)) { + throw new TypeError('Cannot get an iterator for a stream that is locked'); + } + const reader = acquireReadableStreamDefaultReader(this); + + const iter = ObjectCreate(ReadableStreamAsyncIteratorPrototype); + iteratorStateMap.set(iter, { + reader, + preventCancel: !!preventCancel, + state: { done: false, current: undefined }, + started: false, + }); + return iter; + } + + [SymbolAsyncIterator](options?: { + preventCancel?: boolean; + }): AsyncIterableIterator { + return this.values(options); + } + + [SymbolToStringTag] = 'ReadableStream'; +} + +ObjectDefineProperties(ReadableStreamDefaultReader.prototype, { + closed: { enumerable: true }, + cancel: { enumerable: true }, + read: { enumerable: true }, + releaseLock: { enumerable: true }, +}); +ObjectDefineProperties(ReadableStreamBYOBReader.prototype, { + closed: { enumerable: true }, + cancel: { enumerable: true }, + read: { enumerable: true }, + releaseLock: { enumerable: true }, +}); +ObjectDefineProperties(ReadableStream, { + from: { enumerable: true }, +}); +ObjectDefineProperties(ReadableStream.prototype, { + locked: { enumerable: true }, + cancel: { enumerable: true }, + getReader: { enumerable: true }, + pipeThrough: { enumerable: true }, + pipeTo: { enumerable: true }, + tee: { enumerable: true }, + values: { enumerable: true }, + [SymbolAsyncIterator]: { enumerable: true }, +}); +ObjectDefineProperties(ReadableStreamDefaultController, { + length: { value: 0 }, +}); +ObjectDefineProperties(ReadableByteStreamController, { + length: { value: 0 }, +}); +ObjectDefineProperties(ReadableStreamBYOBRequest, { + length: { value: 0 }, +}); +ObjectDefineProperties(ReadableStreamDefaultController.prototype, { + close: { enumerable: true }, + enqueue: { enumerable: true }, + error: { enumerable: true }, + desiredSize: { enumerable: true }, +}); +ObjectDefineProperties(ReadableByteStreamController.prototype, { + close: { enumerable: true }, + enqueue: { enumerable: true }, + error: { enumerable: true }, + byobRequest: { enumerable: true }, + desiredSize: { enumerable: true }, +}); +ObjectDefineProperties(ReadableStreamBYOBRequest.prototype, { + respond: { enumerable: true }, + respondWithNewView: { enumerable: true }, + view: { enumerable: true }, +}); +ObjectDefineProperties(ReadableStreamDefaultController.prototype.enqueue, { + length: { value: 0 }, +}); +ObjectDefineProperties(ReadableByteStreamController.prototype.enqueue, { + length: { value: 1 }, +}); + +module.exports = { + ReadableStream, + ReadableStreamDefaultReader, + ReadableStreamBYOBReader, + ReadableStreamDefaultController, + ReadableByteStreamController, + ReadableStreamBYOBRequest, + // Internal-only (not re-exported by streams.ts): the bulk-read path for + // pipeTo and the C++ bridge. See Open Question 3 in the design doc for + // possible future public exposure. + ReadableStreamDrainingReader, + // Internal operations consumed by the TransformStream cancel/flush + // coordination (finishPromise guard). Unreachable from user code. + internalsForTransform: { + getState: (stream: ReadableStream) => + getReadableStreamGetState(stream), + getStoredError: (stream: ReadableStream) => + getReadableStreamStoredError(stream), + }, +}; diff --git a/src/per_isolate/webstreams/strategies.ts b/src/per_isolate/webstreams/strategies.ts new file mode 100644 index 00000000000..56d638eab38 --- /dev/null +++ b/src/per_isolate/webstreams/strategies.ts @@ -0,0 +1,98 @@ +'use strict'; + +// ByteLengthQueuingStrategy and CountQueuingStrategy (WHATWG Streams §7). + +const { ObjectDefineProperties, SymbolToStringTag, TypeError } = primordials; + +function isActualObject(value: unknown): boolean { + return value != null && typeof value === 'object'; +} + +// Per spec, the size function is the SAME function object for every +// strategy instance in the realm, and its name is 'size' (method shorthand +// gives the right .name and .length). The byteLength read is a deliberate, +// user-observable property access — that IS the specified behavior (the +// chunk is arbitrary user data, not a trusted view). +const byteLengthSize = { + size(chunk: ArrayBufferView): number { + return chunk.byteLength; + }, +}.size; + +const countSize = { + size(): number { + return 1; + }, +}.size; + +class ByteLengthQueuingStrategy { + #highWaterMark: number; + + // The init type is optional-shaped because user input is arbitrary; the + // required-member check is the explicit TypeError below. + constructor(init: { highWaterMark?: number }) { + if (!isActualObject(init)) { + throw new TypeError('init must be an object'); + } + if (init.highWaterMark === undefined) { + throw new TypeError('init.highWaterMark is required'); + } + // Stored as-converted WITHOUT range validation: per spec (and WPT), + // bogus values like NaN or -1 are accepted here and only rejected by + // ExtractHighWaterMark at stream construction time. + this.#highWaterMark = +init.highWaterMark; + } + + get highWaterMark(): number { + if (!(#highWaterMark in this)) throw new TypeError('Illegal invocation'); + return this.#highWaterMark; + } + + get size(): (chunk: ArrayBufferView) => number { + if (!(#highWaterMark in this)) throw new TypeError('Illegal invocation'); + return byteLengthSize; + } + + [SymbolToStringTag] = 'ByteLengthQueuingStrategy'; +} + +class CountQueuingStrategy { + #highWaterMark: number; + + constructor(init: { highWaterMark?: number }) { + if (!isActualObject(init)) { + throw new TypeError('init must be an object'); + } + if (init.highWaterMark === undefined) { + throw new TypeError('init.highWaterMark is required'); + } + this.#highWaterMark = +init.highWaterMark; + } + + get highWaterMark(): number { + if (!(#highWaterMark in this)) throw new TypeError('Illegal invocation'); + return this.#highWaterMark; + } + + get size(): () => number { + if (!(#highWaterMark in this)) throw new TypeError('Illegal invocation'); + return countSize; + } + + [SymbolToStringTag] = 'CountQueuingStrategy'; +} + +ObjectDefineProperties(ByteLengthQueuingStrategy.prototype, { + highWaterMark: { enumerable: true }, + size: { enumerable: true }, +}); + +ObjectDefineProperties(CountQueuingStrategy.prototype, { + highWaterMark: { enumerable: true }, + size: { enumerable: true }, +}); + +module.exports = { + ByteLengthQueuingStrategy, + CountQueuingStrategy, +}; diff --git a/src/per_isolate/webstreams/streams.ts b/src/per_isolate/webstreams/streams.ts new file mode 100644 index 00000000000..bf6025efef3 --- /dev/null +++ b/src/per_isolate/webstreams/streams.ts @@ -0,0 +1,72 @@ +'use strict'; + +const { + ReadableStream, + ReadableStreamDefaultReader, + ReadableStreamBYOBReader, + ReadableStreamDefaultController, + ReadableByteStreamController, + ReadableStreamBYOBRequest, + ReadableStreamDrainingReader, +} = require('webstreams/readable'); + +const { + ByteLengthQueuingStrategy, + CountQueuingStrategy, +} = require('webstreams/strategies'); + +const { + WritableStream, + WritableStreamDefaultWriter, + WritableStreamDefaultController, +} = require('webstreams/writable'); + +const { + TransformStream, + TransformStreamDefaultController, +} = require('webstreams/transform'); + +const { + IdentityTransformStream, + FixedLengthStream, +} = require('webstreams/identity'); + +const { TextEncoderStream, TextDecoderStream } = require('webstreams/encoding'); + +// TEMPORARY (native-stream-integration phase 2): the native-source marker +// symbol, exposed so native-marked underlying sources can be constructed +// from tests before the real JSG plumbing hands capabilities over from the +// C++ side. This module is only reachable through the (equally temporary) +// lazy `globalThis.streams` dev surface installed by main.ts — neither is +// part of the public API, and this export is REMOVED when the phase 3 +// C++ handshake lands. Everything else in nativeStreamInternals stays +// module-private. +const { nativeStreamInternals } = require('webstreams/native'); + +module.exports = { + ReadableStream, + ReadableStreamDefaultReader, + ReadableStreamBYOBReader, + ReadableStreamDefaultController, + ReadableByteStreamController, + ReadableStreamBYOBRequest, + ByteLengthQueuingStrategy, + CountQueuingStrategy, + WritableStream, + WritableStreamDefaultWriter, + WritableStreamDefaultController, + TransformStream, + TransformStreamDefaultController, + IdentityTransformStream, + FixedLengthStream, + TextEncoderStream, + TextDecoderStream, + // TEMPORARY — see the comment at the require above. + kNativeSource: nativeStreamInternals.kNativeSource, + kExtractNativeSource: nativeStreamInternals.kExtractNativeSource, + kNativeSink: nativeStreamInternals.kNativeSink, + kExtractNativeSink: nativeStreamInternals.kExtractNativeSink, + // TEMPORARY: internal-only reader (the C++ bridge surface), exposed so + // tests can exercise expectedLength pass-through and draining reads. + ReadableStreamDrainingReader, +}; diff --git a/src/per_isolate/webstreams/transform.ts b/src/per_isolate/webstreams/transform.ts new file mode 100644 index 00000000000..b1a0822a28f --- /dev/null +++ b/src/per_isolate/webstreams/transform.ts @@ -0,0 +1,712 @@ +'use strict'; + +// TransformStream and TransformStreamDefaultController (WHATWG Streams §6). +// +// A TransformStream is a {readable, writable} pair: the writable side's sink +// runs the transformer over incoming chunks, which enqueue results onto the +// readable side. Backpressure flows from the readable side to the writable +// side through a change-promise that the sink's write algorithm awaits. +// +// The two inner streams are created with module-owned source/sink objects +// (safe to property-read), and all cross-controller operations dispatch +// through prototype methods CAPTURED at bootstrap — never through the +// user-patchable prototypes. + +import type { + PromiseWithResolvers as PromiseWithResolversType, + QueuingStrategy, + ReadableStream as ReadableStreamType, + Transformer, + TransformStreamDefaultController as TransformStreamDefaultControllerType, + WritableStream as WritableStreamType, +} from './types'; + +const { + ObjectDefineProperties, + ObjectGetOwnPropertyDescriptor, + PromiseResolve, + PromiseReject, + PromiseWithResolvers, + PromisePrototypeThen, + RangeError, + Symbol, + SymbolToStringTag, + TypeError, + uncurryThis, +} = primordials; + +const { markPromiseHandled } = utils; + +const kPrivateSymbol: symbol = Symbol('private'); + +function isActualObject(value: unknown): boolean { + return value != null && typeof value === 'object'; +} + +function assertPrivateSymbol(symbol: symbol): void { + if (symbol !== kPrivateSymbol) { + throw new TypeError('Illegal constructor'); + } +} + +// --- Inner stream classes + captured internal dispatch ------------------- +// Capturing our own prototype methods at bootstrap time gives safe internal +// dispatch with zero refactoring of the inner classes. + +const { + ReadableStream, + ReadableStreamDefaultController, + internalsForTransform: readableInternals, +} = require('webstreams/readable'); +const { + WritableStream, + WritableStreamDefaultController, + internalsForPipe: writableInternals, +} = require('webstreams/writable'); + +const readableControllerEnqueue = uncurryThis( + ReadableStreamDefaultController.prototype.enqueue +) as (controller: object, chunk: unknown) => void; +const readableControllerClose = uncurryThis( + ReadableStreamDefaultController.prototype.close +) as (controller: object) => void; +const readableControllerError = uncurryThis( + ReadableStreamDefaultController.prototype.error +) as (controller: object, reason: unknown) => void; +const readableControllerDesiredSizeGet = (() => { + const desc = ObjectGetOwnPropertyDescriptor( + ReadableStreamDefaultController.prototype, + 'desiredSize' + ); + if (desc === undefined || desc.get === undefined) { + throw new TypeError( + "Expected accessor property 'desiredSize' on prototype" + ); + } + return uncurryThis(desc.get); +})() as (controller: object) => number | null; +const writableControllerError = uncurryThis( + WritableStreamDefaultController.prototype.error +) as (controller: object, reason: unknown) => void; + +// --------------------------------------------------------------------------- + +let transformStreamDefaultControllerInit: ( + controller: TransformStreamDefaultController, + stream: TransformStream +) => void; + +let transformStreamEnqueue: ( + stream: TransformStream, + chunk: O +) => void; +let transformStreamError: ( + stream: TransformStream, + reason: unknown +) => void; +let transformStreamTerminate: (stream: TransformStream) => void; +let transformStreamDesiredSize: ( + stream: TransformStream +) => number | null; + +class TransformStreamDefaultController< + I = unknown, + O = unknown, +> implements TransformStreamDefaultControllerType { + #stream: TransformStream | undefined; + + static { + transformStreamDefaultControllerInit = (controller, stream) => { + controller.#stream = stream; + }; + } + + constructor(privateSymbol: symbol) { + assertPrivateSymbol(privateSymbol); + } + + get desiredSize(): number | null { + if (!(#stream in this)) throw new TypeError('Illegal invocation'); + const stream = this.#stream; + if (stream === undefined) { + throw new TypeError('Controller is not attached to a stream'); + } + return transformStreamDesiredSize(stream); + } + + enqueue(chunk: O = undefined as O): void { + if (!(#stream in this)) throw new TypeError('Illegal invocation'); + const stream = this.#stream; + if (stream === undefined) { + throw new TypeError('Controller is not attached to a stream'); + } + transformStreamEnqueue(stream, chunk); + } + + error(reason: unknown = undefined): void { + if (!(#stream in this)) throw new TypeError('Illegal invocation'); + const stream = this.#stream; + if (stream === undefined) { + throw new TypeError('Controller is not attached to a stream'); + } + transformStreamError(stream, reason); + } + + terminate(): void { + if (!(#stream in this)) throw new TypeError('Illegal invocation'); + const stream = this.#stream; + if (stream === undefined) { + throw new TypeError('Controller is not attached to a stream'); + } + transformStreamTerminate(stream); + } + + [SymbolToStringTag] = 'TransformStreamDefaultController'; +} + +class TransformStream { + #readable: ReadableStreamType; + #writable: WritableStreamType; + // Undefined for ELIDED (zero-algorithm) transforms — no controller is + // allocated when there are no transformer callbacks to receive it. + // @ts-expect-error -- retained for debugging/future use + #controller?: TransformStreamDefaultController | undefined; + #readableController: object | undefined; + #writableController: object | undefined; + // Backpressure starts ON: the readable side must pull once before the + // sink transforms the first chunk. + #backpressure: boolean = true; + #backpressureChange: PromiseWithResolversType; + // Set by the constructor to a closure that clears the transformer + // algorithm references. Called from #errorWritableAndUnblockWrite + // (which lives outside the constructor scope and cannot access the + // algorithm locals directly). Undefined for ELIDED transforms. + #clearAlgorithms: (() => void) | undefined; + + static { + transformStreamDesiredSize = (stream) => { + const readableController = stream.#readableController; + return readableController === undefined + ? null + : readableControllerDesiredSizeGet(readableController); + }; + + transformStreamEnqueue = ( + stream: TransformStream, + chunk: O + ) => { + const readableController = stream.#readableController; + if (readableController === undefined) { + throw new TypeError('TransformStream is not fully initialized'); + } + // Spec TransformStreamDefaultControllerEnqueue step 4: + // if ReadableStreamDefaultControllerCanCloseOrEnqueue is false, + // throw a TypeError. This pre-check must happen BEFORE we attempt + // the enqueue, so that enqueue-after-error throws TypeError (not + // the storedError). The readable controller's own enqueue() does + // the same check internally; the pre-check here ensures the + // try/catch below only catches size()-originated errors. + if (readableInternals.getState(stream.#readable) !== 'readable') { + throw new TypeError( + 'Cannot enqueue a chunk into a stream that is closed or has been errored' + ); + } + try { + readableControllerEnqueue(readableController, chunk); + } catch (e) { + // size() threw — error the writable side and unblock backpressure. + stream.#errorWritableAndUnblockWrite(e); + // Spec step 5.2: throw stream.[[readable]].[[storedError]], not + // the caught exception. When size() calls controller.error(e1) + // then throws e2, the storedError is e1 — the first error wins. + const storedError = readableInternals.getStoredError(stream.#readable); + throw storedError !== undefined ? storedError : e; + } + // Mirror the readable side's backpressure state. + const desiredSize = readableControllerDesiredSizeGet(readableController); + const backpressure = desiredSize !== null && desiredSize <= 0; + if (backpressure !== stream.#backpressure) { + stream.#setBackpressure(backpressure); + } + }; + + transformStreamError = ( + stream: TransformStream, + reason: unknown + ) => { + const readableController = stream.#readableController; + if (readableController !== undefined) { + readableControllerError(readableController, reason); + } + stream.#errorWritableAndUnblockWrite(reason); + }; + + transformStreamTerminate = (stream: TransformStream) => { + const readableController = stream.#readableController; + if (readableController !== undefined) { + try { + readableControllerClose(readableController); + } catch { + // Already closed or errored — nothing to do. + } + } + stream.#errorWritableAndUnblockWrite( + new TypeError('The transform stream has been terminated') + ); + }; + } + + #setBackpressure(backpressure: boolean): void { + // Resolve the previous change promise and mint a fresh one — anyone + // awaiting the old promise (the sink's write algorithm) proceeds. + this.#backpressureChange.resolve(); + const replacement = + PromiseWithResolvers() as PromiseWithResolversType; + markPromiseHandled(replacement.promise); + this.#backpressureChange = replacement; + this.#backpressure = backpressure; + } + + #errorWritableAndUnblockWrite(reason: unknown): void { + // Spec: TransformStreamErrorWritableAndUnblockWrite step 1 — + // TransformStreamDefaultControllerClearAlgorithms. Prevents a + // later readable.cancel() from invoking the transformer's cancel + // callback after the stream has already errored/terminated. + if (this.#clearAlgorithms !== undefined) { + this.#clearAlgorithms(); + this.#clearAlgorithms = undefined; + } + const writableController = this.#writableController; + if (writableController !== undefined) { + writableControllerError(writableController, reason); + } + if (this.#backpressure) { + this.#setBackpressure(false); + } + } + + constructor( + transformer?: Transformer, + writableStrategy?: QueuingStrategy, + readableStrategy?: QueuingStrategy + ) { + transformer ??= {} as Transformer; + writableStrategy ??= {} as QueuingStrategy; + readableStrategy ??= {} as QueuingStrategy; + + if (!isActualObject(transformer)) { + throw new TypeError('transformer must be an object'); + } + + // --- Transformer method extraction (alphabetical property reads) --- + const cancelFn = transformer.cancel; + if (cancelFn !== undefined && typeof cancelFn !== 'function') { + throw new TypeError('transformer.cancel must be a function'); + } + const flushFn = transformer.flush; + if (flushFn !== undefined && typeof flushFn !== 'function') { + throw new TypeError('transformer.flush must be a function'); + } + if (transformer.readableType !== undefined) { + throw new RangeError('transformer.readableType must be undefined'); + } + const startFn = transformer.start; + if (startFn !== undefined && typeof startFn !== 'function') { + throw new TypeError('transformer.start must be a function'); + } + const transformFn = transformer.transform; + if (transformFn !== undefined && typeof transformFn !== 'function') { + throw new TypeError('transformer.transform must be a function'); + } + if (transformer.writableType !== undefined) { + throw new RangeError('transformer.writableType must be undefined'); + } + + const initialBackpressureChange = + PromiseWithResolvers() as PromiseWithResolversType; + markPromiseHandled(initialBackpressureChange.promise); + this.#backpressureChange = initialBackpressureChange; + + // --- ELISION CHECK --- + // A transformer with ZERO algorithms (no transform/flush/start/cancel) + // is semantically equivalent to no transformer: every write enqueues + // the chunk unchanged into the readable queue. The elided path skips + // controller allocation, the start-promise gate, and per-write + // algorithm wrappers while preserving the spec-observable backpressure + // handshake (writer.desiredSize, writer.ready, write-settlement + // timing). An empty transformer {} is equivalent to undefined. + const isElided = + cancelFn === undefined && + flushFn === undefined && + startFn === undefined && + transformFn === undefined; + + if (isElided) { + // ---- ELIDED PATH ---- + // No controller, no start gating, no algorithm wrappers. + + const sinkWrite = async (chunk: I): Promise => { + while (this.#backpressure) { + await this.#backpressureChange.promise; + const state = writableInternals.getState(this.#writable); + if (state === 'erroring' || state === 'errored') { + throw writableInternals.getStoredError(this.#writable); + } + } + const rc = this.#readableController as object; + readableControllerEnqueue(rc, chunk); + const desiredSize = readableControllerDesiredSizeGet(rc); + const backpressure = desiredSize !== null && desiredSize <= 0; + if (backpressure !== this.#backpressure) { + this.#setBackpressure(backpressure); + } + }; + const sinkClose = (): void => { + readableControllerClose(this.#readableController as object); + }; + const sinkAbort = (reason: unknown): void => { + readableControllerError(this.#readableController as object, reason); + }; + + this.#writable = new WritableStream( + { + start: (c: object) => { + this.#writableController = c; + }, + write: sinkWrite, + close: sinkClose, + abort: sinkAbort, + }, + writableStrategy + ); + + const sourcePull = (): Promise => { + this.#setBackpressure(false); + return this.#backpressureChange.promise; + }; + const sourceCancel = (reason: unknown): void => { + this.#errorWritableAndUnblockWrite(reason); + }; + + const readableHWM = + readableStrategy.highWaterMark === undefined + ? 0 + : readableStrategy.highWaterMark; + this.#readable = new ReadableStream( + { + start: (c: object) => { + this.#readableController = c; + }, + pull: sourcePull, + cancel: sourceCancel, + }, + { highWaterMark: readableHWM, size: readableStrategy.size } + ); + } else { + // ---- STANDARD PATH (transformer has algorithms) ---- + + const controller = new TransformStreamDefaultController( + kPrivateSymbol + ); + transformStreamDefaultControllerInit(controller, this); + this.#controller = controller; + + let transformAlgorithm: (chunk: I) => Promise; + if (transformFn === undefined) { + transformAlgorithm = (chunk: I) => { + try { + transformStreamEnqueue(this, chunk as unknown as O); + return PromiseResolve() as Promise; + } catch (e) { + return PromiseReject(e) as Promise; + } + }; + } else { + const callTransform = uncurryThis(transformFn); + transformAlgorithm = (chunk: I) => { + try { + return PromiseResolve( + callTransform(transformer, chunk, controller) + ) as Promise; + } catch (e) { + return PromiseReject(e) as Promise; + } + }; + } + let cancelAlgorithm: ((reason: unknown) => Promise) | undefined; + if (cancelFn !== undefined) { + const callCancel = uncurryThis(cancelFn); + cancelAlgorithm = (reason: unknown) => { + try { + return PromiseResolve( + callCancel(transformer, reason) + ) as Promise; + } catch (e) { + return PromiseReject(e) as Promise; + } + }; + } + let flushAlgorithm: (() => Promise) | undefined; + if (flushFn !== undefined) { + const callFlush = uncurryThis(flushFn); + flushAlgorithm = () => { + try { + return PromiseResolve( + callFlush(transformer, controller) + ) as Promise; + } catch (e) { + return PromiseReject(e) as Promise; + } + }; + } + + // Spec: TransformStreamDefaultControllerClearAlgorithms — releases + // the transformer closures so they cannot be invoked after + // close/abort/cancel/error/terminate. Stored on the instance so + // #errorWritableAndUnblockWrite (outside this closure scope) can + // reach them. + const clearAlgorithms = (): void => { + cancelAlgorithm = undefined; + flushAlgorithm = undefined; + }; + this.#clearAlgorithms = clearAlgorithms; + + // Spec §6.4.2: [[finishPromise]] — coordinates cancel, abort, and + // close so that whichever runs first wins; parallel operations + // return the SAME promise. + let finishPromise: Promise | undefined; + + // Both inner streams' start algorithms return THIS promise, so + // neither side processes anything until transformer.start() + // settles. + const startHolder = + PromiseWithResolvers() as PromiseWithResolversType; + + const sinkWrite = async (chunk: I): Promise => { + while (this.#backpressure) { + await this.#backpressureChange.promise; + const state = writableInternals.getState(this.#writable); + if (state === 'erroring' || state === 'errored') { + throw writableInternals.getStoredError(this.#writable); + } + } + // Spec TransformStreamDefaultControllerPerformTransform: a + // rejection from the transform algorithm errors BOTH sides + // (TransformStreamError), then rethrows to reject the write. + return PromisePrototypeThen( + transformAlgorithm(chunk), + undefined, + (e: unknown) => { + transformStreamError(this, e); + throw e; + } + ) as Promise; + }; + // Spec: TransformStreamDefaultSinkCloseAlgorithm + const sinkClose = (): Promise => { + if (finishPromise !== undefined) return finishPromise; + const { + promise, + resolve: finishResolve, + reject: finishReject, + } = PromiseWithResolvers() as PromiseWithResolversType; + finishPromise = promise; + const flushResult = + flushAlgorithm !== undefined + ? flushAlgorithm() + : (PromiseResolve() as Promise); + // Spec: TransformStreamDefaultControllerClearAlgorithms + clearAlgorithms(); + markPromiseHandled( + PromisePrototypeThen( + flushResult, + () => { + if (readableInternals.getState(this.#readable) === 'errored') { + finishReject(readableInternals.getStoredError(this.#readable)); + } else { + const rc = this.#readableController; + if (rc !== undefined) { + try { + readableControllerClose(rc); + } catch { + // Already closed — acceptable per spec. + } + } + finishResolve(); + } + }, + (r: unknown) => { + const rc = this.#readableController; + if (rc !== undefined) { + readableControllerError(rc, r); + } + finishReject(r); + } + ) + ); + return finishPromise; + }; + // Spec: TransformStreamDefaultSinkAbortAlgorithm + const sinkAbort = (reason: unknown): Promise => { + if (finishPromise !== undefined) return finishPromise; + const { + promise, + resolve: finishResolve, + reject: finishReject, + } = PromiseWithResolvers() as PromiseWithResolversType; + finishPromise = promise; + const cancelResult = + cancelAlgorithm !== undefined + ? cancelAlgorithm(reason) + : (PromiseResolve() as Promise); + // Spec: TransformStreamDefaultControllerClearAlgorithms + clearAlgorithms(); + markPromiseHandled( + PromisePrototypeThen( + cancelResult, + () => { + if (readableInternals.getState(this.#readable) === 'errored') { + finishReject(readableInternals.getStoredError(this.#readable)); + } else { + const rc = this.#readableController; + if (rc !== undefined) { + readableControllerError(rc, reason); + } + finishResolve(); + } + }, + (r: unknown) => { + const rc = this.#readableController; + if (rc !== undefined) { + readableControllerError(rc, r); + } + finishReject(r); + } + ) + ); + return finishPromise; + }; + + this.#writable = new WritableStream( + { + start: (c: object) => { + this.#writableController = c; + return startHolder.promise; + }, + write: sinkWrite, + close: sinkClose, + abort: sinkAbort, + }, + writableStrategy + ); + + const sourcePull = (): Promise => { + this.#setBackpressure(false); + return this.#backpressureChange.promise; + }; + // Spec: TransformStreamDefaultSourceCancelAlgorithm + const sourceCancel = (reason: unknown): Promise => { + if (finishPromise !== undefined) return finishPromise; + const { + promise, + resolve: finishResolve, + reject: finishReject, + } = PromiseWithResolvers() as PromiseWithResolversType; + finishPromise = promise; + // Snapshot writable state before calling the cancel algorithm so we + // can detect if the callback itself errored the writable (via + // controller.error()). The spec checks writable.[[state]] at + // microtask time, but that also catches unrelated user code + // (e.g. controller.enqueue() on the now-closed readable) that + // runs between cancel() and the microtask. Comparing before/after + // detects only errors caused by the cancel algorithm. + const writableCleanBeforeCancel = + writableInternals.getState(this.#writable) === 'writable'; + const cancelResult = + cancelAlgorithm !== undefined + ? cancelAlgorithm(reason) + : (PromiseResolve() as Promise); + // Spec: TransformStreamDefaultControllerClearAlgorithms + clearAlgorithms(); + const writableErroredByCancel = + writableCleanBeforeCancel && + writableInternals.getState(this.#writable) !== 'writable'; + markPromiseHandled( + PromisePrototypeThen( + cancelResult, + () => { + if (writableErroredByCancel) { + finishReject(writableInternals.getStoredError(this.#writable)); + } else { + this.#errorWritableAndUnblockWrite(reason); + finishResolve(); + } + }, + (r: unknown) => { + this.#errorWritableAndUnblockWrite(r); + finishReject(r); + } + ) + ); + return finishPromise; + }; + + const readableHWM = + readableStrategy.highWaterMark === undefined + ? 0 + : readableStrategy.highWaterMark; + this.#readable = new ReadableStream( + { + start: (c: object) => { + this.#readableController = c; + return startHolder.promise; + }, + pull: sourcePull, + cancel: sourceCancel, + }, + { highWaterMark: readableHWM, size: readableStrategy.size } + ); + + // --- Start the transformer --- + const startResult: unknown = + startFn === undefined + ? undefined + : uncurryThis(startFn)(transformer, controller); + startHolder.resolve(startResult as void | PromiseLike); + } + } + + get readable(): ReadableStreamType { + if (!(#readable in this)) throw new TypeError('Illegal invocation'); + return this.#readable; + } + + get writable(): WritableStreamType { + if (!(#writable in this)) throw new TypeError('Illegal invocation'); + return this.#writable; + } + + [SymbolToStringTag] = 'TransformStream'; +} + +ObjectDefineProperties(TransformStream, { + length: { value: 0 }, +}); +ObjectDefineProperties(TransformStreamDefaultController, { + length: { value: 0 }, +}); +ObjectDefineProperties(TransformStream.prototype, { + readable: { enumerable: true }, + writable: { enumerable: true }, +}); +ObjectDefineProperties(TransformStreamDefaultController.prototype, { + desiredSize: { enumerable: true }, + enqueue: { enumerable: true }, + error: { enumerable: true }, + terminate: { enumerable: true }, +}); + +module.exports = { + TransformStream, + TransformStreamDefaultController, +}; diff --git a/src/per_isolate/webstreams/types.d.ts b/src/per_isolate/webstreams/types.d.ts new file mode 100644 index 00000000000..cd3659aa38e --- /dev/null +++ b/src/per_isolate/webstreams/types.d.ts @@ -0,0 +1,213 @@ +export declare class ReadableByteStreamController { + readonly byobRequest: ReadableStreamBYOBRequest | null; + readonly desiredSize: number | null; + enqueue(chunk: ArrayBufferView): void; + close(): void; + error(reason?: unknown): void; +} + +export declare class ReadableStreamBYOBRequest { + readonly view: Uint8Array | null; + // Non-standard workerd extension: the minimum number of BYTES the source + // must still deliver to satisfy the outstanding read (the read's `min`, + // in bytes, less what has already been filled; the view's element size + // for min-less reads). null once the request is invalidated. + readonly atLeast: number | null; + respond(bytesWritten: number): void; + respondWithNewView(view: ArrayBufferView): void; +} + +export declare class ReadableStreamDefaultController { + readonly desiredSize: number | null; + enqueue(chunk?: R): void; + close(): void; + error(reason?: unknown): void; +} + +export type UnderlyingSource = + | UnderlyingDefaultSource + | UnderlyingByteSource; + +export interface UnderlyingDefaultSource { + start?: ( + controller: ReadableStreamDefaultController + ) => void | Promise; + pull?: ( + controller: ReadableStreamDefaultController + ) => void | Promise; + cancel?: (reason?: unknown) => void | Promise; + type?: undefined; +} + +export interface UnderlyingByteSource { + start?: (controller: ReadableByteStreamController) => void | Promise; + pull?: (controller: ReadableByteStreamController) => void | Promise; + cancel?: (reason?: unknown) => void | Promise; + type: 'bytes'; + autoAllocateChunkSize?: number; + // Non-standard workerd extension: the TOTAL bytes this source promises + // to produce over its lifetime (undefined = unknown). Accepts a + // non-negative bigint or integer number (normalized to bigint). The + // stream errors on overflow (delivering past it) or underflow (closing + // short of it); consumer-initiated cancel is exempt. Used by the C++ + // bridge to set Content-Length instead of chunked encoding. + expectedLength?: bigint | number; +} + +export interface QueuingStrategy { + highWaterMark?: number; + size?: (chunk: T) => number; +} + +export interface ReadableStreamDefaultReader { + readonly closed: Promise; + cancel(reason?: unknown): Promise; + read(): Promise>; + releaseLock(): void; +} + +export interface ReadableStreamBYOBReaderReadOptions { + min?: number; +} + +export interface ReadableStreamBYOBReader { + readonly closed: Promise; + cancel(reason?: unknown): Promise; + read( + view: T, + options?: ReadableStreamBYOBReaderReadOptions + ): Promise>; + readAtLeast( + minElements: number, + view: T + ): Promise>; + releaseLock(): void; +} + +export type ReadableStreamReader = + | ReadableStreamDefaultReader + | ReadableStreamBYOBReader; + +// NOTE: done: true usually carries no value, but it is NOT always +// valueless: a BYOB read committed when the stream closes resolves with the +// partially-filled view AND done: true in a single result (spec +// CommitPullIntoDescriptor in the closed state; WPT +// readable-byte-streams/read-min.any.js "read({ min }) when closed before +// view is filled"). lib.dom models this the same way +// (ReadableStreamReadDoneResult = { done: true; value?: T }). The +// explicit `| undefined` keeps `{ done: true, value: undefined }` +// assignable under exactOptionalPropertyTypes. +export type ReadableStreamReadResult = + | { done: false; value: T } + | { done: true; value?: T | undefined }; + +export interface StreamPipeOptions { + preventClose?: boolean; + preventAbort?: boolean; + preventCancel?: boolean; + signal?: AbortSignal; +} + +export declare class ReadableStream { + constructor( + underlyingSource?: UnderlyingSource, + strategy?: QueuingStrategy + ); + static from( + asyncIterable: Iterable | AsyncIterable | R + ): ReadableStream; + readonly locked: boolean; + cancel(reason?: unknown): Promise; + getReader(): ReadableStreamDefaultReader; + getReader(options: { mode: 'byob' }): ReadableStreamBYOBReader; + pipeThrough( + transform: TransformStream, + options?: StreamPipeOptions + ): ReadableStream; + pipeTo( + destination: WritableStream, + options?: StreamPipeOptions + ): Promise; + tee(): [ReadableStream, ReadableStream]; + values(options?: { preventCancel?: boolean }): AsyncIterableIterator; + [Symbol.asyncIterator](options?: { + preventCancel?: boolean; + }): AsyncIterableIterator; +} + +export declare class WritableStream { + constructor( + underlyingSink?: UnderlyingSink, + strategy?: QueuingStrategy + ); + readonly locked: boolean; + abort(reason?: unknown): Promise; + close(): Promise; + getWriter(): WritableStreamDefaultWriter; +} + +export declare class WritableStreamDefaultWriter { + constructor(stream: WritableStream); + readonly closed: Promise; + readonly desiredSize: number | null; + readonly ready: Promise; + abort(reason?: unknown): Promise; + close(): Promise; + releaseLock(): void; + write(chunk?: W): Promise; +} + +export declare class WritableStreamDefaultController { + readonly signal: AbortSignal; + error(reason?: unknown): void; +} + +export interface UnderlyingSink { + start?: (controller: WritableStreamDefaultController) => void | Promise; + write?: ( + chunk: W, + controller: WritableStreamDefaultController + ) => void | Promise; + close?: () => void | Promise; + abort?: (reason?: unknown) => void | Promise; + type?: undefined; +} + +export declare class TransformStream { + constructor( + transformer?: Transformer, + writableStrategy?: QueuingStrategy, + readableStrategy?: QueuingStrategy + ); + readonly readable: ReadableStream; + readonly writable: WritableStream; +} + +export interface Transformer { + start?: ( + controller: TransformStreamDefaultController + ) => void | Promise; + transform?: ( + chunk: I, + controller: TransformStreamDefaultController + ) => void | Promise; + flush?: ( + controller: TransformStreamDefaultController + ) => void | Promise; + cancel?: (reason?: unknown) => void | Promise; + readableType?: undefined; + writableType?: undefined; +} + +export interface TransformStreamDefaultController { + readonly desiredSize: number | null; + enqueue(chunk: O): void; + error(reason?: unknown): void; + terminate(): void; +} + +interface PromiseWithResolvers { + promise: Promise; + resolve: (value: T | PromiseLike) => void; + reject: (reason?: unknown) => void; +} diff --git a/src/per_isolate/webstreams/writable.ts b/src/per_isolate/webstreams/writable.ts new file mode 100644 index 00000000000..1e570d60e0e --- /dev/null +++ b/src/per_isolate/webstreams/writable.ts @@ -0,0 +1,1323 @@ +'use strict'; + +// WritableStream, WritableStreamDefaultWriter, WritableStreamDefaultController +// (WHATWG Streams §5). Unlike the readable side, the writable side has a +// single consumer (the sink), so there is no shared-queue/cursor machinery — +// the controller keeps a simple spec-shaped FIFO of {chunk, size} plus a +// running total for backpressure. +// +// The state machine follows the spec closely: states are 'writable' → +// ('erroring' →) 'errored', or 'writable' → 'closed'; write/close requests +// are promise pairs; at most one sink operation (write or close) is in +// flight at a time. + +import type { + PromiseWithResolvers as PromiseWithResolversType, + QueuingStrategy, + UnderlyingSink, + WritableStreamDefaultController as WritableStreamDefaultControllerType, + WritableStreamDefaultWriter as WritableStreamDefaultWriterType, +} from './types'; + +const { + AbortController, + AbortControllerAbort, + AbortControllerSignalGet, + ArrayPrototypePush, + ArrayPrototypeShift, + NumberIsNaN, + ObjectDefineProperties, + ObjectDefineProperty, + PromiseResolve, + PromiseReject, + PromiseWithResolvers, + PromisePrototypeThen, + RangeError, + Symbol, + SymbolToStringTag, + TypeError, + uncurryThis, +} = primordials; + +const { isPromise, markPromiseHandled } = utils; + +// The native backend (leaf module — see the fence conventions in +// native.ts). The cast restores the real shape. +import type { NativeStreamInternals } from './native'; +const { nativeStreamInternals } = require('webstreams/native') as { + nativeStreamInternals: NativeStreamInternals; +}; +const { kExtractNativeSink, isNativeUnderlyingSink } = nativeStreamInternals; + +const kPrivateSymbol: symbol = Symbol('private'); +// Marker for a queued close request in the controller's FIFO. +const kCloseMarker: symbol = Symbol('close'); + +function isActualObject(value: unknown): boolean { + return value != null && typeof value === 'object'; +} + +function assertPrivateSymbol(symbol: symbol): void { + if (symbol !== kPrivateSymbol) { + throw new TypeError('Illegal constructor'); + } +} + +type WritableState = 'writable' | 'erroring' | 'errored' | 'closed'; + +interface PendingAbortRequest { + promise: Promise; + resolve: () => void; + reject: (reason: unknown) => void; + reason: unknown; + wasAlreadyErroring: boolean; +} + +// --------------------------------------------------------------------------- +// Cross-class accessors (assigned in static blocks). + +let getWritableStreamState: (stream: WritableStream) => WritableState; +let getWritableStreamStoredError: (stream: WritableStream) => unknown; +let isWritableStreamLocked: (stream: WritableStream) => boolean; +let setWritableStreamWriter: ( + stream: WritableStream, + writer: WritableStreamDefaultWriter | undefined +) => void; +let writableStreamAbort: ( + stream: WritableStream, + reason?: unknown +) => Promise; +let writableStreamCloseInternal: ( + stream: WritableStream +) => Promise; +let writableStreamAddWriteRequest: ( + stream: WritableStream +) => Promise; +let writableStreamDealWithRejection: ( + stream: WritableStream, + error: unknown +) => void; +let writableStreamStartErroring: ( + stream: WritableStream, + reason: unknown +) => void; +let writableStreamFinishErroringIfNeeded: ( + stream: WritableStream +) => void; +let writableStreamMarkFirstWriteRequestInFlight: ( + stream: WritableStream +) => void; +let writableStreamFinishInFlightWrite: (stream: WritableStream) => void; +let writableStreamFinishInFlightWriteWithError: ( + stream: WritableStream, + error: unknown +) => void; +let writableStreamMarkCloseRequestInFlight: ( + stream: WritableStream +) => void; +let writableStreamFinishInFlightClose: (stream: WritableStream) => void; +let writableStreamFinishInFlightCloseWithError: ( + stream: WritableStream, + error: unknown +) => void; +let writableStreamCloseQueuedOrInFlight: ( + stream: WritableStream +) => boolean; +let writableStreamUpdateBackpressure: ( + stream: WritableStream, + backpressure: boolean +) => void; +let writableStreamHasOperationMarkedInFlight: ( + stream: WritableStream +) => boolean; +let getWritableStreamController: ( + stream: WritableStream +) => WritableStreamDefaultController | undefined; + +let controllerGetChunkSize: ( + controller: WritableStreamDefaultController, + chunk: W +) => number; +let controllerWrite: ( + controller: WritableStreamDefaultController, + chunk: W, + chunkSize: number +) => void; +let controllerClose: ( + controller: WritableStreamDefaultController +) => void; +let controllerGetDesiredSize: ( + controller: WritableStreamDefaultController +) => number; +let controllerErrorSteps: ( + controller: WritableStreamDefaultController +) => void; +let controllerAbortSteps: ( + controller: WritableStreamDefaultController, + reason: unknown +) => Promise; +let controllerSignalAbort: ( + controller: WritableStreamDefaultController, + reason: unknown +) => void; + +let writerEnsureReadyPromiseRejected: ( + writer: WritableStreamDefaultWriter, + error: unknown +) => void; +let writerEnsureClosedPromiseRejected: ( + writer: WritableStreamDefaultWriter, + error: unknown +) => void; +let writerResolveReadyPromise: ( + writer: WritableStreamDefaultWriter +) => void; +let writerResetReadyPromise: ( + writer: WritableStreamDefaultWriter +) => void; +let writerResolveClosedPromise: ( + writer: WritableStreamDefaultWriter +) => void; +let getWriterStream: ( + writer: WritableStreamDefaultWriter +) => WritableStream | undefined; +let writableStreamBackpressureOf: (stream: WritableStream) => boolean; +// Internal writer operations: the pipe implementation must never dispatch +// through the public prototype methods (user-interceptable once the classes +// are installed on the global). +let writerWriteInternal: ( + writer: WritableStreamDefaultWriter, + chunk: W +) => Promise; +let writerCloseInternal: ( + writer: WritableStreamDefaultWriter +) => Promise; +let writerReleaseInternal: (writer: WritableStreamDefaultWriter) => void; +let getWriterReadyPromiseInternal: ( + writer: WritableStreamDefaultWriter +) => Promise; +let getWriterClosedPromiseInternal: ( + writer: WritableStreamDefaultWriter +) => Promise; + +// --------------------------------------------------------------------------- +// WritableStream + +// The shared extractor function installed as kExtractNativeSink on +// native-backed WritableStream instances. Assigned in the static block. +// THIS-BOUND (called as stream[kExtractNativeSink]()), mirroring the +// readable side's kExtractNativeSource calling convention so the C++ +// TypeWrapper uses one uniform protocol for both directions. +let extractNativeSink: (this: WritableStream) => object; + +class WritableStream { + #controller?: WritableStreamDefaultController | undefined; + #writer?: WritableStreamDefaultWriter | undefined; + #state: WritableState = 'writable'; + #storedError: unknown = undefined; + #writeRequests: PromiseWithResolversType[] = []; + #inFlightWriteRequest: PromiseWithResolversType | undefined; + #closeRequest: PromiseWithResolversType | undefined; + #inFlightCloseRequest: PromiseWithResolversType | undefined; + #pendingAbortRequest: PendingAbortRequest | undefined; + #backpressure: boolean = false; + // Back-reference to the native underlying sink (undefined for + // JS-backed streams). Kept for extraction — C++ unwraps the backing + // class from the returned object. + #nativeSink?: object | undefined; + + static { + getWritableStreamState = (stream) => stream.#state; + getWritableStreamStoredError = (stream) => stream.#storedError; + isWritableStreamLocked = (stream) => stream.#writer !== undefined; + writableStreamBackpressureOf = (stream) => stream.#backpressure; + setWritableStreamWriter = (stream, writer) => { + stream.#writer = writer; + }; + getWritableStreamController = (stream) => stream.#controller; + + // JS-to-C++ extraction for native-backed writable streams. + // Mirrors extractNativeSource on the readable side: atomic + // validate → extract → lock. No "disturbed" concept on writable. + extractNativeSink = function (this: WritableStream): object { + if (!(#state in this)) { + throw new TypeError('Illegal invocation'); + } + if (isWritableStreamLocked(this)) { + throw new TypeError( + 'Cannot extract a native sink from a locked stream' + ); + } + const sink = this.#nativeSink; + if (sink === undefined) { + throw new TypeError('This stream is not backed by a native sink'); + } + // Permanently lock via an internal writer (never exposed, never + // released) — same pattern as readable-side extraction. + new WritableStreamDefaultWriter(this); + return sink; + }; + + writableStreamCloseQueuedOrInFlight = (stream) => { + return ( + stream.#closeRequest !== undefined || + stream.#inFlightCloseRequest !== undefined + ); + }; + + writableStreamHasOperationMarkedInFlight = (stream) => { + return ( + stream.#inFlightWriteRequest !== undefined || + stream.#inFlightCloseRequest !== undefined + ); + }; + + writableStreamAddWriteRequest = (stream) => { + // Caller guarantees: locked and state 'writable'. + const request = PromiseWithResolvers() as PromiseWithResolversType; + ArrayPrototypePush(stream.#writeRequests, request); + return request.promise; + }; + + writableStreamAbort = (stream: WritableStream, reason?: unknown) => { + const state = stream.#state; + if (state === 'closed' || state === 'errored') { + return PromiseResolve(undefined) as Promise; + } + const controller = stream.#controller; + if (controller !== undefined) { + // The controller's AbortSignal fires as soon as an abort is + // requested, letting in-flight sink writes cancel their work. + controllerSignalAbort(controller, reason); + } + // signalAbort dispatches 'abort' events SYNCHRONOUSLY — the sink may + // have registered listeners on controller.signal, and that user code + // can re-enter (write/error/close). Re-read the state, per spec. + const stateNow = stream.#state; + if (stateNow === 'closed' || stateNow === 'errored') { + return PromiseResolve(undefined) as Promise; + } + if (stream.#pendingAbortRequest !== undefined) { + return stream.#pendingAbortRequest.promise; + } + const wasAlreadyErroring = stateNow === 'erroring'; + const abortReason = wasAlreadyErroring ? undefined : reason; + const { promise, resolve, reject } = + PromiseWithResolvers() as PromiseWithResolversType; + stream.#pendingAbortRequest = { + promise, + resolve, + reject, + reason: abortReason, + wasAlreadyErroring, + }; + if (!wasAlreadyErroring) { + writableStreamStartErroring(stream, abortReason); + } + return promise; + }; + + writableStreamCloseInternal = (stream: WritableStream) => { + const state = stream.#state; + if (state === 'closed' || state === 'errored') { + return PromiseReject( + new TypeError('Cannot close a stream that is closed or errored') + ) as Promise; + } + // Caller (writer.close / stream.close) has rejected the + // close-queued-or-in-flight case already; assert-not anyway. + if (writableStreamCloseQueuedOrInFlight(stream)) { + return PromiseReject( + new TypeError('Stream is already closing') + ) as Promise; + } + const request = PromiseWithResolvers() as PromiseWithResolversType; + stream.#closeRequest = request; + const writer = stream.#writer; + if ( + writer !== undefined && + stream.#backpressure && + state === 'writable' + ) { + // Close relieves backpressure waits. + writerResolveReadyPromise(writer); + } + const controller = stream.#controller; + if (controller !== undefined) { + controllerClose(controller); + } + return request.promise; + }; + + writableStreamDealWithRejection = ( + stream: WritableStream, + error: unknown + ) => { + const state = stream.#state; + if (state === 'writable') { + writableStreamStartErroring(stream, error); + return; + } + // state is 'erroring' + writableStreamFinishErroring(stream); + }; + + writableStreamStartErroring = ( + stream: WritableStream, + reason: unknown + ) => { + // assert: storedError undefined, state 'writable' + stream.#state = 'erroring'; + stream.#storedError = reason; + const writer = stream.#writer; + if (writer !== undefined) { + writerEnsureReadyPromiseRejected(writer, reason); + } + const controller = stream.#controller; + if ( + !writableStreamHasOperationMarkedInFlight(stream) && + controller !== undefined && + controllerStartedOf(controller) + ) { + writableStreamFinishErroring(stream); + } + }; + + writableStreamFinishErroringIfNeeded = (stream: WritableStream) => { + if ( + stream.#state === 'erroring' && + !writableStreamHasOperationMarkedInFlight(stream) + ) { + writableStreamFinishErroring(stream); + } + }; + + const writableStreamFinishErroring = (stream: WritableStream) => { + // assert: state 'erroring', no operations in flight + stream.#state = 'errored'; + const controller = stream.#controller; + if (controller !== undefined) { + controllerErrorSteps(controller); // reset the controller queue + } + const storedError = stream.#storedError; + const writeRequests = stream.#writeRequests; + stream.#writeRequests = []; + for (let i = 0; i < writeRequests.length; i++) { + const request = writeRequests[i] as PromiseWithResolversType; + request.reject(storedError); + } + const abortRequest = stream.#pendingAbortRequest; + if (abortRequest === undefined) { + writableStreamRejectCloseAndClosedPromiseIfNeeded(stream); + return; + } + stream.#pendingAbortRequest = undefined; + if (abortRequest.wasAlreadyErroring) { + abortRequest.reject(storedError); + writableStreamRejectCloseAndClosedPromiseIfNeeded(stream); + return; + } + const promise = + controller !== undefined + ? controllerAbortSteps(controller, abortRequest.reason) + : (PromiseResolve() as Promise); + PromisePrototypeThen( + promise, + () => { + abortRequest.resolve(); + writableStreamRejectCloseAndClosedPromiseIfNeeded(stream); + }, + (e: unknown) => { + abortRequest.reject(e); + writableStreamRejectCloseAndClosedPromiseIfNeeded(stream); + } + ); + }; + + const writableStreamRejectCloseAndClosedPromiseIfNeeded = ( + stream: WritableStream + ) => { + // assert: state 'errored' + const storedError = stream.#storedError; + const closeRequest = stream.#closeRequest; + if (closeRequest !== undefined) { + // assert: no in-flight close + closeRequest.reject(storedError); + stream.#closeRequest = undefined; + } + const writer = stream.#writer; + if (writer !== undefined) { + writerEnsureClosedPromiseRejected(writer, storedError); + } + }; + + writableStreamMarkFirstWriteRequestInFlight = (stream) => { + // assert: no in-flight write; writeRequests non-empty + stream.#inFlightWriteRequest = ArrayPrototypeShift( + stream.#writeRequests + ) as PromiseWithResolversType; + }; + + writableStreamFinishInFlightWrite = (stream) => { + // assert: in-flight write request is set (caller guarantee) + const request = stream + .#inFlightWriteRequest as PromiseWithResolversType; + stream.#inFlightWriteRequest = undefined; + request.resolve(); + }; + + writableStreamFinishInFlightWriteWithError = (stream, error) => { + // assert: in-flight write request is set (caller guarantee) + const request = stream + .#inFlightWriteRequest as PromiseWithResolversType; + stream.#inFlightWriteRequest = undefined; + request.reject(error); + writableStreamDealWithRejection(stream, error); + }; + + writableStreamMarkCloseRequestInFlight = (stream) => { + // assert: no in-flight close; closeRequest set + stream.#inFlightCloseRequest = stream.#closeRequest; + stream.#closeRequest = undefined; + }; + + writableStreamFinishInFlightClose = (stream) => { + // assert: in-flight close request is set (caller guarantee) + const request = stream + .#inFlightCloseRequest as PromiseWithResolversType; + stream.#inFlightCloseRequest = undefined; + request.resolve(); + // assert: state 'writable' or 'erroring' + if (stream.#state === 'erroring') { + // The close succeeded — the abort (if any) is moot. + stream.#storedError = undefined; + const abortRequest = stream.#pendingAbortRequest; + if (abortRequest !== undefined) { + abortRequest.resolve(); + stream.#pendingAbortRequest = undefined; + } + } + stream.#state = 'closed'; + const writer = stream.#writer; + if (writer !== undefined) { + writerResolveClosedPromise(writer); + } + // assert: pendingAbortRequest undefined, storedError undefined + }; + + writableStreamFinishInFlightCloseWithError = (stream, error) => { + // assert: in-flight close request is set (caller guarantee) + const request = stream + .#inFlightCloseRequest as PromiseWithResolversType; + stream.#inFlightCloseRequest = undefined; + request.reject(error); + const abortRequest = stream.#pendingAbortRequest; + if (abortRequest !== undefined) { + abortRequest.reject(error); + stream.#pendingAbortRequest = undefined; + } + writableStreamDealWithRejection(stream, error); + }; + + writableStreamUpdateBackpressure = (stream, backpressure) => { + // assert: state 'writable', close not queued or in flight + const writer = stream.#writer; + if (writer !== undefined && backpressure !== stream.#backpressure) { + if (backpressure) { + writerResetReadyPromise(writer); + } else { + writerResolveReadyPromise(writer); + } + } + stream.#backpressure = backpressure; + }; + } + + constructor( + underlyingSink: UnderlyingSink = {}, + strategy: QueuingStrategy = {} + ) { + // --- WebIDL strategy dictionary conversion (BEFORE sink reads) --- + // Per WebIDL, dictionary-typed arguments are converted at the IDL + // layer before the constructor body runs. strategy is QueuingStrategy + // (a dictionary); underlyingSink is plain object. + const sizeFn = strategy.size; + let sizeAlgorithm: (chunk: W) => number; + if (sizeFn === undefined) { + sizeAlgorithm = () => 1; + } else if (typeof sizeFn !== 'function') { + throw new TypeError('strategy.size must be a function'); + } else { + const callSize = uncurryThis(sizeFn); + sizeAlgorithm = (chunk: W) => callSize(undefined, chunk); + } + let highWaterMark = 1; + if (strategy.highWaterMark !== undefined) { + highWaterMark = +strategy.highWaterMark; + if (NumberIsNaN(highWaterMark) || highWaterMark < 0) { + throw new RangeError('Invalid highWaterMark'); + } + } + + // --- Now read underlyingSink properties --- + if (!isActualObject(underlyingSink)) { + throw new TypeError('underlyingSink must be an object'); + } + if (underlyingSink.type !== undefined) { + throw new RangeError('Invalid underlying sink type'); + } + + // Native sink detection: the standard WritableStream machinery + // drives the sink via start/write/close/abort as-is. The marker + // exists for pipe dispatch and extraction only. + if (isNativeUnderlyingSink(underlyingSink)) { + this.#nativeSink = underlyingSink; + // Install the extraction marker (same descriptor shape as the + // readable side: own, non-enumerable, non-writable, + // non-configurable). + ObjectDefineProperty(this, kExtractNativeSink, { + __proto__: null, + value: extractNativeSink, + } as PropertyDescriptor); + } + + this.#controller = new WritableStreamDefaultController( + kPrivateSymbol, + this, + underlyingSink, + sizeAlgorithm, + highWaterMark + ); + } + + get locked(): boolean { + if (!(#state in this)) throw new TypeError('Illegal invocation'); + return isWritableStreamLocked(this); + } + + abort(reason: unknown = undefined): Promise { + try { + if (!(#state in this)) throw new TypeError('Illegal invocation'); + if (isWritableStreamLocked(this)) { + throw new TypeError('Cannot abort a stream that is locked'); + } + return writableStreamAbort(this, reason); + } catch (e) { + return PromiseReject(e) as Promise; + } + } + + close(): Promise { + try { + if (!(#state in this)) throw new TypeError('Illegal invocation'); + if (isWritableStreamLocked(this)) { + throw new TypeError('Cannot close a stream that is locked'); + } + if (writableStreamCloseQueuedOrInFlight(this)) { + throw new TypeError('Stream is already closing'); + } + return writableStreamCloseInternal(this); + } catch (e) { + return PromiseReject(e) as Promise; + } + } + + getWriter(): WritableStreamDefaultWriterType { + if (!(#state in this)) throw new TypeError('Illegal invocation'); + return new WritableStreamDefaultWriter(this); + } + + [SymbolToStringTag] = 'WritableStream'; +} + +// Started-flag peek used by the stream's erroring machinery; assigned in +// the controller's static block. +let controllerStartedOf: ( + controller: WritableStreamDefaultController +) => boolean; + +// --------------------------------------------------------------------------- +// WritableStreamDefaultController + +interface QueuedWrite { + value: W | typeof kCloseMarker; + size: number; +} + +class WritableStreamDefaultController< + W = unknown, +> implements WritableStreamDefaultControllerType { + #stream: WritableStream; + #queue: QueuedWrite[] = []; + #queueTotalSize: number = 0; + #started: boolean = false; + #strategyHWM: number; + #sizeAlgorithm: ((chunk: W) => number) | undefined; + #writeAlgorithm: ((chunk: W) => Promise) | undefined; + #closeAlgorithm: (() => Promise) | undefined; + #abortAlgorithm: ((reason: unknown) => Promise) | undefined; + #abortController = new AbortController(); + + static { + controllerStartedOf = (controller) => controller.#started; + + controllerGetDesiredSize = (controller) => { + return controller.#strategyHWM - controller.#queueTotalSize; + }; + + controllerSignalAbort = (controller, reason) => { + AbortControllerAbort(controller.#abortController, reason); + }; + + controllerErrorSteps = (controller) => { + controller.#queue = []; + controller.#queueTotalSize = 0; + }; + + controllerAbortSteps = (controller, reason) => { + const abortAlgorithm = controller.#abortAlgorithm; + controller.#clearAlgorithms(); + return abortAlgorithm === undefined + ? (PromiseResolve() as Promise) + : abortAlgorithm(reason); + }; + + controllerClose = (controller) => { + // Enqueue the close marker (size 0) and advance. + ArrayPrototypePush(controller.#queue, { + value: kCloseMarker, + size: 0, + }); + controller.#advanceQueueIfNeeded(); + }; + + // WritableStreamDefaultControllerGetChunkSize (spec §5.5.4) + // Runs the strategy size algorithm; on failure, errors the stream and + // returns 1 (spec step 3). Called BEFORE state checks / write-request + // enqueue per WritableStreamDefaultWriterWrite step 4. + controllerGetChunkSize = ( + controller: WritableStreamDefaultController, + chunk: W + ): number => { + const sizeAlgorithm = controller.#sizeAlgorithm; + if (sizeAlgorithm === undefined) return 1; + try { + const size = +sizeAlgorithm(chunk); + if (NumberIsNaN(size) || size < 0 || size === Infinity) { + throw new RangeError('Invalid chunk size'); + } + return size; + } catch (e) { + controller.#errorIfNeeded(e); + return 1; + } + }; + + // WritableStreamDefaultControllerWrite (spec §5.5.4) + // Enqueues chunk with pre-computed chunkSize (from controllerGetChunkSize), + // updates backpressure, and advances the queue. + controllerWrite = ( + controller: WritableStreamDefaultController, + chunk: W, + chunkSize: number + ) => { + ArrayPrototypePush(controller.#queue, { value: chunk, size: chunkSize }); + controller.#queueTotalSize += chunkSize; + const stream = controller.#stream; + if ( + !writableStreamCloseQueuedOrInFlight(stream) && + getWritableStreamState(stream) === 'writable' + ) { + writableStreamUpdateBackpressure( + stream, + controllerGetDesiredSize(controller) <= 0 + ); + } + controller.#advanceQueueIfNeeded(); + }; + } + + constructor( + privateSymbol: symbol, + stream: WritableStream, + underlyingSink: UnderlyingSink, + sizeAlgorithm: (chunk: W) => number, + highWaterMark: number + ) { + assertPrivateSymbol(privateSymbol); + this.#stream = stream; + + // --- Sink method extraction (alphabetical property reads) --- + const abortFn = underlyingSink.abort; + if (abortFn !== undefined && typeof abortFn !== 'function') { + throw new TypeError('underlyingSink.abort must be a function'); + } + const closeFn = underlyingSink.close; + if (closeFn !== undefined && typeof closeFn !== 'function') { + throw new TypeError('underlyingSink.close must be a function'); + } + const startFn = underlyingSink.start; + if (startFn !== undefined && typeof startFn !== 'function') { + throw new TypeError('underlyingSink.start must be a function'); + } + const writeFn = underlyingSink.write; + if (writeFn !== undefined && typeof writeFn !== 'function') { + throw new TypeError('underlyingSink.write must be a function'); + } + + if (abortFn !== undefined) { + const callAbort = uncurryThis(abortFn); + this.#abortAlgorithm = (reason: unknown) => { + try { + return PromiseResolve( + callAbort(underlyingSink, reason) + ) as Promise; + } catch (e) { + return PromiseReject(e) as Promise; + } + }; + } + if (closeFn !== undefined) { + const callClose = uncurryThis(closeFn); + this.#closeAlgorithm = () => { + try { + return PromiseResolve(callClose(underlyingSink)) as Promise; + } catch (e) { + return PromiseReject(e) as Promise; + } + }; + } + if (writeFn !== undefined) { + const callWrite = uncurryThis(writeFn); + this.#writeAlgorithm = (chunk: W) => { + try { + return PromiseResolve( + callWrite(underlyingSink, chunk, this) + ) as Promise; + } catch (e) { + return PromiseReject(e) as Promise; + } + }; + } + + // Strategy already extracted by caller (WebIDL conversion order). + this.#sizeAlgorithm = sizeAlgorithm; + this.#strategyHWM = highWaterMark; + + // Initial backpressure signal. + writableStreamUpdateBackpressure( + stream, + controllerGetDesiredSize(this) <= 0 + ); + + // --- Start (sync throw propagates out of the WritableStream ctor) --- + const startResult: unknown = + startFn === undefined + ? undefined + : uncurryThis(startFn)(underlyingSink, this); + PromisePrototypeThen( + PromiseResolve(startResult), + () => { + this.#started = true; + this.#advanceQueueIfNeeded(); + }, + (e: unknown) => { + this.#started = true; + writableStreamDealWithRejection(this.#stream, e); + } + ); + } + + get signal(): AbortSignal { + if (!(#stream in this)) throw new TypeError('Illegal invocation'); + return AbortControllerSignalGet(this.#abortController); + } + + error(reason: unknown = undefined): void { + if (!(#stream in this)) throw new TypeError('Illegal invocation'); + if (getWritableStreamState(this.#stream) !== 'writable') return; + this.#errorStream(reason); + } + + #errorIfNeeded(error: unknown): void { + if (getWritableStreamState(this.#stream) === 'writable') { + this.#errorStream(error); + } + } + + #errorStream(error: unknown): void { + // WritableStreamDefaultControllerError: clear algorithms, start erroring. + // Note: the queue is NOT cleared here — it is cleared later during + // WritableStreamFinishErroring (via controllerErrorSteps). + this.#clearAlgorithms(); + writableStreamStartErroring(this.#stream, error); + } + + #clearAlgorithms(): void { + this.#writeAlgorithm = undefined; + this.#closeAlgorithm = undefined; + this.#abortAlgorithm = undefined; + this.#sizeAlgorithm = undefined; + } + + #advanceQueueIfNeeded(): void { + if (!this.#started) return; + const stream = this.#stream; + if (writableStreamHasOperationMarkedInFlight(stream)) return; + const state = getWritableStreamState(stream); + if (state === 'closed' || state === 'errored') return; + if (state === 'erroring') { + writableStreamFinishErroringIfNeeded(stream); + return; + } + const head = this.#queue[0]; + if (head === undefined) return; + if (head.value === kCloseMarker) { + this.#processClose(); + } else { + this.#processWrite(head.value as W); + } + } + + #processClose(): void { + const stream = this.#stream; + writableStreamMarkCloseRequestInFlight(stream); + // Dequeue the close marker; the queue must then be empty. + ArrayPrototypeShift(this.#queue); + this.#queueTotalSize = 0; + const closeAlgorithm = this.#closeAlgorithm; + this.#clearAlgorithms(); + const promise = + closeAlgorithm === undefined + ? (PromiseResolve() as Promise) + : closeAlgorithm(); + PromisePrototypeThen( + promise, + () => { + writableStreamFinishInFlightClose(stream); + }, + (e: unknown) => { + writableStreamFinishInFlightCloseWithError(stream, e); + } + ); + } + + #processWrite(chunk: W): void { + const stream = this.#stream; + writableStreamMarkFirstWriteRequestInFlight(stream); + const writeAlgorithm = this.#writeAlgorithm; + const promise = + writeAlgorithm === undefined + ? (PromiseResolve() as Promise) + : writeAlgorithm(chunk); + PromisePrototypeThen( + promise, + () => { + writableStreamFinishInFlightWrite(stream); + const state = getWritableStreamState(stream); + // Dequeue AFTER the write completes (spec ordering). + const entry = ArrayPrototypeShift(this.#queue) as QueuedWrite; + this.#queueTotalSize -= entry.size; + if (this.#queueTotalSize < 0) this.#queueTotalSize = 0; + if ( + !writableStreamCloseQueuedOrInFlight(stream) && + state === 'writable' + ) { + writableStreamUpdateBackpressure( + stream, + controllerGetDesiredSize(this) <= 0 + ); + } + this.#advanceQueueIfNeeded(); + }, + (e: unknown) => { + if (getWritableStreamState(stream) === 'writable') { + this.#clearAlgorithms(); + } + writableStreamFinishInFlightWriteWithError(stream, e); + } + ); + } + + [SymbolToStringTag] = 'WritableStreamDefaultController'; +} + +// --------------------------------------------------------------------------- +// WritableStreamDefaultWriter + +class WritableStreamDefaultWriter< + W = unknown, +> implements WritableStreamDefaultWriterType { + #stream: WritableStream | undefined; + #readyPromise!: Promise | PromiseWithResolversType; + #closedPromise!: Promise | PromiseWithResolversType; + + static { + getWriterStream = (writer) => writer.#stream; + + getWriterReadyPromiseInternal = (writer) => { + const promise = writer.#readyPromise; + if (isPromise(promise)) return promise as Promise; + return (promise as PromiseWithResolversType).promise; + }; + + getWriterClosedPromiseInternal = (writer) => { + const promise = writer.#closedPromise; + if (isPromise(promise)) return promise as Promise; + return (promise as PromiseWithResolversType).promise; + }; + + // WritableStreamDefaultWriterWrite (spec §5.3.4) + // Step order is critical: size algorithm runs BEFORE state re-checks, + // because size() can re-entrantly call close()/releaseLock()/error(). + writerWriteInternal = ( + writer: WritableStreamDefaultWriter, + chunk: W + ) => { + const stream = writer.#stream; + if (stream === undefined) { + return PromiseReject( + new TypeError('This writer has been released') + ) as Promise; + } + const controller = getWritableStreamController(stream); + + // Step 4: GetChunkSize — runs size() which may re-entrantly mutate + // stream state (e.g. writer.close() called from size()). + const chunkSize = + controller !== undefined + ? controllerGetChunkSize(controller, chunk) + : 1; + + // Step 5: Re-check writer.[[stream]] — size() may have called + // releaseLock(). + if (writer.#stream !== stream) { + return PromiseReject( + new TypeError('This writer has been released') + ) as Promise; + } + + // Steps 6–9: State checks (AFTER size ran). + const state = getWritableStreamState(stream); + if (state === 'errored') { + return PromiseReject( + getWritableStreamStoredError(stream) + ) as Promise; + } + if (writableStreamCloseQueuedOrInFlight(stream) || state === 'closed') { + return PromiseReject( + new TypeError('Cannot write to a stream that is closing or closed') + ) as Promise; + } + if (state === 'erroring') { + return PromiseReject( + getWritableStreamStoredError(stream) + ) as Promise; + } + + // Step 11: Add write request. + const promise = writableStreamAddWriteRequest(stream); + + // Step 12: ControllerWrite (with pre-computed chunkSize, no size() + // call — that already happened in step 4). + if (controller !== undefined) { + controllerWrite(controller, chunk, chunkSize); + } + return promise; + }; + + writerCloseInternal = (writer: WritableStreamDefaultWriter) => { + const stream = writer.#stream; + if (stream === undefined) { + return PromiseReject( + new TypeError('This writer has been released') + ) as Promise; + } + if (writableStreamCloseQueuedOrInFlight(stream)) { + return PromiseReject( + new TypeError('Stream is already closing') + ) as Promise; + } + return writableStreamCloseInternal(stream); + }; + + writerReleaseInternal = (writer: WritableStreamDefaultWriter) => { + const stream = writer.#stream; + if (stream === undefined) return; + const releaseError = new TypeError('This writer has been released'); + writerEnsureReadyPromiseRejected(writer, releaseError); + writerEnsureClosedPromiseRejected(writer, releaseError); + setWritableStreamWriter(stream, undefined); + writer.#stream = undefined; + }; + + writerResolveReadyPromise = (writer) => { + const ready = writer.#readyPromise as PromiseWithResolversType; + if (typeof ready.resolve === 'function') { + ready.resolve(); + writer.#readyPromise = ready.promise; + } + }; + + writerResetReadyPromise = (writer) => { + const replacement = + PromiseWithResolvers() as PromiseWithResolversType; + markPromiseHandled(replacement.promise); + writer.#readyPromise = replacement; + }; + + writerEnsureReadyPromiseRejected = (writer, error) => { + const ready = writer.#readyPromise as PromiseWithResolversType; + if (typeof ready.reject === 'function') { + ready.reject(error); + writer.#readyPromise = ready.promise; + } else { + const replacement = + PromiseWithResolvers() as PromiseWithResolversType; + markPromiseHandled(replacement.promise); + replacement.reject(error); + writer.#readyPromise = replacement.promise; + } + }; + + writerResolveClosedPromise = (writer) => { + const closed = writer.#closedPromise as PromiseWithResolversType; + if (typeof closed.resolve === 'function') { + closed.resolve(); + writer.#closedPromise = closed.promise; + } + }; + + writerEnsureClosedPromiseRejected = (writer, error) => { + const closed = writer.#closedPromise as PromiseWithResolversType; + if (typeof closed.reject === 'function') { + closed.reject(error); + writer.#closedPromise = closed.promise; + } else { + const replacement = + PromiseWithResolvers() as PromiseWithResolversType; + markPromiseHandled(replacement.promise); + replacement.reject(error); + writer.#closedPromise = replacement.promise; + } + }; + } + + constructor(stream: WritableStream) { + if (!isActualObject(stream)) { + throw new TypeError('stream must be a WritableStream'); + } + // The locked check doubles as the brand check: the private-field access + // inside isWritableStreamLocked throws TypeError for non-WritableStream + // objects. + if (isWritableStreamLocked(stream)) { + throw new TypeError('Cannot get a writer for a stream that is locked'); + } + this.#stream = stream; + setWritableStreamWriter(stream, this); + + const state = getWritableStreamState(stream); + switch (state) { + case 'writable': { + const closed = PromiseWithResolvers() as PromiseWithResolversType; + markPromiseHandled(closed.promise); + this.#closedPromise = closed; + if ( + !writableStreamCloseQueuedOrInFlight(stream) && + writableStreamBackpressureOf(stream) + ) { + const ready = + PromiseWithResolvers() as PromiseWithResolversType; + markPromiseHandled(ready.promise); + this.#readyPromise = ready; + } else { + this.#readyPromise = PromiseResolve() as Promise; + } + break; + } + case 'erroring': { + const storedError = getWritableStreamStoredError(stream); + const ready = PromiseWithResolvers() as PromiseWithResolversType; + markPromiseHandled(ready.promise); + ready.reject(storedError); + this.#readyPromise = ready.promise; + const closed = PromiseWithResolvers() as PromiseWithResolversType; + markPromiseHandled(closed.promise); + this.#closedPromise = closed; + break; + } + case 'closed': { + this.#readyPromise = PromiseResolve() as Promise; + this.#closedPromise = PromiseResolve() as Promise; + break; + } + default: { + // 'errored' + const storedError = getWritableStreamStoredError(stream); + const ready = PromiseWithResolvers() as PromiseWithResolversType; + markPromiseHandled(ready.promise); + ready.reject(storedError); + this.#readyPromise = ready.promise; + const closed = PromiseWithResolvers() as PromiseWithResolversType; + markPromiseHandled(closed.promise); + closed.reject(storedError); + this.#closedPromise = closed.promise; + break; + } + } + } + + get closed(): Promise { + try { + const promise = this.#closedPromise; + if (isPromise(promise)) return promise as Promise; + return (promise as PromiseWithResolversType).promise; + } catch (e) { + return PromiseReject(e) as Promise; + } + } + + get ready(): Promise { + try { + const promise = this.#readyPromise; + if (isPromise(promise)) return promise as Promise; + return (promise as PromiseWithResolversType).promise; + } catch (e) { + return PromiseReject(e) as Promise; + } + } + + get desiredSize(): number | null { + const stream = this.#stream; + if (stream === undefined) { + throw new TypeError('This writer has been released'); + } + const state = getWritableStreamState(stream); + if (state === 'errored' || state === 'erroring') return null; + if (state === 'closed') return 0; + const controller = getWritableStreamController(stream); + return controller === undefined ? 0 : controllerGetDesiredSize(controller); + } + + abort(reason: unknown = undefined): Promise { + try { + const stream = this.#stream; + if (stream === undefined) { + throw new TypeError('This writer has been released'); + } + return writableStreamAbort(stream, reason); + } catch (e) { + return PromiseReject(e) as Promise; + } + } + + close(): Promise { + try { + // Brand check via private access. + if (this.#stream === undefined) { + throw new TypeError('This writer has been released'); + } + return writerCloseInternal(this); + } catch (e) { + return PromiseReject(e) as Promise; + } + } + + write(chunk: W = undefined as W): Promise { + try { + if (!(#stream in this)) throw new TypeError('Illegal invocation'); + return writerWriteInternal(this, chunk); + } catch (e) { + return PromiseReject(e) as Promise; + } + } + + releaseLock(): void { + if (!(#stream in this)) throw new TypeError('Illegal invocation'); + writerReleaseInternal(this); + } + + [SymbolToStringTag] = 'WritableStreamDefaultWriter'; +} + +// Spec WritableStreamDefaultWriterCloseWithErrorPropagation — the pipe's +// close path: an already-closing/closed destination is success, an errored +// one propagates its stored error. +function writerCloseWithErrorPropagation( + writer: WritableStreamDefaultWriter +): Promise { + const stream = getWriterStream(writer); + if (stream === undefined) { + return PromiseReject( + new TypeError('This writer has been released') + ) as Promise; + } + const state = getWritableStreamState(stream); + if (writableStreamCloseQueuedOrInFlight(stream) || state === 'closed') { + return PromiseResolve() as Promise; + } + if (state === 'errored') { + return PromiseReject(getWritableStreamStoredError(stream)) as Promise; + } + return writerCloseInternal(writer); +} + +ObjectDefineProperties(WritableStream, { + length: { value: 0 }, +}); +ObjectDefineProperties(WritableStreamDefaultController, { + length: { value: 0 }, +}); +ObjectDefineProperties(WritableStream.prototype, { + locked: { enumerable: true }, + abort: { enumerable: true }, + close: { enumerable: true }, + getWriter: { enumerable: true }, +}); +ObjectDefineProperties(WritableStreamDefaultWriter.prototype, { + closed: { enumerable: true }, + ready: { enumerable: true }, + desiredSize: { enumerable: true }, + abort: { enumerable: true }, + close: { enumerable: true }, + write: { enumerable: true }, + releaseLock: { enumerable: true }, +}); +ObjectDefineProperties(WritableStreamDefaultController.prototype, { + signal: { enumerable: true }, + error: { enumerable: true }, +}); + +module.exports = { + WritableStream, + WritableStreamDefaultWriter, + WritableStreamDefaultController, + // Internal operations consumed by the pipe implementation in readable.ts. + // Unreachable from user code: the bootstrap require() is not exposed, and + // streams.ts deliberately does not re-export this. + // Sink-side symbols for the pipe dispatch in readable.ts. + kExtractNativeSink, + internalsForPipe: { + acquireWriter( + stream: WritableStream + ): WritableStreamDefaultWriter { + return new WritableStreamDefaultWriter(stream); + }, + writerWrite: (writer: WritableStreamDefaultWriter, chunk: W) => + writerWriteInternal(writer, chunk), + writerCloseWithErrorPropagation, + writerRelease: (writer: WritableStreamDefaultWriter) => + writerReleaseInternal(writer), + writableStreamAbort: (stream: WritableStream, reason: unknown) => + writableStreamAbort(stream, reason), + getState: (stream: WritableStream) => getWritableStreamState(stream), + getStoredError: (stream: WritableStream) => + getWritableStreamStoredError(stream), + closeQueuedOrInFlight: (stream: WritableStream) => + writableStreamCloseQueuedOrInFlight(stream), + getWriterReadyPromise: (writer: WritableStreamDefaultWriter) => + getWriterReadyPromiseInternal(writer), + getWriterClosedPromise: (writer: WritableStreamDefaultWriter) => + getWriterClosedPromiseInternal(writer), + }, +}; diff --git a/src/workerd/api/tests/BUILD.bazel b/src/workerd/api/tests/BUILD.bazel index a11facc4e8b..303947a8b01 100644 --- a/src/workerd/api/tests/BUILD.bazel +++ b/src/workerd/api/tests/BUILD.bazel @@ -1133,6 +1133,12 @@ wd_test( data = ["htmlrewriter-transform-cancel-test.js"], ) +wd_test( + src = "ts-webstreams-test.wd-test", + args = ["--experimental"], + data = ["ts-webstreams-test.js"], +) + sh_test( name = "abortIsolate", size = "medium", diff --git a/src/workerd/api/tests/ts-webstreams-test.js b/src/workerd/api/tests/ts-webstreams-test.js new file mode 100644 index 00000000000..862d7e765f5 --- /dev/null +++ b/src/workerd/api/tests/ts-webstreams-test.js @@ -0,0 +1,50 @@ +// Copyright (c) 2026 Cloudflare, Inc. +// Licensed under the Apache 2.0 license found in the LICENSE file or at: +// https://opensource.org/licenses/Apache-2.0 workerd/require-copyright-header +const { + ReadableStream, + ReadableStreamDefaultReader, + ReadableStreamBYOBReader, + ReadableStreamDefaultController, + ReadableByteStreamController, + ReadableStreamBYOBRequest, + ByteLengthQueuingStrategy, + CountQueuingStrategy, + WritableStream, + WritableStreamDefaultWriter, + WritableStreamDefaultController, + TransformStream, + TransformStreamDefaultController, + IdentityTransformStream, + FixedLengthStream, + TextEncoderStream, + TextDecoderStream, +} = globalThis; + +import { doesNotMatch } from 'node:assert'; + +export const existenceTest = { + test() { + // None of the classes should report as "native code" + doesNotMatch(ReadableStream.toString(), /\[native code\]/); + doesNotMatch(ReadableStreamDefaultReader.toString(), /\[native code\]/); + doesNotMatch(ReadableStreamBYOBReader.toString(), /\[native code\]/); + doesNotMatch(ReadableStreamDefaultController.toString(), /\[native code\]/); + doesNotMatch(ReadableByteStreamController.toString(), /\[native code\]/); + doesNotMatch(ReadableStreamBYOBRequest.toString(), /\[native code\]/); + doesNotMatch(ByteLengthQueuingStrategy.toString(), /\[native code\]/); + doesNotMatch(CountQueuingStrategy.toString(), /\[native code\]/); + doesNotMatch(WritableStream.toString(), /\[native code\]/); + doesNotMatch(WritableStreamDefaultWriter.toString(), /\[native code\]/); + doesNotMatch(WritableStreamDefaultController.toString(), /\[native code\]/); + doesNotMatch(TransformStream.toString(), /\[native code\]/); + doesNotMatch( + TransformStreamDefaultController.toString(), + /\[native code\]/ + ); + doesNotMatch(IdentityTransformStream.toString(), /\[native code\]/); + doesNotMatch(FixedLengthStream.toString(), /\[native code\]/); + doesNotMatch(TextEncoderStream.toString(), /\[native code\]/); + doesNotMatch(TextDecoderStream.toString(), /\[native code\]/); + }, +}; diff --git a/src/workerd/api/tests/ts-webstreams-test.wd-test b/src/workerd/api/tests/ts-webstreams-test.wd-test new file mode 100644 index 00000000000..a4686f361d5 --- /dev/null +++ b/src/workerd/api/tests/ts-webstreams-test.wd-test @@ -0,0 +1,21 @@ +using Workerd = import "/workerd/workerd.capnp"; + +const unitTests :Workerd.Config = ( + services = [ + ( name = "ts-webstreams-test", + worker = ( + modules = [ + (name = "worker", esModule = embed "ts-webstreams-test.js") + ], + compatibilityFlags = [ + "nodejs_compat", + "typescript_implemented_streams", + "experimental", + ], + ) + ), + ], + autogates = [ + "workerd-autogate-per-isolate-javascript-bootstrap", + ], +); diff --git a/src/workerd/io/compatibility-date.capnp b/src/workerd/io/compatibility-date.capnp index 3bd565deee3..fcfe4079550 100644 --- a/src/workerd/io/compatibility-date.capnp +++ b/src/workerd/io/compatibility-date.capnp @@ -1610,4 +1610,10 @@ struct CompatibilityFlags @0x8f8c1b68151b6cef { # dispatches its methods as JSRPC calls on the inner fetcher instead of HTTP # requests against the binding-shim worker. Without the flag the legacy HTTP # transport is used. + + typeScriptImplementedStreams @183 :Bool + $compatEnableFlag("typescript_implemented_streams") + $experimental; + # When enabled, the workers runtime uses the new typescript Web Streams + # implementation. } diff --git a/src/wpt/BUILD.bazel b/src/wpt/BUILD.bazel index 14dd77e2b02..c7a0dfd1637 100644 --- a/src/wpt/BUILD.bazel +++ b/src/wpt/BUILD.bazel @@ -116,6 +116,15 @@ wpt_test( wpt_directory = "@wpt//:streams@module", ) +wpt_test( + name = "streams-ts", + size = "large", + autogates = ["workerd-autogate-per-isolate-javascript-bootstrap"], + compat_flags = ["typescript_implemented_streams"], + config = "streams-test-ts.ts", + wpt_directory = "@wpt//:streams@module", +) + wpt_test( name = "fs", size = "large", diff --git a/src/wpt/harness/assertions.ts b/src/wpt/harness/assertions.ts index bb69276b507..a647916cf97 100644 --- a/src/wpt/harness/assertions.ts +++ b/src/wpt/harness/assertions.ts @@ -239,8 +239,32 @@ globalThis.assert_array_equals = (actual, expected, description): void => { } }; +// Recursively copy null-prototype objects into regular {__proto__: Object.prototype} +// objects so that deepStrictEqual (which checks [[Prototype]] identity) does not +// reject them. This is necessary because our streams implementation returns +// { value, done } read-result objects with a null prototype to prevent +// Object.prototype.then interception in V8's PromiseResolveFunction — an +// intentional spec variance. WPT tests never rely on read-result prototypes in +// assert_object_equals comparisons, so normalizing is safe. +function normalizeNullPrototype(obj: unknown): unknown { + if (obj === null || typeof obj !== 'object') return obj; + if (Array.isArray(obj)) return (obj as unknown[]).map(normalizeNullPrototype); + const rec = obj as Record; + const keys = Object.keys(rec); + const out: Record = {}; + for (let i = 0; i < keys.length; i++) { + const key = keys[i] as string; + out[key] = normalizeNullPrototype(rec[key]); + } + return out; +} + globalThis.assert_object_equals = (a, b, message): void => { - deepStrictEqual(a, b, message); + deepStrictEqual( + normalizeNullPrototype(a), + normalizeNullPrototype(b), + message + ); }; /** diff --git a/src/wpt/streams-test-ts.ts b/src/wpt/streams-test-ts.ts new file mode 100644 index 00000000000..ecbb24d33f8 --- /dev/null +++ b/src/wpt/streams-test-ts.ts @@ -0,0 +1,220 @@ +// Copyright (c) 2017-2022 Cloudflare, Inc. +// Licensed under the Apache 2.0 license found in the LICENSE file or at: +// https://opensource.org/licenses/Apache-2.0 + +import { type TestRunnerConfig } from 'harness/harness'; + +export default { + 'idlharness.any.js': {}, + 'piping/abort.any.js': { + comment: + 'Microtask ordering: the async pump cannot detect source-close ' + + 'before a same-tick abort fires, so the abort wins the race ' + + 'against the spec condition-3 (source-close) shutdown.', + expectedFailures: ['abort should do nothing after the readable is closed'], + }, + 'piping/close-propagation-backward.any.js': {}, + 'piping/close-propagation-forward.any.js': {}, + 'piping/error-propagation-backward.any.js': {}, + 'piping/error-propagation-forward.any.js': {}, + 'piping/flow-control.any.js': { + comment: + 'Backpressure tracking: desiredSize not decremented during pipe writes due to differences in the way draining read works', + expectedFailures: [ + 'Piping to a WritableStream that does not consume the writes fast enough exerts backpressure on the ReadableStream', + ], + }, + 'piping/general-addition.any.js': {}, + 'piping/general.any.js': {}, + 'piping/multiple-propagation.any.js': {}, + 'piping/pipe-through.any.js': {}, + 'piping/then-interception.any.js': {}, + 'piping/throwing-options.any.js': {}, + 'piping/transform-streams.any.js': {}, + + 'queuing-strategies-size-function-per-global.window.js': { + comment: 'document is not defined', + disabledTests: true, + }, + 'queuing-strategies.any.js': {}, + 'readable-byte-streams/bad-buffers-and-views.any.js': {}, + 'readable-byte-streams/construct-byob-request.any.js': {}, + 'readable-byte-streams/crashtests/tee-locked-stream.any.js': {}, + 'readable-byte-streams/enqueue-with-detached-buffer.any.js': {}, + 'readable-byte-streams/general.any.js': {}, + 'readable-byte-streams/non-transferable-buffers.any.js': {}, + 'readable-byte-streams/patched-global.any.js': { + runInGlobalScope: true, + }, + 'readable-byte-streams/read-min.any.js': { + comment: + 'tee+min byobRequest null due to shared-queue multi-cursor model ' + + '(byobRequest requires singleCursor). Intentional divergence.', + expectedFailures: [ + 'ReadableStream with byte source: tee() with read({ min }) from branch1 and read() from branch2', + ], + }, + 'readable-byte-streams/respond-after-enqueue.any.js': {}, + 'readable-byte-streams/tee.any.js': { + comment: + 'AggregateError cancel reason (intentional divergence, tests hang). ' + + 'byobRequest null during tee (shared-queue multi-cursor model).', + disabledTests: [ + // HANG: AggregateError cancel reason instead of spec [r1, r2] array. + // Tests create an unresolved Promise on assertion failure and the + // validate() Promise.all hangs the entire file. + 'ReadableStream teeing with byte source: canceling both branches should aggregate the cancel reasons into an array', + 'ReadableStream teeing with byte source: canceling both branches in reverse order should aggregate the cancel reasons into an array', + ], + expectedFailures: [ + // AggregateError cancel reason: cancel reason is AggregateError + // instead of spec [r1, r2] array. These fail fast on assertion. + 'ReadableStream teeing with byte source: failing to cancel when canceling both branches in sequence with delay', + 'ReadableStream teeing with byte source: failing to cancel the original stream should cause cancel() to reject on branches', + // Shared-queue tee model: pull count / pull sequencing differs + // from the spec's per-branch clone model. + 'ReadableStream teeing with byte source: stops pulling when original stream errors while both branches are reading', + // Shared-queue tee model: byte offset handling on cloned views + // differs from spec's per-branch transfer. + 'ReadableStream teeing with byte source: reading an array with a byte offset should clone correctly', + // byobRequest null during tee: shared-queue multi-cursor model + // makes byobRequest unavailable. These fail fast (TypeError or + // assertion), no hang risk. + 'ReadableStream teeing with byte source: chunks for BYOB requests from branch 1 should be cloned to branch 2', + 'ReadableStream teeing with byte source: read from branch1 and branch2, cancel branch1, cancel branch2', + 'ReadableStream teeing with byte source: read from branch1 and branch2, cancel branch2, cancel branch1', + 'ReadableStream teeing with byte source: read from branch1 and branch2, cancel branch2, enqueue to branch1', + 'ReadableStream teeing with byte source: read from branch1 and branch2, cancel branch1, respond to branch2', + 'ReadableStream teeing with byte source: pull with BYOB reader, then pull with default reader', + 'ReadableStream teeing with byte source: pull with default reader, then pull with BYOB reader', + 'ReadableStream teeing with byte source: read from branch2, then read from branch1', + 'ReadableStream teeing with byte source: close when both branches have pending BYOB reads', + 'ReadableStream teeing with byte source: respond() and close() while both branches are pulling', + ], + }, + 'readable-byte-streams/templated.any.js': {}, + + 'readable-streams/async-iterator.any.js': {}, + 'readable-streams/bad-strategies.any.js': {}, + 'readable-streams/bad-underlying-sources.any.js': {}, + 'readable-streams/cancel.any.js': {}, + 'readable-streams/constructor.any.js': {}, + 'readable-streams/count-queuing-strategy-integration.any.js': {}, + 'readable-streams/crashtests/empty.js': {}, + 'readable-streams/crashtests/garbage-collection.any.js': {}, + 'readable-streams/crashtests/strategy-worker.js': { + comment: 'ReferenceError: importScripts is not defined', + disabledTests: true, + }, + 'readable-streams/cross-realm-crash.window.js': { + comment: 'document is not defined', + expectedFailures: [ + 'should not crash on reading from stream cancelled in destroyed realm', + ], + }, + 'readable-streams/default-reader.any.js': {}, + 'readable-streams/floating-point-total-queue-size.any.js': {}, + 'readable-streams/from.any.js': { + comment: + 'Intentional divergence: we treat strings as single chunks, not code-point iterables', + expectedFailures: [ + // INTENTIONAL SPEC DIVERGENCE: The spec iterates strings code-point- + // by-code-point, but that is surprising to users and has terrible + // performance. We treat strings as single chunks instead. + 'ReadableStream.from accepts a string', + ], + }, + 'readable-streams/garbage-collection.any.js': {}, + 'readable-streams/general.any.js': {}, + 'readable-streams/owning-type-message-port.any.js': { + comment: + 'Enable once MessageChannel/MessagePort is implemented and owning type is implemented', + disabledTests: true, + }, + 'readable-streams/owning-type-video-frame.any.js': { + comment: 'VideoFrame is not implemented', + disabledTests: true, + }, + 'readable-streams/owning-type.any.js': { + comment: "Type 'owning' is not part of the spec yet — not implemented", + disabledTests: true, + }, + 'readable-streams/patched-global.any.js': { + runInGlobalScope: true, + }, + 'readable-streams/read-task-handling.window.js': { + comment: 'document is not defined', + disabledTests: true, + }, + 'readable-streams/reentrant-strategies.any.js': {}, + 'readable-streams/tee.any.js': { + comment: + 'Intentional divergence: tee cancel uses AggregateError instead of spec array, ' + + 'and shared-queue architecture causes different pull counts vs spec clone model', + disabledTests: true, + // These two HANG: assert_array_equals(AggregateError, [...]) throws inside + // the cancel callback, preventing the test promise from ever resolving. + // 'ReadableStream teeing: canceling both branches should aggregate the cancel reasons into an array', + // 'ReadableStream teeing: canceling both branches in reverse order should aggregate the cancel reasons into an array', + }, + 'readable-streams/templated.any.js': {}, + + 'transferable/deserialize-error.window.js': { + comment: 'ReferenceError: document is not defined', + disabledTests: true, + }, + 'transferable/transfer-with-messageport.window.js': { + comment: 'Enable once MessagePort is supported.', + expectedFailures: [ + 'ReadableStream must not be serializable', + 'WritableStream must not be serializable', + 'TransformStream must not be serializable', + 'Transferring a MessagePort with a ReadableStream should set `.ports`', + 'Transferring a MessagePort with a WritableStream should set `.ports`', + 'Transferring a MessagePort with a TransformStream should set `.ports`', + 'Transferring a MessagePort with a ReadableStream should set `.ports`, advanced', + 'Transferring a MessagePort with a WritableStream should set `.ports`, advanced', + 'Transferring a MessagePort with a TransformStream should set `.ports`, advanced', + 'Transferring a MessagePort with multiple streams should set `.ports`', + ], + }, + 'transferable/transform-stream-members.any.js': { + comment: 'Appears to be about the wrong type of error', + expectedFailures: [ + 'Transferring [object TransformStream],[object ReadableStream] should fail', + 'Transferring [object ReadableStream],[object TransformStream] should fail', + 'Transferring [object TransformStream],[object WritableStream] should fail', + 'Transferring [object WritableStream],[object TransformStream] should fail', + ], + }, + + 'transform-streams/backpressure.any.js': {}, + 'transform-streams/cancel.any.js': {}, + 'transform-streams/errors.any.js': {}, + 'transform-streams/flush.any.js': {}, + 'transform-streams/general.any.js': {}, + 'transform-streams/lipfuzz.any.js': {}, + 'transform-streams/patched-global.any.js': { + runInGlobalScope: true, + }, + 'transform-streams/properties.any.js': {}, + 'transform-streams/reentrant-strategies.any.js': {}, + 'transform-streams/strategies.any.js': {}, + 'transform-streams/terminate.any.js': {}, + 'writable-streams/aborting.any.js': {}, + 'writable-streams/bad-strategies.any.js': {}, + 'writable-streams/bad-underlying-sinks.any.js': {}, + 'writable-streams/byte-length-queuing-strategy.any.js': {}, + 'writable-streams/close.any.js': {}, + 'writable-streams/constructor.any.js': {}, + 'writable-streams/count-queuing-strategy.any.js': {}, + 'writable-streams/crashtests/garbage-collection.any.js': {}, + 'writable-streams/error.any.js': {}, + 'writable-streams/floating-point-total-queue-size.any.js': {}, + 'writable-streams/garbage-collection.any.js': {}, + 'writable-streams/general.any.js': {}, + 'writable-streams/properties.any.js': {}, + 'writable-streams/reentrant-strategy.any.js': {}, + 'writable-streams/start.any.js': {}, + 'writable-streams/write.any.js': {}, +} satisfies TestRunnerConfig; From dcfcfda837a7f4815c310b5f80ea74277596c2f3 Mon Sep 17 00:00:00 2001 From: LongYinan Date: Tue, 7 Jul 2026 10:15:15 +0800 Subject: [PATCH 033/101] Document DataUrl base64 compatibility behavior --- src/workerd/api/data-url.c++ | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/workerd/api/data-url.c++ b/src/workerd/api/data-url.c++ index 5ea494df9a1..11b3d264682 100644 --- a/src/workerd/api/data-url.c++ +++ b/src/workerd/api/data-url.c++ @@ -12,6 +12,9 @@ namespace { kj::Array decodeDataUrlBase64(kj::ArrayPtr input) { kj::Vector filtered(input.size()); + // DataUrl historically ignored kj::decodeBase64().hadErrors. That decoder skips bytes outside + // the base64 alphabet, including padding, and decodes the remaining characters. Preserve that + // permissive behavior before passing the input to simdutf's stricter decoder. for (kj::byte c: input) { if (isAlpha(c) || isDigit(c) || c == '+' || c == '/') { filtered.add(static_cast(c)); @@ -19,6 +22,8 @@ kj::Array decodeDataUrlBase64(kj::ArrayPtr input) { } auto base64 = filtered.asPtr(); + // KJ drops a final lone base64 character because it cannot produce a complete byte. simdutf + // rejects inputs whose length is 1 modulo 4, so trim that character to preserve KJ's result. if (base64.size() % 4 == 1) { base64 = base64.first(base64.size() - 1); } @@ -29,6 +34,8 @@ kj::Array decodeDataUrlBase64(kj::ArrayPtr input) { auto size = simdutf::maximal_binary_length_from_base64(base64.begin(), base64.size()); auto decoded = kj::heapArray(size); + // The default loose tail handling preserves KJ's acceptance of unpadded two- and three-character + // tails and non-zero unused bits. auto result = simdutf::base64_to_binary( base64.begin(), base64.size(), decoded.asChars().begin(), simdutf::base64_default); if (result.error != simdutf::SUCCESS || result.count == 0) { From 81303b4c5fdce833df9535be273e2725d0553dd1 Mon Sep 17 00:00:00 2001 From: Ashley Peacock Date: Mon, 6 Jul 2026 10:41:47 +0100 Subject: [PATCH 034/101] Fix hibernatable WebSocket DOs failing to hibernate on concurrent upgrades IncomingRequest::drain() only consumed its `self` owner on the non-early-return path, so when a newer request had already taken over, the caller's owner lingered for the lifetime of its Own. For hibernatable WebSockets that owner lives for the whole connection, pinning the IncomingRequest (and its actor ActiveRequest) and preventing the Durable Object from hibernating. Move self into a local so every return path drops the request, and add a unit test covering the early-return contract. Release note: None --- src/workerd/io/BUILD.bazel | 8 ++++ src/workerd/io/incoming-request-test.c++ | 61 ++++++++++++++++++++++++ src/workerd/io/io-context.c++ | 5 +- 3 files changed, 73 insertions(+), 1 deletion(-) create mode 100644 src/workerd/io/incoming-request-test.c++ diff --git a/src/workerd/io/BUILD.bazel b/src/workerd/io/BUILD.bazel index c05a4f02f61..6875eb2afbd 100644 --- a/src/workerd/io/BUILD.bazel +++ b/src/workerd/io/BUILD.bazel @@ -551,3 +551,11 @@ kj_test( "//src/workerd/tests:test-fixture", ], ) + +kj_test( + src = "incoming-request-test.c++", + deps = [ + ":io", + "//src/workerd/tests:test-fixture", + ], +) diff --git a/src/workerd/io/incoming-request-test.c++ b/src/workerd/io/incoming-request-test.c++ new file mode 100644 index 00000000000..7d43aa940b8 --- /dev/null +++ b/src/workerd/io/incoming-request-test.c++ @@ -0,0 +1,61 @@ +// Copyright (c) 2026 Cloudflare, Inc. +// Licensed under the Apache 2.0 license found in the LICENSE file or at: +// https://opensource.org/licenses/Apache-2.0 +// +// Tests for IoContext::IncomingRequest lifecycle behavior. + +#include +#include +#include + +#include +#include + +namespace workerd { +namespace { + +class ErrorHandlerImpl: public kj::TaskSet::ErrorHandler { + public: + void taskFailed(kj::Exception&& exception) override { + KJ_FAIL_EXPECT(exception); + } +}; + +// Regression test: two IncomingRequests share a single actor IoContext, as happens when a Durable +// Object receives overlapping requests. Draining the older, superseded request hits drain()'s +// "a newer request has taken over" early return. +// +// drain() takes ownership of the request via an rvalue-reference `self` parameter, but it must +// actually consume `self` on *every* return path — including the early return. Otherwise the +// caller's owner lingers for as long as the caller holds its kj::Own. In production the caller is +// a hibernatable WebSocket's deferred-proxy task, whose owner lives for the entire connection, so +// failing to release it here pins the IncomingRequest (and the actor ActiveRequest it carries) and +// prevents the Durable Object from ever hibernating. +KJ_TEST("IoContext::IncomingRequest::drain() releases a superseded (non-front) request") { + TestFixture fixture({.actorId = Worker::Actor::Id(kj::str("drain-test"))}); + + // One IoContext (the actor) with two IncomingRequests delivered against it. delivered() adds the + // request to the front of the list, so `second` becomes the front and `first` is superseded. + auto context = fixture.newIoContext(); + auto first = fixture.newIncomingRequest(*context); + auto second = fixture.newIncomingRequest(*context); + + ErrorHandlerImpl errorHandler; + kj::TaskSet waitUntilTasks(errorHandler); + + // Drain the superseded request. `kj::mv(first)` only casts to an rvalue reference; the Own is + // not cleared unless drain() moves out of it. So `first` being null afterwards proves that + // drain() took ownership on the early-return path. + first->drain(waitUntilTasks, kj::mv(first)); + KJ_EXPECT(first.get() == nullptr, + "drain() must consume `self` even when a newer request has already taken over"); + + // The early-return path schedules no background work. + KJ_EXPECT(waitUntilTasks.isEmpty()); + + // Tidy up the still-live front request so its destructor doesn't warn about undrained tasks. + fixture.drainAndDestroy(kj::mv(second)); +} + +} // namespace +} // namespace workerd diff --git a/src/workerd/io/io-context.c++ b/src/workerd/io/io-context.c++ index 469096f23a6..2cec0b1105f 100644 --- a/src/workerd/io/io-context.c++ +++ b/src/workerd/io/io-context.c++ @@ -584,6 +584,9 @@ kj::Promise IoContext::IncomingRequest::maybeAddGcPassForTest(kj::Promise // Mark ourselves so we know that we made a best effort attempt to wait for waitUntilTasks. void IoContext::IncomingRequest::drain( kj::TaskSet& waitUntilTasks, kj::Own&& self) { + // Passing by rvalue reference keeps the call-site evaluation order safe, but does not itself + // consume the caller's owner. Move immediately so early returns still drop the request. + auto ownedSelf = kj::mv(self); waitedForWaitUntil = true; if (&context->incomingRequests.front() != this) { @@ -618,7 +621,7 @@ void IoContext::IncomingRequest::drain( .exclusiveJoin(kj::mv(timeoutPromise)) .exclusiveJoin(context->onAbort()); - result = result.attach(kj::mv(self)); + result = result.attach(kj::mv(ownedSelf)); KJ_IF_SOME(a, context->actor) { // Make sure the drain is canceled and the IncomingRequest dropped on actor abort. From 09ca3e9f8b92c6a24f6f13535112e62e70b20bba Mon Sep 17 00:00:00 2001 From: James M Snell Date: Tue, 30 Jun 2026 14:29:01 -0700 Subject: [PATCH 035/101] Simplify Body::Buffer handling Now that kj::Rc allows for `T` that is not refcounted, we can `kj::Rc` directly, drop `RefcountedBytes`, and eliminate the `Body::Buffer::clone()` method. Step one in simplifying the object structure around payloads in preparation for adding support for the TS-backed streams. --- src/workerd/api/http.c++ | 64 ++++++++++++++++------------------------ src/workerd/api/http.h | 34 ++++++++------------- 2 files changed, 39 insertions(+), 59 deletions(-) diff --git a/src/workerd/api/http.c++ b/src/workerd/api/http.c++ index 8f723dc9c88..9610ddb66bc 100644 --- a/src/workerd/api/http.c++ +++ b/src/workerd/api/http.c++ @@ -74,25 +74,9 @@ jsg::Optional getCacheModeName(Request::CacheMode mode) { // capitalization). So, it's certainly not worth it to try to keep the original capitalization // across serialization. -Body::Buffer Body::Buffer::clone(jsg::Lock& js) { - Buffer result; - result.view = view; - KJ_SWITCH_ONEOF(ownBytes) { - KJ_CASE_ONEOF(ref, jsg::JsRef) { - result.ownBytes = ref.addRef(js); - } - KJ_CASE_ONEOF(refcounted, kj::Own) { - result.ownBytes = kj::addRef(*refcounted); - } - KJ_CASE_ONEOF(blob, jsg::Ref) { - result.ownBytes = blob.addRef(); - } - } - return result; -} - -Body::ExtractedBody::ExtractedBody( - jsg::Ref stream, kj::Maybe buffer, kj::Maybe contentType) +Body::ExtractedBody::ExtractedBody(jsg::Ref stream, + kj::Maybe> buffer, + kj::Maybe contentType) : impl{kj::mv(stream), kj::mv(buffer)}, contentType(kj::mv(contentType)) { // This check is in the constructor rather than `extractBody()`, because we often construct @@ -103,7 +87,7 @@ Body::ExtractedBody::ExtractedBody( } Body::ExtractedBody Body::extractBody(jsg::Lock& js, Initializer init) { - Buffer buffer; + kj::Maybe> buffer; kj::Maybe contentType; KJ_SWITCH_ONEOF(init) { @@ -115,7 +99,7 @@ Body::ExtractedBody Body::extractBody(jsg::Lock& js, Initializer init) { } KJ_CASE_ONEOF(text, kj::String) { contentType = kj::str(MimeType::PLAINTEXT_STRING); - buffer = kj::mv(text); + buffer = kj::rc(kj::mv(text)); } KJ_CASE_ONEOF(bytesRef, jsg::JsRef) { // Per the Fetch spec we must copy the input buffer here. Beyond spec conformance, this @@ -123,7 +107,7 @@ Body::ExtractedBody Body::extractBody(jsg::Lock& js, Initializer init) { // be freed if the original ArrayBuffer is detached and transferred (e.g. via structuredClone // with a transfer list) and then garbage collected. This applies to both resizable and // fixed-size buffers. Copying severs the dependency on the V8 backing store. - buffer = kj::heapArray(bytesRef.getHandle(js).asArrayPtr()); + buffer = kj::rc(kj::heapArray(bytesRef.getHandle(js).asArrayPtr())); } KJ_CASE_ONEOF(blob, jsg::Ref) { // Blobs always have a type, but it defaults to an empty string. We should NOT set @@ -132,7 +116,7 @@ Body::ExtractedBody Body::extractBody(jsg::Lock& js, Initializer init) { if (blobType != nullptr) { contentType = kj::str(blobType); } - buffer = kj::mv(blob); + buffer = kj::rc(kj::mv(blob)); } KJ_CASE_ONEOF(formData, jsg::Ref) { // Make an array of characters containing random hexadecimal digits. @@ -145,29 +129,31 @@ Body::ExtractedBody Body::extractBody(jsg::Lock& js, Initializer init) { workerd::getEntropy(boundaryBuffer); auto boundary = kj::encodeHex(boundaryBuffer); contentType = MimeType::formDataWithBoundary(boundary); - buffer = formData->serialize(boundary); + buffer = kj::rc(formData->serialize(boundary)); } KJ_CASE_ONEOF(searchParams, jsg::Ref) { contentType = MimeType::formUrlEncodedWithCharset("UTF-8"_kj); - buffer = searchParams->toString(); + buffer = kj::rc(searchParams->toString()); } KJ_CASE_ONEOF(searchParams, jsg::Ref) { contentType = MimeType::formUrlEncodedWithCharset("UTF-8"_kj); - buffer = searchParams->toString(); + buffer = kj::rc(searchParams->toString()); } } - auto buf = buffer.clone(js); - // We use streams::newMemorySource() here rather than newSystemStream() wrapping a // newMemoryInputStream() because we do NOT want deferred proxying for bodies with // V8 heap provenance. Some buffer types (e.g. Blob data) may reference V8 heap memory // and we must ensure the data is consumed and destroyed while under the isolate lock, // which means deferred proxying is not allowed. - auto rs = streams::newMemorySource(buf.view, kj::heap(kj::mv(buf.ownBytes))); + auto& buf = KJ_ASSERT_NONNULL(buffer); + auto rs = streams::newMemorySource(buf->view, buf.addRef().toOwn()); - return {js.alloc(IoContext::current(), kj::mv(rs)), kj::mv(buffer), - kj::mv(contentType)}; + return { + js.alloc(IoContext::current(), kj::mv(rs)), + kj::mv(buffer), + kj::mv(contentType), + }; } Body::Body(jsg::Lock& js, kj::Maybe init, Headers& headers) @@ -193,10 +179,10 @@ Body::Body(jsg::Lock& js, kj::Maybe init, Headers& headers) })), headersRef(headers) {} -kj::Maybe Body::getBodyBuffer(jsg::Lock& js) { +kj::Maybe> Body::getBodyBuffer() { KJ_IF_SOME(i, impl) { KJ_IF_SOME(b, i.buffer) { - return b.clone(js); + return b.addRef(); } } return kj::none; @@ -215,14 +201,15 @@ void Body::rewindBody(jsg::Lock& js) { KJ_DASSERT(canRewindBody()); KJ_IF_SOME(i, impl) { - auto bufferCopy = KJ_ASSERT_NONNULL(i.buffer).clone(js); + auto bufferCopy = KJ_ASSERT_NONNULL(i.buffer).addRef(); // We use streams::newMemorySource() here rather than newSystemStream() wrapping a // newMemoryInputStream() because we do NOT want deferred proxying for bodies with // V8 heap provenance. Specifically, the bufferCopy.view here, while being a kj::ArrayPtr, // will typically be wrapping a v8::BackingStore, and we must ensure that is is consumed // and destroyed while under the isolate lock, which means deferred proxying is not allowed. - auto rs = streams::newMemorySource(bufferCopy.view, kj::heap(kj::mv(bufferCopy.ownBytes))); + auto view = bufferCopy->view; + auto rs = streams::newMemorySource(view, bufferCopy.toOwn()); i.stream = js.alloc(IoContext::current(), kj::mv(rs)); } } @@ -360,7 +347,8 @@ kj::Maybe Body::clone(jsg::Lock& js) { i.stream = kj::mv(branches[0]); - return ExtractedBody{kj::mv(branches[1]), i.buffer.map([&](Buffer& b) { return b.clone(js); })}; + return ExtractedBody{ + kj::mv(branches[1]), i.buffer.map([&](kj::Rc& b) { return b.addRef(); })}; } return kj::none; @@ -456,7 +444,7 @@ jsg::Ref Request::constructor( // body below. We don't need to do that. Instead, we just create a new ReadableStream // that takes over ownership of the internals of the given stream. The given stream // is left in a locked/disturbed mode so that it can no longer be used. - body = Body::ExtractedBody((oldJsBody)->detach(js), oldRequest->getBodyBuffer(js)); + body = Body::ExtractedBody((oldJsBody)->detach(js), oldRequest->getBodyBuffer()); } } cacheMode = oldRequest->getCacheMode(); @@ -1146,7 +1134,7 @@ jsg::Ref Response::constructor(jsg::Lock& js, "Response with null body status (101, 204, 205, or 304) cannot have a body."); // Fail if the body is backed by a non-zero-length buffer. - JSG_REQUIRE(buffer.view.size() == 0, TypeError, + JSG_REQUIRE(buffer->view.size() == 0, TypeError, "Response with null body status (101, 204, 205, or 304) cannot have a body."); auto& context = IoContext::current(); diff --git a/src/workerd/api/http.h b/src/workerd/api/http.h index 8d0cd54b960..27b71903cba 100644 --- a/src/workerd/api/http.h +++ b/src/workerd/api/http.h @@ -52,14 +52,6 @@ class Body: public jsg::Object { jsg::Ref, jsg::AsyncGeneratorIgnoringStrings>; - struct RefcountedBytes final: public kj::Refcounted { - kj::Array bytes; - RefcountedBytes(kj::Array&& bytes): bytes(kj::mv(bytes)) {} - JSG_MEMORY_INFO(RefcountedBytes) { - tracker.trackFieldWithSize("bytes", bytes.size()); - } - }; - // The Fetch spec calls this type the body's "source", even though it really is a buffer. I end // talking about things like "a buffer-backed body", whereas in standardese I should say // "a body with a non-null source". @@ -69,11 +61,11 @@ class Body: public jsg::Object { // In order to reconstruct buffer-backed ReadableStreams without gratuitous array copying, we // need to be able to tie the lifetime of the source buffer to the lifetime of the // ReadableStream's native stream, AND the lifetime of the Body itself. Thus we need - // refcounting. + // refcounting. The Buffer instance itself will be refcounted. // // NOTE: ownBytes may contain a v8::Global reference, hence instances of `Buffer` must exist // only within the V8 heap space. - kj::OneOf, jsg::Ref, jsg::JsRef> ownBytes; + kj::OneOf, jsg::Ref, jsg::JsRef> ownBytes; // TODO(cleanup): When we integrate with V8's garbage collection APIs, we need to account for // that here. @@ -92,26 +84,24 @@ class Body: public jsg::Object { KJ_ASSERT(!source.isResizable()); } Buffer(kj::Array array) - : ownBytes(kj::refcounted(kj::mv(array))), - view(ownBytes.get>()->bytes) {} + : ownBytes(kj::mv(array)), + view(ownBytes.get>()) {} Buffer(kj::String string) - : ownBytes(kj::refcounted(string.releaseArray().releaseAsBytes())), + : ownBytes(string.releaseArray().releaseAsBytes()), view([this] { - auto bytesIncludingNull = ownBytes.get>()->bytes.asPtr(); + auto bytesIncludingNull = ownBytes.get>().asPtr(); return bytesIncludingNull.first(bytesIncludingNull.size() - 1); }()) {} Buffer(jsg::Ref blob) : ownBytes(kj::mv(blob)), view(ownBytes.get>()->getData()) {} - Buffer clone(jsg::Lock& js); - JSG_MEMORY_INFO(Buffer) { KJ_SWITCH_ONEOF(ownBytes) { KJ_CASE_ONEOF(ref, jsg::JsRef) { tracker.trackField("ref", ref); } - KJ_CASE_ONEOF(bytes, kj::Own) { + KJ_CASE_ONEOF(bytes, kj::Array) { tracker.trackField("bytes", bytes); } KJ_CASE_ONEOF(blob, jsg::Ref) { @@ -123,16 +113,18 @@ class Body: public jsg::Object { struct Impl { jsg::Ref stream; - kj::Maybe buffer; + kj::Maybe> buffer; JSG_MEMORY_INFO(Impl) { tracker.trackField("stream", stream); - tracker.trackField("buffer", buffer); + KJ_IF_SOME(buf, buffer) { + tracker.trackFieldWithSize("buffer", buf->view.size()); + } } }; struct ExtractedBody { ExtractedBody(jsg::Ref stream, - kj::Maybe source = kj::none, + kj::Maybe> source = kj::none, kj::Maybe contentType = kj::none); Impl impl; @@ -145,7 +137,7 @@ class Body: public jsg::Object { explicit Body(jsg::Lock& js, kj::Maybe init, Headers& headers); - kj::Maybe getBodyBuffer(jsg::Lock& js); + kj::Maybe> getBodyBuffer(); // The following body rewind/nullification functions are helpers for implementing fetch() redirect // handling. From 289bd135276c78909a6a1911bf1146542261cfe0 Mon Sep 17 00:00:00 2001 From: James M Snell Date: Wed, 1 Jul 2026 10:52:56 -0700 Subject: [PATCH 036/101] Add JsReadableStream abstraction --- src/workerd/api/AGENTS.md | 1 + src/workerd/api/BUILD.bazel | 14 + src/workerd/api/api-rtti-test.c++ | 10 + src/workerd/api/blob.c++ | 6 +- src/workerd/api/blob.h | 4 +- src/workerd/api/container.c++ | 36 +- src/workerd/api/container.h | 24 +- src/workerd/api/eventsource.c++ | 17 +- src/workerd/api/eventsource.h | 4 +- src/workerd/api/global-scope.c++ | 8 +- src/workerd/api/html-rewriter.c++ | 78 ++-- src/workerd/api/html-rewriter.h | 28 +- src/workerd/api/http.c++ | 314 ++++++------- src/workerd/api/http.h | 123 ++--- src/workerd/api/js-readable-stream-test.c++ | 478 ++++++++++++++++++++ src/workerd/api/js-readable-stream.c++ | 472 +++++++++++++++++++ src/workerd/api/js-readable-stream.h | 252 +++++++++++ src/workerd/api/kv.c++ | 10 +- src/workerd/api/kv.h | 8 +- src/workerd/api/r2-bucket.c++ | 43 +- src/workerd/api/r2-bucket.h | 14 +- src/workerd/api/r2-multipart.c++ | 14 +- src/workerd/api/r2-rpc.c++ | 104 +++-- src/workerd/api/r2-rpc.h | 13 +- src/workerd/api/sockets.c++ | 14 +- src/workerd/api/sockets.h | 12 +- src/workerd/api/streams.h | 1 + src/workerd/api/streams/AGENTS.md | 5 + src/workerd/jsg/README.md | 2 +- src/workerd/jsg/rtti.h | 20 + src/workerd/jsg/type-wrapper.h | 5 + src/workerd/tests/bench-response.c++ | 2 +- 32 files changed, 1655 insertions(+), 481 deletions(-) create mode 100644 src/workerd/api/js-readable-stream-test.c++ create mode 100644 src/workerd/api/js-readable-stream.c++ create mode 100644 src/workerd/api/js-readable-stream.h diff --git a/src/workerd/api/AGENTS.md b/src/workerd/api/AGENTS.md index a304c1aca82..5b515ea085a 100644 --- a/src/workerd/api/AGENTS.md +++ b/src/workerd/api/AGENTS.md @@ -35,6 +35,7 @@ tests/ # JS integration tests (238 entries); each test = .js + .wd | KV / SyncKV | `kv.h`, `sync-kv.h` | | SQL (DO) | `sql.h` | | Body mixin (shared Req/Res) | `http.h` — `Body` class, `Body::Initializer` OneOf | +| ReadableStream from C++ code | `js-readable-stream.{h,c++}` — `JsReadableStream` abstraction; ALWAYS use it (not `jsg::Ref`) outside `streams/` | ## CONVENTIONS diff --git a/src/workerd/api/BUILD.bazel b/src/workerd/api/BUILD.bazel index e548f11c99d..fad22e7493a 100644 --- a/src/workerd/api/BUILD.bazel +++ b/src/workerd/api/BUILD.bazel @@ -25,6 +25,7 @@ filegroup( "hibernatable-adapter.c++", "hibernatable-web-socket.c++", "http.c++", + "js-readable-stream.c++", "messagechannel.c++", "performance.c++", "queue.c++", @@ -75,6 +76,7 @@ filegroup( "hibernatable-adapter.h", "hibernatable-web-socket.h", "http.h", + "js-readable-stream.h", "messagechannel.h", "performance.h", "queue.h", @@ -235,6 +237,10 @@ wd_cc_library( # for using EW_STREAMS_ISOLATE_TYPES with workerd-api etc. To use the different streams APIs, # include the headers you need directly. +# +# Note: JsReadableStream lives in js-readable-stream.{h,c++}, which is folded into +# //src/workerd/io (via the :srcs/:hdrs filegroups) because http.h depends on it. This target +# re-exports it via streams.h. wd_cc_library( name = "streams", hdrs = ["streams.h"], @@ -603,6 +609,14 @@ kj_test( ], ) +kj_test( + src = "js-readable-stream-test.c++", + deps = [ + "//src/workerd/io", + "//src/workerd/tests:test-fixture", + ], +) + kj_test( src = "system-streams-test.c++", deps = [ diff --git a/src/workerd/api/api-rtti-test.c++ b/src/workerd/api/api-rtti-test.c++ index ddd3ebb166c..47e49ccc635 100644 --- a/src/workerd/api/api-rtti-test.c++ +++ b/src/workerd/api/api-rtti-test.c++ @@ -45,5 +45,15 @@ KJ_TEST("ServiceWorkerGlobalScope") { KJ_EXPECT(builder.structure("workerd::api::DurableObjectId"_kj) != kj::none); } +KJ_TEST("JsReadableStream delegates its RTTI to ReadableStream") { + // JsReadableStream declares `using JsgRttiDelegate = jsg::Ref`, so RTTI (and + // therefore generated TypeScript) must describe it exactly as it describes ReadableStream. + jsg::rtti::Builder builder((CompatibilityFlags::Reader())); + auto type = builder.type(); + KJ_ASSERT(type.isStructure()); + KJ_EXPECT(type.getStructure().getFullyQualifiedName() == "workerd::api::ReadableStream"_kj); + KJ_EXPECT(builder.structure("workerd::api::ReadableStream"_kj) != kj::none); +} + } // namespace } // namespace workerd::api diff --git a/src/workerd/api/blob.c++ b/src/workerd/api/blob.c++ index baf5959df75..9e37bab59fc 100644 --- a/src/workerd/api/blob.c++ +++ b/src/workerd/api/blob.c++ @@ -275,10 +275,10 @@ jsg::Promise> Blob::text(jsg::Lock& js) { return js.resolvedPromise(js.str(data.asChars()).addRef(js)); } -jsg::Ref Blob::stream(jsg::Lock& js) { +JsReadableStream Blob::stream(jsg::Lock& js) { FeatureObserver::maybeRecordUse(FeatureObserver::Feature::BLOB_AS_STREAM); - return js.alloc( - IoContext::current(), streams::newMemorySource(data, kj::heap(JSG_THIS))); + return JsReadableStream::create( + js, IoContext::current(), streams::newMemorySource(data, kj::heap(JSG_THIS))); } // ======================================================================================= diff --git a/src/workerd/api/blob.h b/src/workerd/api/blob.h index 02bdf1a4ae6..7b6efac9a5c 100644 --- a/src/workerd/api/blob.h +++ b/src/workerd/api/blob.h @@ -4,6 +4,7 @@ #pragma once +#include #include #include #include @@ -11,7 +12,6 @@ namespace workerd::api { -class ReadableStream; class File; // An implementation of the Web Platform Standard Blob API @@ -57,7 +57,7 @@ class Blob: public jsg::Object { jsg::Promise> arrayBuffer(jsg::Lock& js); jsg::Promise> bytes(jsg::Lock& js); jsg::Promise> text(jsg::Lock& js); - jsg::Ref stream(jsg::Lock& js); + JsReadableStream stream(jsg::Lock& js); JSG_RESOURCE_TYPE(Blob, CompatibilityFlags::Reader flags) { if (flags.getJsgPropertyOnPrototypeTemplate()) { diff --git a/src/workerd/api/container.c++ b/src/workerd/api/container.c++ index 31109d74b1c..9000910ba83 100644 --- a/src/workerd/api/container.c++ +++ b/src/workerd/api/container.c++ @@ -81,8 +81,8 @@ jsg::JsArrayBuffer ExecOutput::getStderr(jsg::Lock& js) { ExecProcess::ExecProcess(jsg::Lock& js, IoContext& ioContext, jsg::Optional> stdinStream, - jsg::Optional> stdoutStream, - jsg::Optional> stderrStream, + jsg::Optional stdoutStream, + jsg::Optional stderrStream, int pid, rpc::Container::ProcessHandle::Client handle, kj::Maybe> abortSignal) @@ -121,12 +121,12 @@ jsg::Optional> ExecProcess::getStdin() { return stdinStream.map([](jsg::Ref& stream) { return stream.addRef(); }); } -jsg::Optional> ExecProcess::getStdout() { - return stdoutStream.map([](jsg::Ref& stream) { return stream.addRef(); }); +jsg::Optional ExecProcess::getStdout(jsg::Lock& js) { + return stdoutStream.map([&](JsReadableStream& stream) { return stream.addRef(js); }); } -jsg::Optional> ExecProcess::getStderr() { - return stderrStream.map([](jsg::Ref& stream) { return stream.addRef(); }); +jsg::Optional ExecProcess::getStderr(jsg::Lock& js) { + return stderrStream.map([&](JsReadableStream& stream) { return stream.addRef(js); }); } void ExecProcess::ensureExitCodePromise(jsg::Lock& js) { @@ -177,11 +177,10 @@ jsg::Promise> ExecProcess::output(jsg::Lock& js) { auto stdoutPromise = js.resolvedPromise(emptyByteArray()); KJ_IF_SOME(stream, stdoutStream) { - JSG_REQUIRE(!stream->isDisturbed(), TypeError, + JSG_REQUIRE(!stream.isDisturbed(js), TypeError, "Cannot call output() after stdout has started being consumed."); stdoutPromise = - stream->getController() - .readAllBytes(js, IoContext::current().getLimitEnforcer().getBufferingLimit()) + stream.arrayBuffer(js, IoContext::current().getLimitEnforcer().getBufferingLimit()) .then(js, [](jsg::Lock& js, jsg::JsRef bytes) { return bytes.getHandle(js).copy(); }); @@ -189,10 +188,9 @@ jsg::Promise> ExecProcess::output(jsg::Lock& js) { auto stderrPromise = js.resolvedPromise(emptyByteArray()); KJ_IF_SOME(stream, stderrStream) { - JSG_REQUIRE(!stream->isDisturbed(), TypeError, + JSG_REQUIRE(!stream.isDisturbed(js), TypeError, "Cannot call output() after stderr has started being consumed."); - stderrPromise = stream->getController() - .readAllBytes(js, kj::maxValue) + stderrPromise = stream.arrayBuffer(js, kj::maxValue) .then(js, [](jsg::Lock& js, jsg::JsRef bytes) { return bytes.getHandle(js).copy(); }); @@ -560,17 +558,17 @@ jsg::Promise> Container::exec( auto pid = process.getPid(); // Init the ReadableStreams (stdout/stderr) - jsg::Optional> stdoutStream = kj::none; + jsg::Optional stdoutStream = kj::none; KJ_IF_SOME(input, stdoutInput) { auto source = newSystemStream(kj::mv(input), StreamEncoding::IDENTITY, ioContext); - stdoutStream = js.alloc(ioContext, kj::mv(source)); + stdoutStream = JsReadableStream::create(js, ioContext, kj::mv(source)); } // stderrInput is only set if using "pipe" on stderr and not "combined" - jsg::Optional> stderrStream = kj::none; + jsg::Optional stderrStream = kj::none; KJ_IF_SOME(input, stderrInput) { auto source = newSystemStream(kj::mv(input), StreamEncoding::IDENTITY, ioContext); - stderrStream = js.alloc(ioContext, kj::mv(source)); + stderrStream = JsReadableStream::create(js, ioContext, kj::mv(source)); } jsg::Optional> stdinStream = kj::none; @@ -586,11 +584,11 @@ jsg::Promise> Container::exec( KJ_SWITCH_ONEOF(stdinOption) { // user sets ReadableStream... - KJ_CASE_ONEOF(readable, jsg::Ref) { + KJ_CASE_ONEOF(readable, JsReadableStream) { auto sink = newSystemStream(kj::mv(stdinWriter), StreamEncoding::IDENTITY, ioContext); auto pipePromise = - (ioContext.waitForDeferredProxy(readable->pumpTo(js, kj::mv(sink), true))); - ioContext.addTask(pipePromise.attach(readable.addRef())); + (ioContext.waitForDeferredProxy(readable.pumpTo(js, kj::mv(sink), EndStream::YES))); + ioContext.addTask(pipePromise.attach(readable.addRef(js))); } // user sets "pipe"... they want to consume the API with the stdin WritableStream KJ_CASE_ONEOF(mode, kj::String) { diff --git a/src/workerd/api/container.h b/src/workerd/api/container.h index 6c64953910d..54b77743ec2 100644 --- a/src/workerd/api/container.h +++ b/src/workerd/api/container.h @@ -6,7 +6,7 @@ // Container management API for Durable Object-attached containers. // #include -#include +#include #include #include #include @@ -53,7 +53,7 @@ class ExecOutput: public jsg::Object { struct ExecOptions { // $ prefix avoids collision with stdin/stdout/stderr macros from ; // JSG_STRUCT strips the $ when exposing to JS. - jsg::Optional, kj::String>> $stdin; + jsg::Optional> $stdin; jsg::Optional $stdout; jsg::Optional $stderr; jsg::Optional cwd; @@ -81,15 +81,15 @@ class ExecProcess: public jsg::Object { ExecProcess(jsg::Lock& js, IoContext& ioContext, jsg::Optional> stdinStream, - jsg::Optional> stdoutStream, - jsg::Optional> stderrStream, + jsg::Optional stdoutStream, + jsg::Optional stderrStream, int pid, rpc::Container::ProcessHandle::Client handle, kj::Maybe> abortSignal = kj::none); jsg::Optional> getStdin(); - jsg::Optional> getStdout(); - jsg::Optional> getStderr(); + jsg::Optional getStdout(jsg::Lock& js); + jsg::Optional getStderr(jsg::Lock& js); int getPid() const { return pid; } @@ -120,8 +120,12 @@ class ExecProcess: public jsg::Object { void visitForMemoryInfo(jsg::MemoryTracker& tracker) const { tracker.trackField("stdin", stdinStream); - tracker.trackField("stdout", stdoutStream); - tracker.trackField("stderr", stderrStream); + KJ_IF_SOME(s, stdoutStream) { + s.visitForMemoryInfo(tracker); + } + KJ_IF_SOME(s, stderrStream) { + s.visitForMemoryInfo(tracker); + } tracker.trackField("exitCodePromise", exitCodePromise); tracker.trackField("exitCodePromiseCopy", exitCodePromiseCopy); } @@ -135,8 +139,8 @@ class ExecProcess: public jsg::Object { void sendKill(int signo); jsg::Optional> stdinStream; - jsg::Optional> stdoutStream; - jsg::Optional> stderrStream; + jsg::Optional stdoutStream; + jsg::Optional stderrStream; int pid; IoOwn handle; kj::Maybe>> exitCodePromise; diff --git a/src/workerd/api/eventsource.c++ b/src/workerd/api/eventsource.c++ index 3b71a9fb460..21e3e35687c 100644 --- a/src/workerd/api/eventsource.c++ +++ b/src/workerd/api/eventsource.c++ @@ -251,12 +251,12 @@ jsg::Ref EventSource::constructor( return kj::mv(eventsource); } -jsg::Ref EventSource::from(jsg::Lock& js, jsg::Ref readable) { +jsg::Ref EventSource::from(jsg::Lock& js, JsReadableStream readable) { JSG_REQUIRE(IoContext::hasCurrent(), DOMNotSupportedError, "An EventSource can only be created within the context of a worker request."); - JSG_REQUIRE(!readable->isLocked(), TypeError, "This ReadableStream is locked."); + JSG_REQUIRE(!readable.isLocked(js), TypeError, "This ReadableStream is locked."); JSG_REQUIRE( - !readable->isDisturbed(), TypeError, "This ReadableStream has already been read from."); + !readable.isDisturbed(js), TypeError, "This ReadableStream has already been read from."); auto eventsource = js.alloc(js); eventsource->run(js, kj::mv(readable), false /* No reconnection attempts */); return kj::mv(eventsource); @@ -388,7 +388,7 @@ void EventSource::start(jsg::Lock& js) { } // Extra else block to squash compiler warning } - KJ_IF_SOME(body, response->getBody()) { + KJ_IF_SOME(body, response->getBody(js)) { // Well, ok! We're ready to start trying to process the stream! We do so by // pumping the body into an EventSourceSink until the body is closed, canceled, // or errored. @@ -441,7 +441,7 @@ kj::Maybe> addRef(kj::Maybe>& ref) { } // namespace void EventSource::run(jsg::Lock& js, - jsg::Ref readable, + JsReadableStream readable, bool withReconnection, kj::Maybe> response, kj::Maybe> fetcher) { @@ -453,7 +453,7 @@ void EventSource::run(jsg::Lock& js, } auto onSuccess = - JSG_VISITABLE_LAMBDA((self = JSG_THIS, readable = readable.addRef(), withReconnection, + JSG_VISITABLE_LAMBDA((self = JSG_THIS, readable = readable.addRef(js), withReconnection, response = addRef(response), fetcher = addRef(fetcher)), (self, readable, response, fetcher), (jsg::Lock& js) { // The pump finished. Did the server disconnect? If so, try reconnecting if we can. @@ -474,8 +474,9 @@ void EventSource::run(jsg::Lock& js, // pumping the body into an EventSourceSink until the body is closed, canceled, // or errored. context - .awaitIo( - js, processBody(context, readable->pumpTo(js, kj::heap(*this), true))) + .awaitIo(js, + processBody( + context, readable.pumpTo(js, kj::heap(*this), EndStream::YES))) .then(js, kj::mv(onSuccess), kj::mv(onFailed)); } diff --git a/src/workerd/api/eventsource.h b/src/workerd/api/eventsource.h index bd0ac493478..70158abf21b 100644 --- a/src/workerd/api/eventsource.h +++ b/src/workerd/api/eventsource.h @@ -65,7 +65,7 @@ class EventSource: public EventTarget { // or underlying fetch used. The ReadableStream instance must produce bytes. It will be // locked and disturbed, and will be read until it either ends or errors. Calling close() // will cause the stream to be canceled. - static jsg::Ref from(jsg::Lock& js, jsg::Ref stream); + static jsg::Ref from(jsg::Lock& js, JsReadableStream stream); kj::Maybe getOnOpen(jsg::Lock& js) { return onopenValue.map( @@ -190,7 +190,7 @@ class EventSource: public EventTarget { // The run() method handles the actual processing of the stream. void run(jsg::Lock& js, - jsg::Ref stream, + JsReadableStream stream, bool withReconnection = true, kj::Maybe> response = kj::none, kj::Maybe> fetcher = kj::none); diff --git a/src/workerd/api/global-scope.c++ b/src/workerd/api/global-scope.c++ index ede0606fca0..58dca5c247a 100644 --- a/src/workerd/api/global-scope.c++ +++ b/src/workerd/api/global-scope.c++ @@ -261,7 +261,7 @@ kj::Promise> ServiceWorkerGlobalScope::request(kj::HttpMetho CfProperty cf(cfBlobJson); // We only create the body stream if there is a body to read. - kj::Maybe> maybeJsStream = kj::none; + kj::Maybe maybeJsStream = kj::none; // If the request has "no body", we want `request.body` to be null. But, this is not the same // thing as the request having a body that happens to be empty. Unfortunately, KJ HTTP gives us @@ -290,8 +290,8 @@ kj::Promise> ServiceWorkerGlobalScope::request(kj::HttpMetho // We do not automatically decode gzipped request bodies because the fetch() standard doesn't // specify any automatic encoding of requests. https://github.com/whatwg/fetch/issues/589 auto b = newSystemStream(kj::addRef(*ownRequestBody), StreamEncoding::IDENTITY); - auto jsStream = js.alloc(ioContext, kj::mv(b)); - body = Body::ExtractedBody(jsStream.addRef()); + auto jsStream = JsReadableStream::create(js, ioContext, kj::mv(b)); + body = Body::ExtractedBody(js, jsStream.addRef(js)); maybeJsStream = kj::mv(jsStream); } @@ -379,7 +379,7 @@ kj::Promise> ServiceWorkerGlobalScope::request(kj::HttpMetho } KJ_IF_SOME(jsStream, maybeJsStream) { - if (jsStream->isDisturbed()) { + if (jsStream.isDisturbed(js)) { lock.logUncaughtException( "Script consumed request body but didn't call respondWith(). Can't forward request."); return addNoopDeferredProxy( diff --git a/src/workerd/api/html-rewriter.c++ b/src/workerd/api/html-rewriter.c++ index 0d2556b391a..bd8c7c8c2b0 100644 --- a/src/workerd/api/html-rewriter.c++ +++ b/src/workerd/api/html-rewriter.c++ @@ -227,7 +227,7 @@ class Rewriter final: public WritableStreamSink { // Implementation for `Element::onEndTag` to avoid exposing private details of Rewriter. void onEndTag(lol_html_element_t* element, ElementCallbackFunction&& callback); - lol_html_streaming_handler_t registerReplacer(jsg::Ref content, bool isHtml); + lol_html_streaming_handler_t registerReplacer(JsReadableStream content, bool isHtml); ~Rewriter() { KJ_ASSERT(registeredReplacers.size() == 0, "Some replacers were leaked by lol-html"); @@ -272,7 +272,7 @@ class Rewriter final: public WritableStreamSink { public: Rewriter& rewriter; bool isHtml; - jsg::Ref stream; + JsReadableStream stream; }; struct RegisteredHandler { @@ -621,8 +621,7 @@ kj::Promise Rewriter::thunkPromise(CType* content, RegisteredHandler& regi }); } -lol_html_streaming_handler_t Rewriter::registerReplacer( - jsg::Ref content, bool isHtml) { +lol_html_streaming_handler_t Rewriter::registerReplacer(JsReadableStream content, bool isHtml) { auto replacer = kj::heap(*this, isHtml, kj::mv(content)); auto userData = replacer.get(); registeredReplacers.insert(userData, kj::mv(replacer)); @@ -727,7 +726,7 @@ kj::Promise Rewriter::replacerThunkPromise( auto streamSink = kj::heap(sink, registration.isHtml); return ioContext.waitForDeferredProxy( - registration.stream->pumpTo(lock, kj::mv(streamSink), true)); + registration.stream.pumpTo(lock, kj::mv(streamSink), EndStream::YES)); }); } @@ -782,7 +781,7 @@ HTMLRewriter::Token::ImplBase::~ImplBase() noexcept(false) {} template template void HTMLRewriter::Token::ImplBase::rewriteContentGeneric( - Content content, jsg::Optional options) { + jsg::Lock& js, Content content, jsg::Optional options) { auto isHtml = options.orDefault({}).html.orDefault(false); KJ_SWITCH_ONEOF(content) { @@ -790,13 +789,13 @@ void HTMLRewriter::Token::ImplBase::rewriteContentGeneric( check(Func(&element, stringContent.cStr(), stringContent.size(), isHtml)); } - KJ_CASE_ONEOF(streamContent, jsg::Ref) { + KJ_CASE_ONEOF(streamContent, JsReadableStream) { auto handler = rewriter.registerReplacer(kj::mv(streamContent), isHtml); check(StreamingFunc(&element, &handler)); } KJ_CASE_ONEOF(responseContent, jsg::Ref) { - KJ_IF_SOME(body, responseContent->getBody()) { + KJ_IF_SOME(body, responseContent->getBody(js)) { auto handler = rewriter.registerReplacer(kj::mv(body), isHtml); check(StreamingFunc(&element, &handler)); } @@ -898,44 +897,50 @@ kj::String unwrapContent(Content content) { } } // namespace -jsg::Ref Element::before(Content content, jsg::Optional options) { +jsg::Ref Element::before( + jsg::Lock& js, Content content, jsg::Optional options) { checkToken(impl) .rewriteContentGeneric( - kj::mv(content), options); + js, kj::mv(content), options); return JSG_THIS; } -jsg::Ref Element::after(Content content, jsg::Optional options) { +jsg::Ref Element::after( + jsg::Lock& js, Content content, jsg::Optional options) { checkToken(impl).rewriteContentGeneric( - kj::mv(content), options); + js, kj::mv(content), options); return JSG_THIS; } -jsg::Ref Element::prepend(Content content, jsg::Optional options) { +jsg::Ref Element::prepend( + jsg::Lock& js, Content content, jsg::Optional options) { checkToken(impl) .rewriteContentGeneric( - kj::mv(content), options); + js, kj::mv(content), options); return JSG_THIS; } -jsg::Ref Element::append(Content content, jsg::Optional options) { +jsg::Ref Element::append( + jsg::Lock& js, Content content, jsg::Optional options) { checkToken(impl) .rewriteContentGeneric( - kj::mv(content), options); + js, kj::mv(content), options); return JSG_THIS; } -jsg::Ref Element::replace(Content content, jsg::Optional options) { +jsg::Ref Element::replace( + jsg::Lock& js, Content content, jsg::Optional options) { checkToken(impl) .rewriteContentGeneric( - kj::mv(content), options); + js, kj::mv(content), options); return JSG_THIS; } -jsg::Ref Element::setInnerContent(Content content, jsg::Optional options) { +jsg::Ref Element::setInnerContent( + jsg::Lock& js, Content content, jsg::Optional options) { checkToken(impl) .rewriteContentGeneric(kj::mv(content), options); + lol_html_element_streaming_set_inner_content>(js, kj::mv(content), options); return JSG_THIS; } @@ -971,16 +976,18 @@ void EndTag::setName(kj::String text) { check(lol_html_end_tag_name_set(&checkToken(impl).element, text.cStr(), text.size())); } -jsg::Ref EndTag::before(Content content, jsg::Optional options) { +jsg::Ref EndTag::before( + jsg::Lock& js, Content content, jsg::Optional options) { checkToken(impl) .rewriteContentGeneric( - kj::mv(content), kj::mv(options)); + js, kj::mv(content), kj::mv(options)); return JSG_THIS; } -jsg::Ref EndTag::after(Content content, jsg::Optional options) { +jsg::Ref EndTag::after( + jsg::Lock& js, Content content, jsg::Optional options) { checkToken(impl).rewriteContentGeneric( - kj::mv(content), kj::mv(options)); + js, kj::mv(content), kj::mv(options)); return JSG_THIS; } @@ -1125,24 +1132,25 @@ bool Text::getRemoved() { return lol_html_text_chunk_is_removed(&checkToken(impl).element); } -jsg::Ref Text::before(Content content, jsg::Optional options) { +jsg::Ref Text::before(jsg::Lock& js, Content content, jsg::Optional options) { checkToken(impl) .rewriteContentGeneric( - kj::mv(content), kj::mv(options)); + js, kj::mv(content), kj::mv(options)); return JSG_THIS; } -jsg::Ref Text::after(Content content, jsg::Optional options) { +jsg::Ref Text::after(jsg::Lock& js, Content content, jsg::Optional options) { checkToken(impl) .rewriteContentGeneric( - kj::mv(content), kj::mv(options)); + js, kj::mv(content), kj::mv(options)); return JSG_THIS; } -jsg::Ref Text::replace(Content content, jsg::Optional options) { +jsg::Ref Text::replace( + jsg::Lock& js, Content content, jsg::Optional options) { checkToken(impl) .rewriteContentGeneric( - kj::mv(content), kj::mv(options)); + js, kj::mv(content), kj::mv(options)); return JSG_THIS; } @@ -1262,7 +1270,7 @@ jsg::Ref HTMLRewriter::transform(jsg::Lock& js, jsg::Ref res JSG_REQUIRE(response->getType() != "error"_kj, TypeError, "HTMLRewriter cannot transform an error response"); - auto maybeInput = response->getBody(); + auto maybeInput = response->getBody(js); if (maybeInput == kj::none) { // That was easy! @@ -1272,8 +1280,9 @@ jsg::Ref HTMLRewriter::transform(jsg::Lock& js, jsg::Ref res auto& ioContext = IoContext::current(); auto pipe = newIdentityPipe(); - response = Response::constructor( - js, kj::Maybe(js.alloc(ioContext, kj::mv(pipe.in))), kj::mv(response)); + response = Response::constructor(js, + kj::Maybe(Body::Initializer(JsReadableStream::create(js, ioContext, kj::mv(pipe.in)))), + kj::mv(response)); kj::String ownContentType; kj::String encoding = kj::str("utf-8"); @@ -1296,7 +1305,8 @@ jsg::Ref HTMLRewriter::transform(jsg::Lock& js, jsg::Ref res // Drive and flush the parser asynchronously. ioContext.addTask( ioContext - .waitForDeferredProxy(KJ_ASSERT_NONNULL(maybeInput)->pumpTo(js, kj::mv(rewriter), true)) + .waitForDeferredProxy( + KJ_ASSERT_NONNULL(maybeInput).pumpTo(js, kj::mv(rewriter), EndStream::YES)) .catch_([](kj::Exception&& e) { // Errors in pumpTo() are already propagated to the destination stream. We don't want to // throw them from here since it'll cause an uncaught exception to be reported via taskFailed(), diff --git a/src/workerd/api/html-rewriter.h b/src/workerd/api/html-rewriter.h index 1ef9e823895..730073106eb 100644 --- a/src/workerd/api/html-rewriter.h +++ b/src/workerd/api/html-rewriter.h @@ -119,7 +119,7 @@ class HTMLRewriter: public jsg::Object { }; // A chunk of text or HTML which can be passed to content token mutation functions. -using Content = kj::OneOf, jsg::Ref>; +using Content = kj::OneOf>; // TODO(soon): Support ReadableStream/Response types. Requires fibers or lol-html saveable state. // Options bag which can be passed to content token mutation functions. @@ -167,7 +167,8 @@ class HTMLRewriter::Token: public jsg::Object { // Dispatches calls to the underlying lol_html methods for each event (e.g. before, after, replace). // Handles replacements of each supported type (string, ReadableStream, Body). template - void rewriteContentGeneric(Content content, jsg::Optional options); + void rewriteContentGeneric( + jsg::Lock& js, Content content, jsg::Optional options); CType& element; Rewriter& rewriter; @@ -195,14 +196,15 @@ class Element final: public HTMLRewriter::Token { jsg::Ref setAttribute(kj::String name, kj::String value); jsg::Ref removeAttribute(kj::String name); - jsg::Ref before(Content content, jsg::Optional options); - jsg::Ref after(Content content, jsg::Optional options); + jsg::Ref before(jsg::Lock& js, Content content, jsg::Optional options); + jsg::Ref after(jsg::Lock& js, Content content, jsg::Optional options); - jsg::Ref prepend(Content content, jsg::Optional options); - jsg::Ref append(Content content, jsg::Optional options); + jsg::Ref prepend(jsg::Lock& js, Content content, jsg::Optional options); + jsg::Ref append(jsg::Lock& js, Content content, jsg::Optional options); - jsg::Ref replace(Content content, jsg::Optional options); - jsg::Ref setInnerContent(Content content, jsg::Optional options); + jsg::Ref replace(jsg::Lock& js, Content content, jsg::Optional options); + jsg::Ref setInnerContent( + jsg::Lock& js, Content content, jsg::Optional options); jsg::Ref remove(); jsg::Ref removeAndKeepContent(); @@ -293,8 +295,8 @@ class EndTag final: public HTMLRewriter::Token { kj::String getName(); void setName(kj::String); - jsg::Ref before(Content content, jsg::Optional options); - jsg::Ref after(Content content, jsg::Optional options); + jsg::Ref before(jsg::Lock& js, Content content, jsg::Optional options); + jsg::Ref after(jsg::Lock& js, Content content, jsg::Optional options); jsg::Ref remove(); JSG_RESOURCE_TYPE(EndTag) { @@ -365,9 +367,9 @@ class Text final: public HTMLRewriter::Token { bool getRemoved(); - jsg::Ref before(Content content, jsg::Optional options); - jsg::Ref after(Content content, jsg::Optional options); - jsg::Ref replace(Content content, jsg::Optional options); + jsg::Ref before(jsg::Lock& js, Content content, jsg::Optional options); + jsg::Ref after(jsg::Lock& js, Content content, jsg::Optional options); + jsg::Ref replace(jsg::Lock& js, Content content, jsg::Optional options); jsg::Ref remove(); JSG_RESOURCE_TYPE(Text) { diff --git a/src/workerd/api/http.c++ b/src/workerd/api/http.c++ index 9610ddb66bc..69d3cac247b 100644 --- a/src/workerd/api/http.c++ +++ b/src/workerd/api/http.c++ @@ -74,49 +74,45 @@ jsg::Optional getCacheModeName(Request::CacheMode mode) { // capitalization). So, it's certainly not worth it to try to keep the original capitalization // across serialization. -Body::ExtractedBody::ExtractedBody(jsg::Ref stream, - kj::Maybe> buffer, - kj::Maybe contentType) - : impl{kj::mv(stream), kj::mv(buffer)}, +Body::ExtractedBody::ExtractedBody( + jsg::Lock& js, JsReadableStream stream, kj::Maybe contentType) + : stream(kj::mv(stream)), contentType(kj::mv(contentType)) { // This check is in the constructor rather than `extractBody()`, because we often construct // ExtractedBodys from ReadableStreams directly. - JSG_REQUIRE(!impl.stream->isDisturbed(), TypeError, + JSG_REQUIRE(!this->stream.isDisturbed(js), TypeError, "This ReadableStream is disturbed (has already been read from), and cannot " "be used as a body."); } Body::ExtractedBody Body::extractBody(jsg::Lock& js, Initializer init) { - kj::Maybe> buffer; - kj::Maybe contentType; - + // The buffer-like cases below construct buffer-backed (rewindable) JsReadableStreams. The + // buffer handling -- including copying JsBufferSource inputs (a Fetch spec requirement that + // also severs any dependency on a detachable V8 backing store) and the no-deferred-proxying + // rule for data with V8 heap provenance -- lives in JsReadableStream's converting constructors. KJ_SWITCH_ONEOF(init) { - KJ_CASE_ONEOF(stream, jsg::Ref) { - return kj::mv(stream); + KJ_CASE_ONEOF(stream, JsReadableStream) { + return ExtractedBody(js, kj::mv(stream)); } KJ_CASE_ONEOF(gen, jsg::AsyncGeneratorIgnoringStrings) { - return ReadableStream::from(js, gen.release()); + return ExtractedBody(js, ReadableStream::from(js, gen.release())); } KJ_CASE_ONEOF(text, kj::String) { - contentType = kj::str(MimeType::PLAINTEXT_STRING); - buffer = kj::rc(kj::mv(text)); + auto contentType = kj::str(MimeType::PLAINTEXT_STRING); + return ExtractedBody(js, JsReadableStream(js, kj::mv(text)), kj::mv(contentType)); } KJ_CASE_ONEOF(bytesRef, jsg::JsRef) { - // Per the Fetch spec we must copy the input buffer here. Beyond spec conformance, this - // fixes a UAF: the incoming data may alias a v8::BackingStore whose underlying memory can - // be freed if the original ArrayBuffer is detached and transferred (e.g. via structuredClone - // with a transfer list) and then garbage collected. This applies to both resizable and - // fixed-size buffers. Copying severs the dependency on the V8 backing store. - buffer = kj::rc(kj::heapArray(bytesRef.getHandle(js).asArrayPtr())); + return ExtractedBody(js, JsReadableStream(js, kj::mv(bytesRef))); } KJ_CASE_ONEOF(blob, jsg::Ref) { // Blobs always have a type, but it defaults to an empty string. We should NOT set // Content-Type when the blob type is empty. + kj::Maybe contentType; kj::StringPtr blobType = blob->getType(); if (blobType != nullptr) { contentType = kj::str(blobType); } - buffer = kj::rc(kj::mv(blob)); + return ExtractedBody(js, JsReadableStream(js, kj::mv(blob)), kj::mv(contentType)); } KJ_CASE_ONEOF(formData, jsg::Ref) { // Make an array of characters containing random hexadecimal digits. @@ -128,164 +124,125 @@ Body::ExtractedBody Body::extractBody(jsg::Lock& js, Initializer init) { kj::FixedArray boundaryBuffer; workerd::getEntropy(boundaryBuffer); auto boundary = kj::encodeHex(boundaryBuffer); - contentType = MimeType::formDataWithBoundary(boundary); - buffer = kj::rc(formData->serialize(boundary)); + auto contentType = MimeType::formDataWithBoundary(boundary); + return ExtractedBody( + js, JsReadableStream(js, formData->serialize(boundary)), kj::mv(contentType)); } KJ_CASE_ONEOF(searchParams, jsg::Ref) { - contentType = MimeType::formUrlEncodedWithCharset("UTF-8"_kj); - buffer = kj::rc(searchParams->toString()); + return ExtractedBody(js, JsReadableStream(js, kj::mv(searchParams)), + MimeType::formUrlEncodedWithCharset("UTF-8"_kj)); } KJ_CASE_ONEOF(searchParams, jsg::Ref) { - contentType = MimeType::formUrlEncodedWithCharset("UTF-8"_kj); - buffer = kj::rc(searchParams->toString()); + return ExtractedBody(js, JsReadableStream(js, kj::mv(searchParams)), + MimeType::formUrlEncodedWithCharset("UTF-8"_kj)); } } - - // We use streams::newMemorySource() here rather than newSystemStream() wrapping a - // newMemoryInputStream() because we do NOT want deferred proxying for bodies with - // V8 heap provenance. Some buffer types (e.g. Blob data) may reference V8 heap memory - // and we must ensure the data is consumed and destroyed while under the isolate lock, - // which means deferred proxying is not allowed. - auto& buf = KJ_ASSERT_NONNULL(buffer); - auto rs = streams::newMemorySource(buf->view, buf.addRef().toOwn()); - - return { - js.alloc(IoContext::current(), kj::mv(rs)), - kj::mv(buffer), - kj::mv(contentType), - }; + KJ_UNREACHABLE; } -Body::Body(jsg::Lock& js, kj::Maybe init, Headers& headers) - : impl(kj::mv(init).map([&headers](auto i) -> Impl { - KJ_IF_SOME(ct, i.contentType) { - if (!headers.hasCommon(capnp::CommonHeaderName::CONTENT_TYPE)) { - // The spec allows the user to override the Content-Type, if they wish, so we only set - // the Content-Type if it doesn't already exist. - headers.setCommon(capnp::CommonHeaderName::CONTENT_TYPE, kj::mv(ct)); - } else KJ_IF_SOME(parsed, MimeType::tryParse(ct)) { - if (MimeType::FORM_DATA == parsed) { - // Custom content-type request/responses with FormData are broken since they require a - // boundary parameter only the FormData serializer can provide. Let's warn if a dev does this. - IoContext::current().logWarning( - "A FormData body was provided with a custom Content-Type header when constructing " - "a Request or Response object. This will prevent the recipient of the Request or " - "Response from being able to parse the body. Consider omitting the custom " - "Content-Type header."); - } - } +Body::Body(jsg::Lock& js, kj::Maybe init, Headers& headers): headersRef(headers) { + KJ_IF_SOME(i, init) { + KJ_IF_SOME(ct, i.contentType) { + if (!headers.hasCommon(capnp::CommonHeaderName::CONTENT_TYPE)) { + // The spec allows the user to override the Content-Type, if they wish, so we only set + // the Content-Type if it doesn't already exist. + headers.setCommon(capnp::CommonHeaderName::CONTENT_TYPE, kj::mv(ct)); + } else KJ_IF_SOME(parsed, MimeType::tryParse(ct)) { + if (MimeType::FORM_DATA == parsed) { + // Custom content-type request/responses with FormData are broken since they require a + // boundary parameter only the FormData serializer can provide. Let's warn if a dev does this. + IoContext::current().logWarning( + "A FormData body was provided with a custom Content-Type header when constructing " + "a Request or Response object. This will prevent the recipient of the Request or " + "Response from being able to parse the body. Consider omitting the custom " + "Content-Type header."); } - return kj::mv(i.impl); - })), - headersRef(headers) {} - -kj::Maybe> Body::getBodyBuffer() { - KJ_IF_SOME(i, impl) { - KJ_IF_SOME(b, i.buffer) { - return b.addRef(); + } } + bodyStream = kj::mv(i.stream); } - return kj::none; } bool Body::canRewindBody() { - KJ_IF_SOME(i, impl) { - // We can only rewind buffer-backed bodies. - return i.buffer != kj::none; - } - // Null bodies are trivially "rewindable". - return true; + // We can only rewind null or buffer-backed bodies. + return bodyStream.isNull() || bodyStream.isBufferBacked(); } void Body::rewindBody(jsg::Lock& js) { KJ_DASSERT(canRewindBody()); - KJ_IF_SOME(i, impl) { - auto bufferCopy = KJ_ASSERT_NONNULL(i.buffer).addRef(); - - // We use streams::newMemorySource() here rather than newSystemStream() wrapping a - // newMemoryInputStream() because we do NOT want deferred proxying for bodies with - // V8 heap provenance. Specifically, the bufferCopy.view here, while being a kj::ArrayPtr, - // will typically be wrapping a v8::BackingStore, and we must ensure that is is consumed - // and destroyed while under the isolate lock, which means deferred proxying is not allowed. - auto view = bufferCopy->view; - auto rs = streams::newMemorySource(view, bufferCopy.toOwn()); - i.stream = js.alloc(IoContext::current(), kj::mv(rs)); + KJ_IF_SOME(rewound, bodyStream.tryClone(js)) { + bodyStream = kj::mv(rewound); } + // A null body is trivially "rewound" (there is nothing to do). } void Body::nullifyBody() { - impl = kj::none; + bodyStream.nullify(); } -kj::Maybe> Body::getBody() { - KJ_IF_SOME(i, impl) { - return i.stream.addRef(); - } - return kj::none; +uint64_t Body::bufferingLimit() { + return IoContext::current().getLimitEnforcer().getBufferingLimit(); } -bool Body::getBodyUsed() { - KJ_IF_SOME(i, impl) { - return i.stream->isDisturbed(); + +kj::Maybe Body::getBody(jsg::Lock& js) { + if (bodyStream.isNull()) { + return kj::none; } - return false; + return bodyStream.addRef(js); +} +bool Body::getBodyUsed(jsg::Lock& js) { + return bodyStream.isDisturbed(js); } jsg::Promise> Body::arrayBuffer(jsg::Lock& js) { - KJ_IF_SOME(i, impl) { - return js.evalNow([&] { - JSG_REQUIRE(!i.stream->isDisturbed(), TypeError, - "Body has already been used. " - "It can only be used once. Use tee() first if you need to read it twice."); - return i.stream->getController().readAllBytes( - js, IoContext::current().getLimitEnforcer().getBufferingLimit()); - }); - } - - // If there's no body, we just return an empty array. + // A null body yields an empty result without consulting the IoContext (see bufferingLimit()). // See https://fetch.spec.whatwg.org/#concept-body-consume-body - auto ab = jsg::JsArrayBuffer::create(js, 0); - return js.resolvedPromise(ab.addRef(js)); + if (bodyStream.isNull()) { + return bodyStream.arrayBuffer(js, 0); + } + return js.evalNow([&] { return bodyStream.arrayBuffer(js, bufferingLimit()); }); } jsg::Promise> Body::bytes(jsg::Lock& js) { - return arrayBuffer(js).then(js, [](jsg::Lock& js, jsg::JsRef data) { - auto handle = data.getHandle(js); - return jsg::JsUint8Array::create(js, handle).addRef(js); - }); + if (bodyStream.isNull()) { + return bodyStream.bytes(js, 0); + } + return js.evalNow([&] { return bodyStream.bytes(js, bufferingLimit()); }); } jsg::Promise Body::text(jsg::Lock& js) { - KJ_IF_SOME(i, impl) { - return js.evalNow([&] { - JSG_REQUIRE(!i.stream->isDisturbed(), TypeError, - "Body has already been used. " - "It can only be used once. Use tee() first if you need to read it twice."); - - // A common mistake is to call .text() on non-text content, e.g. because you're implementing a - // search-and-replace across your whole site and you forgot that it'll apply to images too. - // When running with a warning handler, let's warn the developer if they do this. - auto& context = IoContext::current(); - if (context.hasWarningHandler()) { - KJ_IF_SOME(type, headersRef.getCommon(js, capnp::CommonHeaderName::CONTENT_TYPE)) { - maybeWarnIfNotText(js, type); - } - } - - return i.stream->getController().readAllText( - js, context.getLimitEnforcer().getBufferingLimit()); - }); + // A null body yields an empty string without consulting the IoContext. + // See https://fetch.spec.whatwg.org/#concept-body-consume-body + if (bodyStream.isNull()) { + return bodyStream.text(js, 0); } - // If there's no body, we just return an empty string. - // See https://fetch.spec.whatwg.org/#concept-body-consume-body - return js.resolvedPromise(kj::String()); + return js.evalNow([&] { + // Check for a disturbed body before emitting the non-text warning below. (bodyStream.text() + // performs the same check with the same error message; this one just runs first.) + JSG_REQUIRE(!bodyStream.isDisturbed(js), TypeError, + "Body has already been used. " + "It can only be used once. Use tee() first if you need to read it twice."); + + // A common mistake is to call .text() on non-text content, e.g. because you're implementing a + // search-and-replace across your whole site and you forgot that it'll apply to images too. + // When running with a warning handler, let's warn the developer if they do this. + auto& context = IoContext::current(); + if (context.hasWarningHandler()) { + KJ_IF_SOME(type, headersRef.getCommon(js, capnp::CommonHeaderName::CONTENT_TYPE)) { + maybeWarnIfNotText(js, type); + } + } + + return bodyStream.text(js, context.getLimitEnforcer().getBufferingLimit()); + }); } jsg::Promise> Body::formData(jsg::Lock& js) { auto formData = js.alloc(); return js.evalNow([&] { - JSG_REQUIRE(!getBodyUsed(), TypeError, + JSG_REQUIRE(!getBodyUsed(js), TypeError, "Body has already been used. " "It can only be used once. Use tee() first if you need to read it twice."); @@ -293,11 +250,9 @@ jsg::Promise> Body::formData(jsg::Lock& js) { JSG_REQUIRE_NONNULL(headersRef.getCommon(js, capnp::CommonHeaderName::CONTENT_TYPE), TypeError, "Parsing a Body as FormData requires a Content-Type header."); - KJ_IF_SOME(i, impl) { - KJ_ASSERT(!i.stream->isDisturbed()); - auto& context = IoContext::current(); - return i.stream->getController() - .readAllText(js, context.getLimitEnforcer().getBufferingLimit()) + if (!bodyStream.isNull()) { + KJ_ASSERT(!bodyStream.isDisturbed(js)); + return bodyStream.text(js, bufferingLimit()) .then(js, [contentType = kj::mv(contentType), formData = kj::mv(formData)]( auto& js, kj::String rawText) mutable { @@ -342,16 +297,15 @@ jsg::Promise> Body::blob(jsg::Lock& js) { } kj::Maybe Body::clone(jsg::Lock& js) { - KJ_IF_SOME(i, impl) { - auto branches = i.stream->tee(js); - - i.stream = kj::mv(branches[0]); - - return ExtractedBody{ - kj::mv(branches[1]), i.buffer.map([&](kj::Rc& b) { return b.addRef(); })}; + if (bodyStream.isNull()) { + return kj::none; } - return kj::none; + // tee() nullifies bodyStream and hands back two branches, each of which carries the retransmit + // buffer (if any). We keep one branch and give the other to the new Body. + auto tee = bodyStream.tee(js); + bodyStream = kj::mv(tee.branch1); + return ExtractedBody(js, kj::mv(tee.branch2)); } // ======================================================================================= @@ -435,16 +389,17 @@ jsg::Ref Request::constructor( headers = js.alloc(js, *oldRequest->headers); cf = oldRequest->cf.deepClone(js); if (!ignoreInputBody) { - JSG_REQUIRE(!oldRequest->getBodyUsed(), TypeError, + JSG_REQUIRE(!oldRequest->getBodyUsed(js), TypeError, "Cannot reconstruct a Request with a used body."); - KJ_IF_SOME(oldJsBody, oldRequest->getBody()) { + KJ_IF_SOME(oldJsBody, oldRequest->getBody(js)) { // The stream spec says to "create a proxy" for the passed in readable, which it // defines generically as creating a TransformStream and using pipeThrough to pass // the input stream through, giving the TransformStream's readable to the extracted - // body below. We don't need to do that. Instead, we just create a new ReadableStream - // that takes over ownership of the internals of the given stream. The given stream - // is left in a locked/disturbed mode so that it can no longer be used. - body = Body::ExtractedBody((oldJsBody)->detach(js), oldRequest->getBodyBuffer()); + // body below. We don't need to do that. Instead, we just create a new stream that + // takes over ownership of the internals of the given stream (carrying the retransmit + // buffer forward, if any). The given stream is left in a locked/disturbed mode so + // that it can no longer be used. + body = Body::ExtractedBody(js, oldJsBody.detach(js)); } } cacheMode = oldRequest->getCacheMode(); @@ -563,11 +518,11 @@ jsg::Ref Request::constructor( signal = otherRequest->getSignal(); headers = js.alloc(js, *otherRequest->headers); cf = otherRequest->cf.deepClone(js); - KJ_IF_SOME(b, otherRequest->getBody()) { + KJ_IF_SOME(b, otherRequest->getBody(js)) { // Note that unlike when `input` (Request ctor's 1st parameter) is a Request object, here // we're NOT stealing the other request's body, because we're supposed to pretend that the // other request is just a dictionary. - body = Body::ExtractedBody(kj::mv(b)); + body = Body::ExtractedBody(js, kj::mv(b)); } } } @@ -825,8 +780,8 @@ void Request::serialize(jsg::Lock& js, .headers = headers.addRef(), - .body = getBody().map([](jsg::Ref stream) -> Body::Initializer { - // jsg::Ref is one of the possible variants of Body::Initializer. + .body = getBody(js).map([](JsReadableStream stream) -> Body::Initializer { + // JsReadableStream is one of the possible variants of Body::Initializer. return kj::mv(stream); }), @@ -1130,11 +1085,12 @@ jsg::Ref Response::constructor(jsg::Lock& js, // no one relies on this behavior, we should remove this non-conformity. // Fail if the body is not backed by a buffer (i.e., it's an opaque ReadableStream). - auto& buffer = JSG_REQUIRE_NONNULL(KJ_ASSERT_NONNULL(body).impl.buffer, TypeError, + auto& extractedBody = KJ_ASSERT_NONNULL(body); + JSG_REQUIRE(extractedBody.stream.isBufferBacked(), TypeError, "Response with null body status (101, 204, 205, or 304) cannot have a body."); // Fail if the body is backed by a non-zero-length buffer. - JSG_REQUIRE(buffer->view.size() == 0, TypeError, + JSG_REQUIRE(KJ_ASSERT_NONNULL(extractedBody.stream.tryGetLength(js)) == 0, TypeError, "Response with null body status (101, 204, 205, or 304) cannot have a body."); auto& context = IoContext::current(); @@ -1273,7 +1229,7 @@ kj::Promise> Response::send(jsg::Lock& js, kj::HttpService::Response& outer, SendOptions options, kj::Maybe maybeReqHeaders) { - JSG_REQUIRE(!getBodyUsed(), TypeError, + JSG_REQUIRE(!getBodyUsed(js), TypeError, "Body has already been used. " "It can only be used once. Use tee() first if you need to read it twice."); @@ -1339,15 +1295,15 @@ kj::Promise> Response::send(jsg::Lock& js, } } return wsPromise; - } else KJ_IF_SOME(jsBody, getBody()) { + } else KJ_IF_SOME(jsBody, getBody(js)) { auto encoding = getContentEncoding(context, outHeaders, bodyEncoding, FeatureFlags::get(js)); - auto maybeLength = jsBody->tryGetLength(encoding); + auto maybeLength = jsBody.tryGetLength(js, encoding); auto stream = newSystemStream(outer.send(statusCode, getStatusText(), outHeaders, maybeLength), encoding); // We need to enter the AsyncContextFrame that was captured when the // Response was created before starting the loop. jsg::AsyncContextFrame::Scope scope(js, asyncContext); - return jsBody->pumpTo(js, kj::mv(stream), true); + return jsBody.pumpTo(js, kj::mv(stream), EndStream::YES); } else { outer.send(statusCode, getStatusText(), outHeaders, static_cast(0)); return addNoopDeferredProxy(kj::READY_NOW); @@ -1396,8 +1352,8 @@ jsg::Optional Response::getCf(jsg::Lock& js) { void Response::serialize(jsg::Lock& js, jsg::Serializer& serializer, const jsg::TypeHandler& initDictHandler, - const jsg::TypeHandler>>& streamHandler) { - serializer.write(js, jsg::JsValue(streamHandler.wrap(js, getBody()))); + const jsg::TypeHandler>& streamHandler) { + serializer.write(js, jsg::JsValue(streamHandler.wrap(js, getBody(js)))); // As with Request, we serialize the initializer dict as a JS object. serializer.write(js, @@ -1427,7 +1383,7 @@ jsg::Ref Response::deserialize(jsg::Lock& js, rpc::SerializationTag tag, jsg::Deserializer& deserializer, const jsg::TypeHandler& initDictHandler, - const jsg::TypeHandler>>& streamHandler) { + const jsg::TypeHandler>& streamHandler) { auto body = KJ_UNWRAP_OR(streamHandler.tryUnwrap(js, deserializer.readValue(js)), { JSG_FAIL_REQUIRE(DOMDataCloneError, "Deserialization failed: could not deserialize Response body"); @@ -1652,11 +1608,11 @@ jsg::Promise> fetchImplNoOutputLock(jsg::Lock& js, }); } else { kj::Maybe nativeRequest; - KJ_IF_SOME(jsBody, jsRequest->getBody()) { + KJ_IF_SOME(jsBody, jsRequest->getBody(js)) { // Note that for requests, we do not automatically handle Content-Encoding, because the fetch() // standard does not say that we should. Hence, we always use StreamEncoding::IDENTITY. // https://github.com/whatwg/fetch/issues/589 - auto maybeLength = jsBody->tryGetLength(StreamEncoding::IDENTITY); + auto maybeLength = jsBody.tryGetLength(js, StreamEncoding::IDENTITY); KJ_IF_SOME(ctx, traceContext) { KJ_IF_SOME(length, maybeLength) { ctx.setTag("http.request.body.size"_kjc, static_cast(length)); @@ -1701,9 +1657,9 @@ jsg::Promise> fetchImplNoOutputLock(jsg::Lock& js, // TODO(someday): Allow deferred proxying for bidirectional streaming. ioContext.addWaitUntil(handleCancelablePump( - AbortSignal::maybeCancelWrap( - js, signal, ioContext.waitForDeferredProxy(jsBody->pumpTo(js, kj::mv(stream), true))), - jsBody.addRef())); + AbortSignal::maybeCancelWrap(js, signal, + ioContext.waitForDeferredProxy(jsBody.pumpTo(js, kj::mv(stream), EndStream::YES))), + jsBody.addRef(js))); } else { nativeRequest = client->request(jsRequest->getMethodEnum(), url, headers, static_cast(0)); } @@ -1949,9 +1905,10 @@ jsg::Ref makeHttpResponse(jsg::Lock& js, // and the Fetch spec doesn't allow users to create Requests with CONNECT methods. kj::Maybe responseBody = kj::none; if (method != kj::HttpMethod::HEAD && !isNullBodyStatusCode(statusCode)) { - responseBody = Body::ExtractedBody(js.alloc(context, - newSystemStream(kj::mv(body), - getContentEncoding(context, headers, bodyEncoding, FeatureFlags::get(js))))); + responseBody = Body::ExtractedBody(js, + JsReadableStream::create(js, context, + newSystemStream(kj::mv(body), + getContentEncoding(context, headers, bodyEncoding, FeatureFlags::get(js))))); } // The Fetch spec defines "response URLs" as having no fragments. Since the last URL in the list @@ -2015,7 +1972,7 @@ jsg::Promise> fetchImplNoOutputLock(jsg::Lock& js, KJ_IF_SOME(dataUrl, DataUrl::tryParse(jsRequest->getUrl())) { // If the URL is a data URL, we need to handle it specially. - kj::Maybe> maybeResponseBody; + kj::Maybe maybeResponseBody; auto type = dataUrl.getMimeType().toString(); // The Fetch spec defines responses to HEAD or CONNECT requests, or responses with null body @@ -2027,7 +1984,8 @@ jsg::Promise> fetchImplNoOutputLock(jsg::Lock& js, if (jsRequest->getMethodEnum() == kj::HttpMethod::GET) { auto view = dataUrl.getData(); auto rs = streams::newMemorySource(view, kj::heap(kj::mv(dataUrl))); - maybeResponseBody.emplace(js.alloc(IoContext::current(), kj::mv(rs))); + maybeResponseBody.emplace( + JsReadableStream::create(js, IoContext::current(), kj::mv(rs))); } auto headers = js.alloc(); @@ -2293,11 +2251,11 @@ static jsg::Promise parseResponse( auto typeName = type.map([](const kj::String& s) -> kj::StringPtr { return s; }).orDefault("text"); if (typeName == "stream") { - KJ_IF_SOME(body, response->getBody()) { + KJ_IF_SOME(body, response->getBody(js)) { return js.resolvedPromise(Fetcher::GetResult(kj::mv(body))); } else { // Empty body. - return js.resolvedPromise(Fetcher::GetResult(js.alloc( + return js.resolvedPromise(Fetcher::GetResult(JsReadableStream::create(js, IoContext::current(), newSystemStream(newNullInputStream(), StreamEncoding::IDENTITY)))); } } diff --git a/src/workerd/api/http.h b/src/workerd/api/http.h index 27b71903cba..5c97f5b4da1 100644 --- a/src/workerd/api/http.h +++ b/src/workerd/api/http.h @@ -13,7 +13,7 @@ #include "web-socket.h" #include "worker-rpc.h" -#include +#include #include #include #include @@ -32,18 +32,18 @@ class Body: public jsg::Object { // The types of objects from which a Body can be created. // // If the object is a ReadableStream, Body will adopt it directly; otherwise the object is some - // sort of buffer-like source. In this case, Body will store its own ReadableStream that wraps the - // source, and it keeps a reference to the source object around. This allows Requests and - // Responses created from Strings, ArrayBuffers, FormDatas, Blobs, or URLSearchParams to be + // sort of buffer-like source. In this case, Body will store a buffer-backed (rewindable) + // JsReadableStream that wraps a copy of / a reference to the source data. This allows Requests + // and Responses created from Strings, ArrayBuffers, FormDatas, Blobs, or URLSearchParams to be // retransmitted. // // For an example of where this is important, consider a POST Request in redirect-follow mode and // containing a body: if passed to a fetch() call that results in a 307 or 308 response, fetch() // will re-POST to the new URL. If the body was constructed from a ReadableStream, this re-POST // will fail, because there is no body source left. On the other hand, if the body was constructed - // from any of the other source types, Body can create a new ReadableStream from the source, and - // the POST will successfully retransmit. - using Initializer = kj::OneOf, + // from any of the other source types, Body can create a new stream from the buffer, and the POST + // will successfully retransmit. + using Initializer = kj::OneOf, jsg::Ref, @@ -52,82 +52,14 @@ class Body: public jsg::Object { jsg::Ref, jsg::AsyncGeneratorIgnoringStrings>; - // The Fetch spec calls this type the body's "source", even though it really is a buffer. I end - // talking about things like "a buffer-backed body", whereas in standardese I should say - // "a body with a non-null source". - // - // I find that confusing, so let's just call it what it is: a Body::Buffer. - struct Buffer { - // In order to reconstruct buffer-backed ReadableStreams without gratuitous array copying, we - // need to be able to tie the lifetime of the source buffer to the lifetime of the - // ReadableStream's native stream, AND the lifetime of the Body itself. Thus we need - // refcounting. The Buffer instance itself will be refcounted. - // - // NOTE: ownBytes may contain a v8::Global reference, hence instances of `Buffer` must exist - // only within the V8 heap space. - kj::OneOf, jsg::Ref, jsg::JsRef> ownBytes; - // TODO(cleanup): When we integrate with V8's garbage collection APIs, we need to account for - // that here. - - // Bodies constructed from buffers rather than ReadableStreams can be retransmitted if necessary - // (e.g. for redirects, authentication). In these cases, we need to keep an ArrayPtr view onto - // the Array source itself, because the source may be a string, and thus have a trailing nul - // byte. - kj::ArrayPtr view; - - Buffer() = default; - Buffer(jsg::Lock& js, jsg::JsBufferSource& source) - : ownBytes(source.addRef(js)), - view(source.asArrayPtr()) { - // If the BufferSource is resizable, then it should have been copied into - // a kj::Array and passed in to that constructor. - KJ_ASSERT(!source.isResizable()); - } - Buffer(kj::Array array) - : ownBytes(kj::mv(array)), - view(ownBytes.get>()) {} - Buffer(kj::String string) - : ownBytes(string.releaseArray().releaseAsBytes()), - view([this] { - auto bytesIncludingNull = ownBytes.get>().asPtr(); - return bytesIncludingNull.first(bytesIncludingNull.size() - 1); - }()) {} - Buffer(jsg::Ref blob) - : ownBytes(kj::mv(blob)), - view(ownBytes.get>()->getData()) {} - - JSG_MEMORY_INFO(Buffer) { - KJ_SWITCH_ONEOF(ownBytes) { - KJ_CASE_ONEOF(ref, jsg::JsRef) { - tracker.trackField("ref", ref); - } - KJ_CASE_ONEOF(bytes, kj::Array) { - tracker.trackField("bytes", bytes); - } - KJ_CASE_ONEOF(blob, jsg::Ref) { - tracker.trackField("blob", blob); - } - } - } - }; - - struct Impl { - jsg::Ref stream; - kj::Maybe> buffer; - JSG_MEMORY_INFO(Impl) { - tracker.trackField("stream", stream); - KJ_IF_SOME(buf, buffer) { - tracker.trackFieldWithSize("buffer", buf->view.size()); - } - } - }; - + // The result of the "extract a body" algorithm: the body's stream plus the Content-Type it + // implies (if any). The stream carries its own retransmit buffer when the body was constructed + // from an in-memory source (see JsReadableStream::isBufferBacked()). struct ExtractedBody { - ExtractedBody(jsg::Ref stream, - kj::Maybe> source = kj::none, - kj::Maybe contentType = kj::none); + ExtractedBody( + jsg::Lock& js, JsReadableStream stream, kj::Maybe contentType = kj::none); - Impl impl; + JsReadableStream stream; kj::Maybe contentType; }; @@ -137,12 +69,10 @@ class Body: public jsg::Object { explicit Body(jsg::Lock& js, kj::Maybe init, Headers& headers); - kj::Maybe> getBodyBuffer(); - // The following body rewind/nullification functions are helpers for implementing fetch() redirect // handling. - // True if this body is null or buffer-backed, false if this body is a ReadableStream. + // True if this body is null or buffer-backed, false if this body is an opaque ReadableStream. bool canRewindBody(); // Reconstruct this body from its backing buffer. Precondition: `canRewindBody() == true`. @@ -154,8 +84,8 @@ class Body: public jsg::Object { // --------------------------------------------------------------------------- // JS API - kj::Maybe> getBody(); - bool getBodyUsed(); + kj::Maybe getBody(jsg::Lock& js); + bool getBodyUsed(jsg::Lock& js); jsg::Promise> arrayBuffer(jsg::Lock& js); jsg::Promise> bytes(jsg::Lock& js); jsg::Promise text(jsg::Lock& js); @@ -194,7 +124,7 @@ class Body: public jsg::Object { } void visitForMemoryInfo(jsg::MemoryTracker& tracker) const { - tracker.trackField("impl", impl); + bodyStream.visitForMemoryInfo(tracker); } protected: @@ -202,7 +132,14 @@ class Body: public jsg::Object { kj::Maybe clone(jsg::Lock& js); private: - kj::Maybe impl; + // The body stream. The null state represents a null body ("no body"). Buffer-backed streams + // carry their own retransmit buffer, making the body rewindable for redirect handling. + JsReadableStream bodyStream; + + // Returns the buffering limit to apply when consuming the body. Deliberately not consulted for + // null bodies: consuming a null body yields an empty result without any I/O, and must keep + // working outside of a request context (where IoContext::current() would throw). + uint64_t bufferingLimit(); // HACK: This `headersRef` variable refers to a Headers object in the Request/Response subclass. // As such, it will briefly dangle during object destruction. While unlikely to be an issue, @@ -210,9 +147,7 @@ class Body: public jsg::Object { Headers& headersRef; void visitForGc(jsg::GcVisitor& visitor) { - KJ_IF_SOME(i, impl) { - visitor.visit(i.stream); - } + visitor.visit(bodyStream); } }; @@ -355,7 +290,7 @@ class Fetcher: public JsRpcClientProvider { jsg::Optional>> requestInit); using GetResult = - kj::OneOf, jsg::JsRef, kj::String, jsg::Value>; + kj::OneOf, kj::String, jsg::Value>; jsg::Promise get(jsg::Lock& js, kj::String url, jsg::Optional type); @@ -1086,12 +1021,12 @@ class Response final: public Body { void serialize(jsg::Lock& js, jsg::Serializer& serializer, const jsg::TypeHandler& initDictHandler, - const jsg::TypeHandler>>& streamHandler); + const jsg::TypeHandler>& streamHandler); static jsg::Ref deserialize(jsg::Lock& js, rpc::SerializationTag tag, jsg::Deserializer& deserializer, const jsg::TypeHandler& initDictHandler, - const jsg::TypeHandler>>& streamHandler); + const jsg::TypeHandler>& streamHandler); JSG_SERIALIZABLE(rpc::SerializationTag::RESPONSE); diff --git a/src/workerd/api/js-readable-stream-test.c++ b/src/workerd/api/js-readable-stream-test.c++ new file mode 100644 index 00000000000..0db77198112 --- /dev/null +++ b/src/workerd/api/js-readable-stream-test.c++ @@ -0,0 +1,478 @@ +// Copyright (c) 2017-2022 Cloudflare, Inc. +// Licensed under the Apache 2.0 license found in the LICENSE file or at: +// https://opensource.org/licenses/Apache-2.0 + +#include +#include +#include + +#include + +namespace workerd::api { +namespace { + +constexpr uint64_t kLimit = 1024 * 1024; +constexpr kj::StringPtr kData = "hello world"_kj; + +// The user-visible message produced when consuming an already-consumed stream. This must match +// the message historically produced by the Body mixin exactly. +constexpr kj::StringPtr kBodyUsedError = + "Body has already been used. It can only be used once. Use tee() first if you need to " + "read it twice."_kj; + +// A ReadableStreamSource serving fixed content with a known length. +class ContentSource final: public ReadableStreamSource { + public: + ContentSource(kj::StringPtr data): data(data) {} + + kj::Promise tryRead(void* buffer, size_t minBytes, size_t maxBytes) override { + auto amount = kj::min(maxBytes, data.size() - offset); + memcpy(buffer, data.begin() + offset, amount); + offset += amount; + return amount; + } + + kj::Maybe tryGetLength(StreamEncoding encoding) override { + if (encoding == StreamEncoding::IDENTITY) { + return data.size() - offset; + } + return kj::none; + } + + private: + kj::StringPtr data; + size_t offset = 0; +}; + +// A WritableStreamSink accumulating written bytes into externally-owned state. +class CollectingSink final: public WritableStreamSink { + public: + CollectingSink(kj::Vector& data, bool& ended): data(data), ended(ended) {} + + kj::Promise write(kj::ArrayPtr buffer) override { + data.addAll(buffer); + return kj::READY_NOW; + } + + kj::Promise write(kj::ArrayPtr> pieces) override { + for (auto& piece: pieces) { + data.addAll(piece); + } + return kj::READY_NOW; + } + + kj::Promise end() override { + ended = true; + return kj::READY_NOW; + } + + void abort(kj::Exception reason) override {} + + private: + kj::Vector& data; + bool& ended; +}; + +KJ_TEST("JsReadableStream null state") { + TestFixture testFixture; + testFixture.runInIoContext([](const TestFixture::Environment& env) { + auto& js = env.js; + + JsReadableStream stream; + KJ_EXPECT(stream.isNull()); + KJ_EXPECT(!stream.isBufferBacked()); + KJ_EXPECT(!stream.isDisturbed(js)); + KJ_EXPECT(!stream.isLocked(js)); + KJ_EXPECT(stream.tryGetLength(js) == kj::none); + KJ_EXPECT(stream.tryClone(js) == kj::none); + KJ_EXPECT(stream.addRef(js).isNull()); + }); +} + +KJ_TEST("JsReadableStream null stream consumption yields empty results") { + TestFixture testFixture; + testFixture.runInIoContext([](const TestFixture::Environment& env) -> kj::Promise { + auto& js = env.js; + + JsReadableStream stream; + auto promise = stream.text(js, kLimit) + .then(js, + [](jsg::Lock& js, kj::String text) { + KJ_EXPECT(text == ""_kj); + return JsReadableStream().arrayBuffer(js, kLimit); + }) + .then(js, + [](jsg::Lock& js, jsg::JsRef buffer) { + KJ_EXPECT(buffer.getHandle(js).size() == 0); + return JsReadableStream().bytes(js, kLimit); + }) + .then(js, [](jsg::Lock& js, jsg::JsRef bytes) { + KJ_EXPECT(bytes.getHandle(js).size() == 0); + return JsReadableStream().blob(js, kLimit, kj::str("text/plain")); + }).then(js, [](jsg::Lock& js, jsg::Ref blob) { + KJ_EXPECT(blob->getSize() == 0); + KJ_EXPECT(blob->getType() == "text/plain"_kj); + }); + return env.context.awaitJs(js, kj::mv(promise)); + }); +} + +KJ_TEST("JsReadableStream null stream json() rejects like parsing an empty string") { + TestFixture testFixture; + testFixture.runInIoContext([](const TestFixture::Environment& env) -> kj::Promise { + auto& js = env.js; + + JsReadableStream stream; + auto promise = stream.json(js, kLimit).then(js, [](jsg::Lock& js, jsg::JsRef) { + KJ_FAIL_REQUIRE("expected json() on a null stream to reject"); + }, [](jsg::Lock& js, jsg::Value exception) { + auto e = js.exceptionToKj(kj::mv(exception)); + KJ_EXPECT(e.getDescription().contains("SyntaxError"), e.getDescription()); + }); + return env.context.awaitJs(js, kj::mv(promise)); + }); +} + +KJ_TEST("JsReadableStream buffer-backed stream") { + TestFixture testFixture; + testFixture.runInIoContext([](const TestFixture::Environment& env) -> kj::Promise { + auto& js = env.js; + + JsReadableStream stream(js, kj::str(kData)); + KJ_EXPECT(!stream.isNull()); + KJ_EXPECT(stream.isBufferBacked()); + KJ_EXPECT(!stream.isDisturbed(js)); + KJ_EXPECT(!stream.isLocked(js)); + KJ_EXPECT(KJ_ASSERT_NONNULL(stream.tryGetLength(js)) == kData.size()); + + auto promise = stream.text(js, kLimit).then(js, JSG_VISITABLE_LAMBDA((stream = kj::mv(stream)), (stream), (jsg::Lock& js, kj::String text) { + KJ_EXPECT(text == kData); + // Consumption leaves the underlying stream disturbed... + KJ_EXPECT(stream.isDisturbed(js)); + // ...but the JsReadableStream remains buffer-backed and therefore rewindable. + KJ_EXPECT(stream.isBufferBacked()); + })); + return env.context.awaitJs(js, kj::mv(promise)); + }); +} + +KJ_TEST("JsReadableStream consuming twice rejects with the historical Body error") { + TestFixture testFixture; + testFixture.runInIoContext([](const TestFixture::Environment& env) -> kj::Promise { + auto& js = env.js; + + JsReadableStream stream(js, kj::str(kData)); + auto promise = stream.text(js, kLimit) + .then(js, + JSG_VISITABLE_LAMBDA((stream = kj::mv(stream)), (stream), + (jsg::Lock& js, kj::String) { return stream.text(js, kLimit); })) + .then(js, [](jsg::Lock& js, kj::String) { + KJ_FAIL_REQUIRE("expected second consumption to reject"); + }, [](jsg::Lock& js, jsg::Value exception) { + auto e = js.exceptionToKj(kj::mv(exception)); + KJ_EXPECT( + e.getDescription() == kj::str("jsg.TypeError: ", kBodyUsedError), e.getDescription()); + }); + return env.context.awaitJs(js, kj::mv(promise)); + }); +} + +KJ_TEST("JsReadableStream consumption helpers return stream content") { + TestFixture testFixture; + testFixture.runInIoContext([](const TestFixture::Environment& env) -> kj::Promise { + auto& js = env.js; + + JsReadableStream forArrayBuffer(js, kj::str(kData)); + JsReadableStream forBytes(js, kj::str(kData)); + auto promise = forArrayBuffer.arrayBuffer(js, kLimit) + .then(js, + JSG_VISITABLE_LAMBDA((forBytes = kj::mv(forBytes)), (forBytes), + (jsg::Lock& js, jsg::JsRef buffer) { + auto handle = buffer.getHandle(js); + KJ_EXPECT(handle.asArrayPtr() == kData.asBytes()); + return forBytes.bytes(js, kLimit); + })) + .then(js, [](jsg::Lock& js, jsg::JsRef bytes) { + KJ_EXPECT(bytes.getHandle(js).size() == kData.size()); + }); + return env.context.awaitJs(js, kj::mv(promise)); + }); +} + +KJ_TEST("JsReadableStream blob carries the given content type") { + TestFixture testFixture; + testFixture.runInIoContext([](const TestFixture::Environment& env) -> kj::Promise { + auto& js = env.js; + + JsReadableStream stream(js, kj::str(kData)); + auto promise = stream.blob(js, kLimit, kj::str("text/plain;charset=UTF-8")) + .then(js, [](jsg::Lock& js, jsg::Ref blob) { + KJ_EXPECT(blob->getSize() == int(kData.size())); + KJ_EXPECT(blob->getType() == "text/plain;charset=UTF-8"_kj); + }); + return env.context.awaitJs(js, kj::mv(promise)); + }); +} + +KJ_TEST("JsReadableStream json parses stream content") { + TestFixture testFixture; + testFixture.runInIoContext([](const TestFixture::Environment& env) -> kj::Promise { + auto& js = env.js; + + JsReadableStream stream(js, kj::str("{\"hello\":\"world\"}")); + auto promise = + stream.json(js, kLimit).then(js, [](jsg::Lock& js, jsg::JsRef value) { + KJ_EXPECT(value.getHandle(js).isObject()); + }); + return env.context.awaitJs(js, kj::mv(promise)); + }); +} + +KJ_TEST("JsReadableStream addRef shares the underlying stream") { + TestFixture testFixture; + testFixture.runInIoContext([](const TestFixture::Environment& env) -> kj::Promise { + auto& js = env.js; + + JsReadableStream stream(js, kj::str(kData)); + auto ref = stream.addRef(js); + KJ_EXPECT(ref.isBufferBacked()); + KJ_EXPECT(!ref.isDisturbed(js)); + + // Consuming through the addRef disturbs the original: both wrap the same stream. + auto promise = ref.text(js, kLimit).then(js, JSG_VISITABLE_LAMBDA((stream = kj::mv(stream)), (stream), (jsg::Lock& js, kj::String text) { + KJ_EXPECT(text == kData); + KJ_EXPECT(stream.isDisturbed(js)); + })); + return env.context.awaitJs(js, kj::mv(promise)); + }); +} + +KJ_TEST("JsReadableStream tryClone rewinds a consumed buffer-backed stream") { + TestFixture testFixture; + testFixture.runInIoContext([](const TestFixture::Environment& env) -> kj::Promise { + auto& js = env.js; + + JsReadableStream stream(js, kj::str(kData)); + auto promise = + stream.text(js, kLimit) + .then(js, + JSG_VISITABLE_LAMBDA((stream = kj::mv(stream)), (stream), + (jsg::Lock& js, kj::String) { + KJ_EXPECT(stream.isDisturbed(js)); + // The clone is a fresh, independent, rewindable stream over the same bytes, even though + // the original has already been consumed. This is the property fetch() redirect handling + // relies upon. + auto clone = KJ_ASSERT_NONNULL(stream.tryClone(js)); + KJ_EXPECT(clone.isBufferBacked()); + KJ_EXPECT(!clone.isDisturbed(js)); + return clone.text(js, kLimit); + })) + .then(js, [](jsg::Lock& js, kj::String text) { KJ_EXPECT(text == kData); }); + return env.context.awaitJs(js, kj::mv(promise)); + }); +} + +KJ_TEST("JsReadableStream tee carries the buffer and nullifies the original") { + TestFixture testFixture; + testFixture.runInIoContext([](const TestFixture::Environment& env) -> kj::Promise { + auto& js = env.js; + + JsReadableStream stream(js, kj::str(kData)); + auto tee = stream.tee(js); + KJ_EXPECT(stream.isNull()); + KJ_EXPECT(tee.branch1.isBufferBacked()); + KJ_EXPECT(tee.branch2.isBufferBacked()); + + auto promise = tee.branch1.text(js, kLimit) + .then(js, + JSG_VISITABLE_LAMBDA((branch2 = kj::mv(tee.branch2)), (branch2), + (jsg::Lock& js, kj::String text) { + KJ_EXPECT(text == kData); + return branch2.text(js, kLimit); + })) + .then(js, [](jsg::Lock& js, kj::String text) { KJ_EXPECT(text == kData); }); + return env.context.awaitJs(js, kj::mv(promise)); + }); +} + +KJ_TEST("JsReadableStream detach takes over the stream, leaving a disturbed husk") { + TestFixture testFixture; + testFixture.runInIoContext([](const TestFixture::Environment& env) -> kj::Promise { + auto& js = env.js; + + JsReadableStream stream(js, kj::str(kData)); + auto detached = stream.detach(js); + // The original still holds the (now locked and disturbed) husk... + KJ_EXPECT(!stream.isNull()); + KJ_EXPECT(stream.isDisturbed(js)); + KJ_EXPECT(stream.isLocked(js)); + // ...while the detached stream took over the state, including the retransmit buffer. + KJ_EXPECT(detached.isBufferBacked()); + KJ_EXPECT(!detached.isDisturbed(js)); + + auto promise = detached.text(js, kLimit).then(js, [](jsg::Lock& js, kj::String text) { + KJ_EXPECT(text == kData); + }); + return env.context.awaitJs(js, kj::mv(promise)); + }); +} + +KJ_TEST("JsReadableStream detach of a consumed stream throws without IgnoreDisturbed") { + TestFixture testFixture; + testFixture.runInIoContext([](const TestFixture::Environment& env) -> kj::Promise { + auto& js = env.js; + + JsReadableStream stream(js, kj::str(kData)); + auto promise = stream.text(js, kLimit).then(js, JSG_VISITABLE_LAMBDA((stream = kj::mv(stream)), (stream), (jsg::Lock& js, kj::String) { + js.tryCatch([&] { + stream.detach(js); // defaults to IgnoreDisturbed::NO + KJ_FAIL_REQUIRE("expected detach() of a consumed stream to throw"); + }, [&](jsg::Value exception) { + auto e = js.exceptionToKj(kj::mv(exception)); + KJ_EXPECT(e.getDescription().contains("TypeError"), e.getDescription()); + }); + })); + return env.context.awaitJs(js, kj::mv(promise)); + }); +} + +KJ_TEST("JsReadableStream detach accepts IgnoreDisturbed::YES") { + TestFixture testFixture; + testFixture.runInIoContext([](const TestFixture::Environment& env) { + auto& js = env.js; + + JsReadableStream stream(js, kj::str(kData)); + auto detached = stream.detach(js, IgnoreDisturbed::YES); + KJ_EXPECT(!detached.isNull()); + KJ_EXPECT(stream.isDisturbed(js)); + }); +} + +KJ_TEST("JsReadableStream cancel disturbs the stream; null cancel is a no-op") { + TestFixture testFixture; + testFixture.runInIoContext([](const TestFixture::Environment& env) -> kj::Promise { + auto& js = env.js; + + // Canceling a null stream is a no-op that yields a resolved promise. + auto nullPromise = JsReadableStream().cancel(js, kj::none); + + JsReadableStream stream(js, kj::str(kData)); + auto promise = + stream.cancel(js, js.error("lost interest")).then(js, JSG_VISITABLE_LAMBDA((stream = kj::mv(stream)), (stream), (jsg::Lock& js) { + // Cancellation counts as disturbing the stream. + KJ_EXPECT(stream.isDisturbed(js)); + })); + return env.context.awaitJs(js, + nullPromise.then( + js, [promise = kj::mv(promise)](jsg::Lock& js) mutable { return kj::mv(promise); })); + }); +} + +KJ_TEST("JsReadableStream forceCancel cancels even when the stream is locked") { + TestFixture testFixture; + testFixture.runInIoContext([](const TestFixture::Environment& env) -> kj::Promise { + auto& js = env.js; + + // detach() leaves the original as a locked (and disturbed) husk, which is a convenient way + // to produce a locked stream without a reader. + JsReadableStream stream(js, kj::str(kData)); + auto detached = stream.detach(js); + KJ_EXPECT(stream.isLocked(js)); + + // cancel() refuses to cancel a locked stream, exactly like ReadableStream.prototype.cancel. + auto promise = stream.cancel(js, kj::none) + .then(js, [](jsg::Lock& js) { + KJ_FAIL_REQUIRE("expected cancel() of a locked stream to reject"); + }, [](jsg::Lock& js, jsg::Value exception) { + auto e = js.exceptionToKj(kj::mv(exception)); + KJ_EXPECT(e.getDescription().contains("currently locked to a reader"), e.getDescription()); + }).then(js, JSG_VISITABLE_LAMBDA((stream = kj::mv(stream)), (stream), (jsg::Lock& js) { + // forceCancel() tears the stream down regardless. + return stream.forceCancel(js, kj::none); + })); + return env.context.awaitJs(js, kj::mv(promise)); + }); +} + +KJ_TEST("JsReadableStream setPendingClosure is safe on live and null streams") { + TestFixture testFixture; + testFixture.runInIoContext([](const TestFixture::Environment& env) { + auto& js = env.js; + + JsReadableStream().setPendingClosure(js); // null: no-op + + JsReadableStream stream(js, kj::str(kData)); + stream.setPendingClosure(js); + KJ_EXPECT(!stream.isDisturbed(js)); + }); +} + +KJ_TEST("JsReadableStream onEof plumbs through to the underlying stream") { + // Note: EOF signaling fires when the stream is consumed through reader-based reads (see the + // signalEof() calls in ReadableStreamInternalController). The JsReadableStream consumption + // helpers remove the underlying source and bypass that path, so end-to-end EOF behavior is + // exercised by the socket tests instead. Here we just verify the plumbing: onEof() can be + // registered and the stream remains fully usable afterwards. + TestFixture testFixture; + testFixture.runInIoContext([](const TestFixture::Environment& env) -> kj::Promise { + auto& js = env.js; + + auto stream = JsReadableStream::create(js, env.context, kj::heap(kData)); + auto eofPromise = stream.onEof(js); + eofPromise.markAsHandled(js); + auto promise = stream.text(js, kLimit).then(js, [](jsg::Lock& js, kj::String text) { + KJ_EXPECT(text == kData); + }); + return env.context.awaitJs(js, kj::mv(promise)); + }); +} + +KJ_TEST("JsReadableStream create wraps a native source") { + TestFixture testFixture; + testFixture.runInIoContext([](const TestFixture::Environment& env) -> kj::Promise { + auto& js = env.js; + + auto stream = JsReadableStream::create(js, env.context, kj::heap(kData)); + KJ_EXPECT(!stream.isNull()); + KJ_EXPECT(!stream.isBufferBacked()); + KJ_EXPECT(KJ_ASSERT_NONNULL(stream.tryGetLength(js)) == kData.size()); + + auto promise = stream.text(js, kLimit).then(js, [](jsg::Lock& js, kj::String text) { + KJ_EXPECT(text == kData); + }); + return env.context.awaitJs(js, kj::mv(promise)); + }); +} + +KJ_TEST("JsReadableStream pumpTo drains into a sink and ends it") { + TestFixture testFixture; + kj::Vector collected; + bool ended = false; + testFixture.runInIoContext([&](const TestFixture::Environment& env) -> kj::Promise { + auto& js = env.js; + + JsReadableStream stream(js, kj::str(kData)); + return stream.pumpTo(js, kj::heap(collected, ended), EndStream::YES) + .then([](DeferredProxy proxy) { return kj::mv(proxy.proxyTask); }); + }); + KJ_EXPECT(collected.asPtr() == kData.asBytes()); + KJ_EXPECT(ended); +} + +KJ_TEST("JsReadableStream pumpTo with EndStream::NO leaves the sink open") { + TestFixture testFixture; + kj::Vector collected; + bool ended = false; + testFixture.runInIoContext([&](const TestFixture::Environment& env) -> kj::Promise { + auto& js = env.js; + + JsReadableStream stream(js, kj::str(kData)); + return stream.pumpTo(js, kj::heap(collected, ended), EndStream::NO) + .then([](DeferredProxy proxy) { return kj::mv(proxy.proxyTask); }); + }); + KJ_EXPECT(collected.asPtr() == kData.asBytes()); + KJ_EXPECT(!ended); +} + +} // namespace +} // namespace workerd::api diff --git a/src/workerd/api/js-readable-stream.c++ b/src/workerd/api/js-readable-stream.c++ new file mode 100644 index 00000000000..b9569372be8 --- /dev/null +++ b/src/workerd/api/js-readable-stream.c++ @@ -0,0 +1,472 @@ +// Copyright (c) 2017-2022 Cloudflare, Inc. +// Licensed under the Apache 2.0 license found in the LICENSE file or at: +// https://opensource.org/licenses/Apache-2.0 + +#include +#include +#include +#include +#include + +#include + +namespace workerd::api { + +namespace { +// The exact error message produced when attempting to consume an already-consumed body. This +// text is user-visible and load-bearing: it matches the message historically produced by the +// Body mixin (see http.c++) and must not change. +constexpr kj::StringPtr kBodyUsedError = + "Body has already been used. It can only be used once. Use tee() first if you need to " + "read it twice."_kj; + +// Convert a kj::String to owned bytes, excluding the trailing NUL terminator. The returned array +// owns the original character buffer but views only the string's bytes. +kj::Array stringToBytes(kj::String data) { + size_t len = data.size(); + auto chars = data.releaseArray(); + return chars.first(len).asBytes().asConst().attach(kj::mv(chars)); +} + +// Copy the bytes viewed by a JsBufferSource into an owned array. Copying (rather than retaining the +// JsBufferSource) severs the dependency on the V8 backing store, which could otherwise be freed if +// the source ArrayBuffer is detached/transferred and then garbage collected. This also satisfies +// the Fetch requirement to copy the input buffer. +kj::Array bufferSourceToBytes( + jsg::Lock& js, jsg::JsRef& view) { + auto copy = kj::heapArray(view.getHandle(js).asArrayPtr()); + return copy.asPtr().asConst().attach(kj::mv(copy)); +} +} // namespace + +// The -Wdangling-field warnings below are false positives. view captures the heap buffer pointer +// managed by data, not the address of the data parameter itself. Moving data into owned transfers +// ownership of that heap buffer without changing its address, so view remains valid. +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wdangling-field" +JsReadableStream::Buffer::Buffer(kj::Array data): view(data), owned(kj::mv(data)) {} + +JsReadableStream::Buffer::Buffer(jsg::Ref data): view(data->getData()), owned(kj::mv(data)) {} +#pragma clang diagnostic pop + +JsReadableStream::Impl JsReadableStream::bufferBackedImpl(jsg::Lock& js, kj::Rc buffer) { + // Use streams::newMemorySource() rather than newSystemStream() wrapping a memory input stream: + // buffer-backed bodies may have V8 heap provenance and therefore must NOT be deferred-proxied. + // The data must be consumed and destroyed while under the isolate lock. + // + // TODO(streams-ts): Like create(), the stream construction here must dispatch on the + // worker's configuration once the TypeScript implementation lands. + auto view = buffer->view; + auto source = streams::newMemorySource(view, buffer.addRef().toOwn()); + return Impl{ + .stream = StreamImpl(js.alloc(IoContext::current(), kj::mv(source))), + .maybeOwnedBuffer = kj::mv(buffer), + }; +} + +JsReadableStream::JsReadableStream(jsg::Ref stream) + : impl(Impl{ + .stream = StreamImpl(kj::mv(stream)), + .maybeOwnedBuffer = kj::none, + }) {} + +JsReadableStream::JsReadableStream(jsg::Lock&, jsg::JsRef) { + KJ_UNIMPLEMENTED("TypeScript-backed ReadableStream is not yet supported"); +} + +JsReadableStream::JsReadableStream(jsg::Lock& js, kj::Array data) + : impl(bufferBackedImpl(js, kj::rc(kj::mv(data)))) {} + +JsReadableStream::JsReadableStream(jsg::Lock& js, kj::String data) + : JsReadableStream(js, stringToBytes(kj::mv(data))) {} + +JsReadableStream::JsReadableStream(jsg::Lock& js, jsg::JsRef view) + : JsReadableStream(js, bufferSourceToBytes(js, view)) {} + +JsReadableStream::JsReadableStream(jsg::Lock& js, jsg::Ref blob) + : impl(bufferBackedImpl(js, kj::rc(kj::mv(blob)))) {} + +JsReadableStream::JsReadableStream(jsg::Lock& js, jsg::Ref urlSearchParams) + : JsReadableStream(js, urlSearchParams->toString()) {} + +JsReadableStream::JsReadableStream( + jsg::Lock& js, jsg::Ref urlSearchParams) + : JsReadableStream(js, urlSearchParams->toString()) {} + +JsReadableStream JsReadableStream::create( + jsg::Lock& js, IoContext& ioContext, kj::Own source) { + // TODO(streams-ts): Dispatch on the worker's configuration to construct either the legacy + // C++ ReadableStream or a TypeScript-backed stream. + return JsReadableStream(js.alloc(ioContext, kj::mv(source))); +} + +JsReadableStream JsReadableStream::addRef(jsg::Lock& js) { + KJ_IF_SOME(i, impl) { + KJ_SWITCH_ONEOF(i.stream) { + KJ_CASE_ONEOF(stream, jsg::Ref) { + return JsReadableStream(Impl{ + .stream = StreamImpl(stream.addRef()), + .maybeOwnedBuffer = i.maybeOwnedBuffer.map([](kj::Rc& b) { return b.addRef(); }), + }); + } + KJ_CASE_ONEOF(obj, jsg::JsRef) { + KJ_UNIMPLEMENTED("TypeScript-backed ReadableStream is not yet supported"); + } + } + KJ_UNREACHABLE; + } + // addRef() of a null stream is a null stream. + return JsReadableStream(); +} + +bool JsReadableStream::isNull() const { + return impl == kj::none; +} + +bool JsReadableStream::isBufferBacked() const { + KJ_IF_SOME(i, impl) { + return i.maybeOwnedBuffer != kj::none; + } + return false; +} + +bool JsReadableStream::isDisturbed(jsg::Lock& js) { + KJ_IF_SOME(i, impl) { + KJ_SWITCH_ONEOF(i.stream) { + KJ_CASE_ONEOF(stream, jsg::Ref) { + return stream->isDisturbed(); + } + KJ_CASE_ONEOF(obj, jsg::JsRef) { + KJ_UNIMPLEMENTED("TypeScript-backed ReadableStream is not yet supported"); + } + } + KJ_UNREACHABLE; + } + return false; +} + +bool JsReadableStream::isLocked(jsg::Lock& js) { + KJ_IF_SOME(i, impl) { + KJ_SWITCH_ONEOF(i.stream) { + KJ_CASE_ONEOF(stream, jsg::Ref) { + return stream->isLocked(); + } + KJ_CASE_ONEOF(obj, jsg::JsRef) { + KJ_UNIMPLEMENTED("TypeScript-backed ReadableStream is not yet supported"); + } + } + KJ_UNREACHABLE; + } + return false; +} + +jsg::Promise JsReadableStream::cancel(jsg::Lock& js, jsg::Optional reason) { + KJ_IF_SOME(i, impl) { + KJ_SWITCH_ONEOF(i.stream) { + KJ_CASE_ONEOF(stream, jsg::Ref) { + return stream->cancel(js, kj::mv(reason)); + } + KJ_CASE_ONEOF(obj, jsg::JsRef) { + KJ_UNIMPLEMENTED("TypeScript-backed ReadableStream is not yet supported"); + } + } + KJ_UNREACHABLE; + } + // Canceling a null stream is a no-op. + return js.resolvedPromise(); +} + +jsg::Promise JsReadableStream::forceCancel( + jsg::Lock& js, jsg::Optional reason) { + KJ_IF_SOME(i, impl) { + KJ_SWITCH_ONEOF(i.stream) { + KJ_CASE_ONEOF(stream, jsg::Ref) { + // Going through the controller (rather than ReadableStream::cancel()) deliberately + // bypasses the "is locked" check: this cancels the stream out from under any reader. + return stream->getController().cancel(js, kj::mv(reason)); + } + KJ_CASE_ONEOF(obj, jsg::JsRef) { + KJ_UNIMPLEMENTED("TypeScript-backed ReadableStream is not yet supported"); + } + } + KJ_UNREACHABLE; + } + // Canceling a null stream is a no-op. + return js.resolvedPromise(); +} + +void JsReadableStream::setPendingClosure(jsg::Lock& js) { + KJ_IF_SOME(i, impl) { + KJ_SWITCH_ONEOF(i.stream) { + KJ_CASE_ONEOF(stream, jsg::Ref) { + stream->getController().setPendingClosure(); + } + KJ_CASE_ONEOF(obj, jsg::JsRef) { + KJ_UNIMPLEMENTED("TypeScript-backed ReadableStream is not yet supported"); + } + } + } +} + +jsg::Promise JsReadableStream::onEof(jsg::Lock& js) { + auto& i = KJ_ASSERT_NONNULL(impl, "onEof() called on a null JsReadableStream"); + KJ_SWITCH_ONEOF(i.stream) { + KJ_CASE_ONEOF(stream, jsg::Ref) { + return stream->onEof(js); + } + KJ_CASE_ONEOF(obj, jsg::JsRef) { + KJ_UNIMPLEMENTED("TypeScript-backed ReadableStream is not yet supported"); + } + } + KJ_UNREACHABLE; +} + +kj::Maybe JsReadableStream::tryGetLength(jsg::Lock& js, StreamEncoding encoding) { + KJ_IF_SOME(i, impl) { + KJ_SWITCH_ONEOF(i.stream) { + KJ_CASE_ONEOF(stream, jsg::Ref) { + return stream->tryGetLength(encoding); + } + KJ_CASE_ONEOF(obj, jsg::JsRef) { + KJ_UNIMPLEMENTED("TypeScript-backed ReadableStream is not yet supported"); + } + } + KJ_UNREACHABLE; + } + return kj::none; +} + +jsg::Promise> JsReadableStream::arrayBuffer( + jsg::Lock& js, uint64_t limit) { + KJ_IF_SOME(i, impl) { + KJ_SWITCH_ONEOF(i.stream) { + KJ_CASE_ONEOF(stream, jsg::Ref) { + if (stream->isDisturbed()) { + return js.rejectedPromise>(js.typeError(kBodyUsedError)); + } + return stream->getController().readAllBytes(js, limit); + } + KJ_CASE_ONEOF(obj, jsg::JsRef) { + KJ_UNIMPLEMENTED("TypeScript-backed ReadableStream is not yet supported"); + } + } + KJ_UNREACHABLE; + } + + // A null stream yields an empty result. + return js.resolvedPromise(jsg::JsArrayBuffer::create(js, 0).addRef(js)); +} + +jsg::Promise JsReadableStream::text(jsg::Lock& js, uint64_t limit) { + KJ_IF_SOME(i, impl) { + KJ_SWITCH_ONEOF(i.stream) { + KJ_CASE_ONEOF(stream, jsg::Ref) { + if (stream->isDisturbed()) { + return js.rejectedPromise(js.typeError(kBodyUsedError)); + } + return stream->getController().readAllText(js, limit); + } + KJ_CASE_ONEOF(obj, jsg::JsRef) { + KJ_UNIMPLEMENTED("TypeScript-backed ReadableStream is not yet supported"); + } + } + KJ_UNREACHABLE; + } + + // A null stream yields an empty result. + return js.resolvedPromise(kj::String()); +} + +jsg::Promise> JsReadableStream::bytes(jsg::Lock& js, uint64_t limit) { + KJ_IF_SOME(i, impl) { + KJ_SWITCH_ONEOF(i.stream) { + KJ_CASE_ONEOF(stream, jsg::Ref) { + if (stream->isDisturbed()) { + return js.rejectedPromise>(js.typeError(kBodyUsedError)); + } + return stream->getController().readAllBytes(js, limit).then( + js, [](jsg::Lock& js, jsg::JsRef data) { + auto handle = data.getHandle(js); + return jsg::JsUint8Array::create(js, handle).addRef(js); + }); + } + KJ_CASE_ONEOF(obj, jsg::JsRef) { + KJ_UNIMPLEMENTED("TypeScript-backed ReadableStream is not yet supported"); + } + } + KJ_UNREACHABLE; + } + + // A null stream yields an empty result. + return js.resolvedPromise(jsg::JsUint8Array::create(js, static_cast(0)).addRef(js)); +} + +jsg::Promise> JsReadableStream::json(jsg::Lock& js, uint64_t limit) { + KJ_IF_SOME(i, impl) { + KJ_SWITCH_ONEOF(i.stream) { + KJ_CASE_ONEOF(stream, jsg::Ref) { + if (stream->isDisturbed()) { + return js.rejectedPromise>(js.typeError(kBodyUsedError)); + } + return stream->getController().readAllText(js, limit).then( + js, [](jsg::Lock& js, kj::String text) { + auto parsed = js.parseJson(text); + return jsg::JsValue(parsed.getHandle(js)).addRef(js); + }); + } + KJ_CASE_ONEOF(obj, jsg::JsRef) { + KJ_UNIMPLEMENTED("TypeScript-backed ReadableStream is not yet supported"); + } + } + KJ_UNREACHABLE; + } + + // A null stream is an empty body. Match Body::json() semantics exactly: resolve the empty + // text first, then parse it in a continuation. Parsing "" as JSON throws a SyntaxError, so + // the returned promise rejects. + return js.resolvedPromise(kj::String()).then(js, [](jsg::Lock& js, kj::String text) { + auto parsed = js.parseJson(text); + return jsg::JsValue(parsed.getHandle(js)).addRef(js); + }); +} + +jsg::Promise> JsReadableStream::blob( + jsg::Lock& js, uint64_t limit, kj::String contentType) { + KJ_IF_SOME(i, impl) { + KJ_SWITCH_ONEOF(i.stream) { + KJ_CASE_ONEOF(stream, jsg::Ref) { + if (stream->isDisturbed()) { + return js.rejectedPromise>(js.typeError(kBodyUsedError)); + } + return stream->getController().readAllBytes(js, limit).then(js, + [contentType = kj::mv(contentType)]( + jsg::Lock& js, jsg::JsRef buffer) mutable { + return js.alloc(js, jsg::JsBufferSource(buffer.getHandle(js)), kj::mv(contentType)); + }); + } + KJ_CASE_ONEOF(obj, jsg::JsRef) { + KJ_UNIMPLEMENTED("TypeScript-backed ReadableStream is not yet supported"); + } + } + KJ_UNREACHABLE; + } + + // A null stream yields an empty Blob with the given Content-Type. + return js.resolvedPromise(js.alloc(kj::mv(contentType))); +} + +kj::Promise> JsReadableStream::pumpTo( + jsg::Lock& js, kj::Own sink, EndStream end) { + auto& i = KJ_ASSERT_NONNULL(impl, "pumpTo() called on a null JsReadableStream"); + KJ_SWITCH_ONEOF(i.stream) { + KJ_CASE_ONEOF(stream, jsg::Ref) { + return stream->pumpTo(js, kj::mv(sink), end == EndStream::YES); + } + KJ_CASE_ONEOF(obj, jsg::JsRef) { + KJ_UNIMPLEMENTED("TypeScript-backed ReadableStream is not yet supported"); + } + } + KJ_UNREACHABLE; +} + +JsReadableStream::Tee JsReadableStream::tee(jsg::Lock& js) { + auto& i = KJ_ASSERT_NONNULL(impl, "tee() called on a null JsReadableStream"); + KJ_SWITCH_ONEOF(i.stream) { + KJ_CASE_ONEOF(stream, jsg::Ref) { + auto branches = stream->tee(js); + // Both branches share the retransmit buffer (if any) so they remain independently rewindable. + auto buffer1 = i.maybeOwnedBuffer.map([](kj::Rc& b) { return b.addRef(); }); + auto buffer2 = i.maybeOwnedBuffer.map([](kj::Rc& b) { return b.addRef(); }); + + // tee() consumes `this`: the original stream is now locked/disturbed and unusable, so + // represent that by nullifying. Everything needed for the result has already been extracted + // into locals above. + impl = kj::none; + + return Tee{ + .branch1 = JsReadableStream(Impl{ + .stream = StreamImpl(kj::mv(branches[0])), + .maybeOwnedBuffer = kj::mv(buffer1), + }), + .branch2 = JsReadableStream(Impl{ + .stream = StreamImpl(kj::mv(branches[1])), + .maybeOwnedBuffer = kj::mv(buffer2), + }), + }; + } + KJ_CASE_ONEOF(obj, jsg::JsRef) { + KJ_UNIMPLEMENTED("TypeScript-backed ReadableStream is not yet supported"); + } + } + KJ_UNREACHABLE; +} + +kj::Maybe JsReadableStream::tryClone(jsg::Lock& js) { + KJ_IF_SOME(i, impl) { + KJ_IF_SOME(buffer, i.maybeOwnedBuffer) { + // Non-mutating: build a fresh stream over the same buffer bytes, carrying the buffer forward + // so the clone is itself rewindable. `this` is left untouched. + return JsReadableStream(bufferBackedImpl(js, buffer.addRef())); + } + } + // Null or not buffer-backed: nothing to rewind. + return kj::none; +} + +JsReadableStream JsReadableStream::detach(jsg::Lock& js, IgnoreDisturbed ignoreDisturbed) { + auto& i = KJ_ASSERT_NONNULL(impl, "detach() called on a null JsReadableStream"); + KJ_SWITCH_ONEOF(i.stream) { + KJ_CASE_ONEOF(stream, jsg::Ref) { + // ReadableStream::detach() takes over the internal state, leaving the original stream (i.e. + // `this`) locked and disturbed. `this` retains a reference to the retransmit buffer. + auto detached = stream->detach(js, ignoreDisturbed == IgnoreDisturbed::YES); + return JsReadableStream(Impl{ + .stream = StreamImpl(kj::mv(detached)), + .maybeOwnedBuffer = i.maybeOwnedBuffer.map([](kj::Rc& b) { return b.addRef(); }), + }); + } + KJ_CASE_ONEOF(obj, jsg::JsRef) { + KJ_UNIMPLEMENTED("TypeScript-backed ReadableStream is not yet supported"); + } + } + KJ_UNREACHABLE; +} + +void JsReadableStream::nullify() { + impl = kj::none; +} + +void JsReadableStream::visitForGc(jsg::GcVisitor& visitor) { + KJ_IF_SOME(i, impl) { + KJ_SWITCH_ONEOF(i.stream) { + KJ_CASE_ONEOF(stream, jsg::Ref) { + visitor.visit(stream); + } + KJ_CASE_ONEOF(obj, jsg::JsRef) { + visitor.visit(obj); + } + } + // Note: the retransmit Buffer's owned Blob reference is intentionally NOT traced here, matching + // the pre-refactor Body behavior (it is kept alive as a strong reference, not via GC tracing). + } +} + +void JsReadableStream::visitForMemoryInfo(jsg::MemoryTracker& tracker) const { + KJ_IF_SOME(i, impl) { + KJ_SWITCH_ONEOF(i.stream) { + KJ_CASE_ONEOF(stream, jsg::Ref) { + tracker.trackField("stream", stream); + } + KJ_CASE_ONEOF(obj, jsg::JsRef) { + // TODO(streams-ts): track the JS object's memory once TS-backed streams are supported. + // (jsg::JsRef does not satisfy the MemoryRetainer concept, so it can't be passed to + // trackField() directly.) + } + } + KJ_IF_SOME(buffer, i.maybeOwnedBuffer) { + tracker.trackFieldWithSize("buffer", buffer->view.size()); + } + } +} + +} // namespace workerd::api diff --git a/src/workerd/api/js-readable-stream.h b/src/workerd/api/js-readable-stream.h new file mode 100644 index 00000000000..e8befde92a0 --- /dev/null +++ b/src/workerd/api/js-readable-stream.h @@ -0,0 +1,252 @@ +// Copyright (c) 2017-2022 Cloudflare, Inc. +// Licensed under the Apache 2.0 license found in the LICENSE file or at: +// https://opensource.org/licenses/Apache-2.0 + +#pragma once + +#include +#include +#include + +#include +#include +#include + +namespace workerd::api { + +class Blob; +class FormData; +class URLSearchParams; +namespace url { +class URLSearchParams; +} + +WD_STRONG_BOOL(EndStream); +WD_STRONG_BOOL(IgnoreDisturbed); + +// An abstraction of a ReadableStream, backed by either a C++ implemented ReadableStream +// (defined in src/workerd/api/streams/*) or a TypeScript implemented ReadableStream (defined +// in src/per_isolate/webstreams). The API is limited strictly to the methods that are needed +// by the C++ side of workerd. It is not intended to be a complete implementation of the +// ReadableStream API. +// +// A JsReadableStream is one of: +// * null / empty (isNull()) -- represents "no body"; +// * buffer-backed -- wraps an in-memory data source that can be rewound and retransmitted +// (isBufferBacked()); or +// * stream-backed -- wraps an opaque, one-shot ReadableStream. +// +// Backend branching: the underlying stream is stored as a kj::OneOf so that a TypeScript +// implemented ReadableStream (represented as a JS object) can eventually be supported alongside +// the legacy C++ ReadableStream. Every method that touches the underlying stream switches on the +// backend. Today only the C++ ReadableStream backend is implemented; the TypeScript backend +// branches are KJ_UNIMPLEMENTED and the corresponding constructor cannot yet be used. When the +// TypeScript implementation lands, fill in those branches -- the stored type (and therefore +// Body / ExtractedBody) will not need to change. +class JsReadableStream final { + public: + // The underlying stream. Today only the legacy C++ ReadableStream alternative is populated; the + // jsg::JsRef alternative is reserved for the future TypeScript implementation. + using StreamImpl = kj::OneOf, jsg::JsRef>; + + // Holds the in-memory data backing a buffer-backed (rewindable) JsReadableStream. Refcounted so + // its lifetime can be tied both to the native stream (as the newMemorySource backing) and to the + // JsReadableStream itself (so the stream can be rebuilt for retransmission). + struct Buffer { + kj::ArrayPtr view; + kj::OneOf, jsg::Ref> owned; + + explicit Buffer(kj::Array data); + explicit Buffer(jsg::Ref data); + }; + + struct Impl { + StreamImpl stream; + kj::Maybe> maybeOwnedBuffer; + }; + + // The result of tee(): two independent branches that each read the same data. + struct Tee; + + // Create a null / empty JsReadableStream. + JsReadableStream() = default; + + // Adopt an existing legacy C++ ReadableStream. The result is not rewindable. + JsReadableStream(jsg::Ref stream); + + // Adopt a TypeScript-implemented ReadableStream (a JS object). Not yet supported. + JsReadableStream(jsg::Lock& js, jsg::JsRef obj); + + // Create a buffer-backed (rewindable) JsReadableStream from various in-memory data sources. The + // kj::String, jsg::JsRef, and URLSearchParams overloads are conveniences + // that copy / serialize the source into owned bytes and delegate to the kj::Array overload. The + // JsBufferSource overload copies (rather than retaining a reference to the view) so the resulting + // stream is unaffected if the source ArrayBuffer is later detached or transferred. + JsReadableStream(jsg::Lock& js, kj::Array data); + JsReadableStream(jsg::Lock& js, kj::String data); + JsReadableStream(jsg::Lock& js, jsg::JsRef view); + JsReadableStream(jsg::Lock& js, jsg::Ref blob); + JsReadableStream(jsg::Lock& js, jsg::Ref urlSearchParams); + JsReadableStream(jsg::Lock& js, jsg::Ref urlSearchParams); + + JsReadableStream(JsReadableStream&&) = default; + JsReadableStream& operator=(JsReadableStream&&) = default; + JsReadableStream(const JsReadableStream&) = delete; + JsReadableStream& operator=(const JsReadableStream&) = delete; + + // Create a stream-backed (non-rewindable) JsReadableStream wrapping the given native data + // source. This is the canonical way for C++ code to mint a new ReadableStream to hand to + // JavaScript. + // + // TODO(streams-ts): This is the future compatibility-flag dispatch point. Once the + // TypeScript implementation lands, this will construct either the legacy C++ ReadableStream + // or a TypeScript-backed stream depending on the worker's configuration. + static JsReadableStream create( + jsg::Lock& js, IoContext& ioContext, kj::Own source); + + // Returns a new JsReadableStream sharing this one's underlying stream (and retransmit + // buffer, if any). Both instances observe the same underlying stream state (e.g. the stream + // becoming disturbed through one is visible through the other), and passing either through + // the type wrapper yields the same JavaScript object. This is what identity-preserving + // accessors (e.g. request.body === request.body) are built from. addRef() of a null stream + // is a null stream. + JsReadableStream addRef(jsg::Lock& js); + + // True if this is a null / empty stream ("no body"). Inspects only C++-side state; a + // jsg::Lock& is not required because it never dispatches to the JS backend. + bool isNull() const; + + // True if this is backed by an in-memory buffer, and therefore rewindable via tryClone(). + // Inspects only C++-side state; a jsg::Lock& is not required. + bool isBufferBacked() const; + + // True if the stream has been read from / consumed at all. Not const: for the C++ backend this + // dispatches to ReadableStream::isDisturbed(), which is non-const. + bool isDisturbed(jsg::Lock& js); + + // True if the underlying stream is currently locked (to a reader, a pending pump, a tee, + // etc.). A null stream is never locked. Not const for the same reason as isDisturbed(). + bool isLocked(jsg::Lock& js); + + // The known length of the stream for the given encoding, if known; kj::none otherwise. Not const + // for the same reason as isDisturbed(). + kj::Maybe tryGetLength( + jsg::Lock& js, StreamEncoding encoding = StreamEncoding::IDENTITY); + + // Cancel the stream with the given reason, indicating a loss of interest in the data. The + // stream is left disturbed and closed. Rejects if the stream is currently locked, matching + // ReadableStream.prototype.cancel(). Canceling a null stream is a no-op (resolved promise). + jsg::Promise cancel(jsg::Lock& js, jsg::Optional reason); + + // Cancel the stream even if it is currently locked. This is a forcible teardown used when the + // stream's underlying connection is going away regardless of what JavaScript is doing with the + // stream (e.g. a Socket's readable side when the socket is closed or errors). Force-canceling + // a null stream is a no-op (resolved promise). + jsg::Promise forceCancel(jsg::Lock& js, jsg::Optional reason); + + // Mark the stream as being in the process of shutting down (e.g. the Socket it belongs to is + // closing), before the closure has actually completed. A no-op on a null stream. + void setPendingClosure(jsg::Lock& js); + + // Returns a promise that resolves when the stream reads EOF. Used by Socket to detect the + // remote end closing the connection. Must be called at most once per stream. Precondition: + // !isNull(). + jsg::Promise onEof(jsg::Lock& js); + + // Fully consume the stream, buffering up to `limit` bytes, and return all of its data as a single + // value. Rejects if the stream has already been disturbed or would exceed `limit`; the stream is + // left locked/consumed. A null stream yields an empty result. + // + // bytes()/json()/blob() are provided directly (rather than being layered on arrayBuffer()/text() + // by the caller) to avoid an extra jsg::Promise allocation and microtask hop per call. blob() + // takes the desired Content-Type (derived by the caller, e.g. from headers) because + // JsReadableStream has no notion of headers. + jsg::Promise> arrayBuffer(jsg::Lock& js, uint64_t limit); + jsg::Promise text(jsg::Lock& js, uint64_t limit); + jsg::Promise> bytes(jsg::Lock& js, uint64_t limit); + jsg::Promise> json(jsg::Lock& js, uint64_t limit); + jsg::Promise> blob(jsg::Lock& js, uint64_t limit, kj::String contentType); + + // Fully drains the stream into the given sink. If end is EndStream::YES, the sink is closed after + // the stream is fully drained. Deferred proxying is supported transparently if the underlying + // source supports it. Precondition: !isNull(). + kj::Promise> pumpTo( + jsg::Lock& js, kj::Own sink, EndStream end); + + // Split a live stream into two independent branches that each read the same data, consuming + // (nullifying) this. Both branches share the retransmit buffer (if any). Works for any stream. + // Precondition: !isNull(). + Tee tee(jsg::Lock& js); + + // If this is buffer-backed, return a fresh, independent JsReadableStream that reads from the + // start of the buffer (carrying the buffer forward, so the result is itself rewindable). This is + // non-mutating -- `this` is left untouched. Returns kj::none if this is null or not + // buffer-backed. + kj::Maybe tryClone(jsg::Lock& js); + + // Take over the internal stream state (and a reference to the retransmit buffer, if any) into a + // new JsReadableStream, leaving `this` locked/disturbed. Precondition: !isNull(). + // + // Detaching an already-disturbed stream is an error unless IgnoreDisturbed::YES is passed + // (used when neutralizing a stream whose underlying data source is being taken over, e.g. + // when a Socket's connection is adopted by another consumer). + JsReadableStream detach(jsg::Lock& js, IgnoreDisturbed ignoreDisturbed = IgnoreDisturbed::NO); + + // Convert this into a null / empty stream. + void nullify(); + + void visitForGc(jsg::GcVisitor& visitor); + void visitForMemoryInfo(jsg::MemoryTracker& tracker) const; + + // Describe this type to RTTI (and therefore to generated TypeScript) exactly as a + // ReadableStream. See the delegated-RTTI support in jsg/rtti.h. + using JsgRttiDelegate = jsg::Ref; + + static v8::Local jsgWrap(auto& typeWrapper, + jsg::Lock& js, + v8::Local context, + kj::Maybe> creator, + JsReadableStream stream) { + // Wrapping a null JsReadableStream indicates a bug: APIs that can produce "no stream" + // express that as kj::Maybe so that absence maps to JS null/undefined + // rather than to a fabricated stream. + auto& impl = KJ_ASSERT_NONNULL(stream.impl, "cannot wrap a null JsReadableStream"); + KJ_SWITCH_ONEOF(impl.stream) { + KJ_CASE_ONEOF(legacy, jsg::Ref) { + return typeWrapper.wrap(js, context, creator, kj::mv(legacy)); + } + KJ_CASE_ONEOF(ts, jsg::JsRef) { + KJ_UNIMPLEMENTED("TypeScript ReadableStream not yet supported"); + } + } + KJ_UNREACHABLE; + } + + static kj::Maybe jsgTryUnwrap(auto& typeWrapper, + jsg::Lock& js, + v8::Local context, + v8::Local handle, + kj::Maybe> parentObject) { + // For now, we only support unwrapping the legacy C++ ReadableStream. + // Later we will also support the TypeScript implementation. + return typeWrapper.tryUnwrap( + js, context, handle, static_cast*>(nullptr), parentObject); + } + + private: + explicit JsReadableStream(Impl impl): impl(kj::mv(impl)) {} + + // Build a buffer-backed Impl: wraps the buffer's bytes in an in-memory ReadableStream (which does + // NOT support deferred proxying, since the bytes may have V8 heap provenance) and retains the + // buffer for retransmission. + static Impl bufferBackedImpl(jsg::Lock& js, kj::Rc buffer); + + kj::Maybe impl; +}; + +struct JsReadableStream::Tee { + JsReadableStream branch1; + JsReadableStream branch2; +}; + +} // namespace workerd::api diff --git a/src/workerd/api/kv.c++ b/src/workerd/api/kv.c++ index 50167a57584..6906feb515d 100644 --- a/src/workerd/api/kv.c++ +++ b/src/workerd/api/kv.c++ @@ -450,7 +450,7 @@ jsg::Promise KvNamespace::getWithMetadataImp if (typeName == "stream") { result = js.resolvedPromise( - KvNamespace::GetResult(js.alloc(context, kj::mv(stream)))); + KvNamespace::GetResult(JsReadableStream::create(js, context, kj::mv(stream)))); } else if (typeName == "text") { // NOTE: In theory we should be using awaitIoLegacy() here since ReadableStreamSource is // supposed to handle pending events on its own, but we also know that the HTTP client @@ -651,8 +651,8 @@ jsg::Promise KvNamespace::put(jsg::Lock& js, expectedBodySize = static_cast(data.size()); traceContext.setTag("cloudflare.kv.query.value_type"_kjc, "ArrayBuffer"_kjc); } - KJ_CASE_ONEOF(stream, jsg::Ref) { - expectedBodySize = stream->tryGetLength(StreamEncoding::IDENTITY); + KJ_CASE_ONEOF(stream, JsReadableStream) { + expectedBodySize = stream.tryGetLength(js, StreamEncoding::IDENTITY); traceContext.setTag("cloudflare.kv.query.value_type"_kjc, "ReadableStream"_kjc); } } @@ -680,12 +680,12 @@ jsg::Promise KvNamespace::put(jsg::Lock& js, KJ_CASE_ONEOF(data, kj::Array) { writePromise = req.body->write(data).attach(kj::mv(data)); } - KJ_CASE_ONEOF(stream, jsg::Ref) { + KJ_CASE_ONEOF(stream, JsReadableStream) { writePromise = context.run( [dest = newSystemStream(kj::mv(req.body), StreamEncoding::IDENTITY, context), stream = kj::mv(stream)](jsg::Lock& js) mutable { return IoContext::current().waitForDeferredProxy( - stream->pumpTo(js, kj::mv(dest), true)); + stream.pumpTo(js, kj::mv(dest), EndStream::YES)); }); } } diff --git a/src/workerd/api/kv.h b/src/workerd/api/kv.h index 3e0ba58ed2f..b61a339caf9 100644 --- a/src/workerd/api/kv.h +++ b/src/workerd/api/kv.h @@ -4,7 +4,7 @@ #pragma once -#include +#include #include #include #include @@ -57,8 +57,8 @@ class KvNamespace: public jsg::Object { }); }; - using GetResult = kj::Maybe< - kj::OneOf, kj::Array, kj::String, jsg::JsRef>>; + using GetResult = + kj::Maybe, kj::String, jsg::JsRef>>; jsg::Promise getSingle(jsg::Lock& js, IoContext& context, @@ -141,7 +141,7 @@ class KvNamespace: public jsg::Object { // we specifically support later. using PutBody = kj::OneOf; - using PutSupportedTypes = kj::OneOf, jsg::Ref>; + using PutSupportedTypes = kj::OneOf, JsReadableStream>; jsg::Promise put(jsg::Lock& js, kj::String name, diff --git a/src/workerd/api/r2-bucket.c++ b/src/workerd/api/r2-bucket.c++ index 467ac66e55d..85fd1412c0d 100644 --- a/src/workerd/api/r2-bucket.c++ +++ b/src/workerd/api/r2-bucket.c++ @@ -591,10 +591,10 @@ R2Bucket::get(jsg::Lock& js, if (r2Result.preconditionFailed()) { result = KJ_ASSERT_NONNULL(parseObjectMetadata(js, "get", r2Result, errorType)); } else { - jsg::Ref body = nullptr; + JsReadableStream body; KJ_IF_SOME(s, r2Result.stream) { - body = js.alloc(context, kj::mv(s)); + body = JsReadableStream::create(js, context, kj::mv(s)); r2Result.stream = kj::none; } result = parseObjectMetadata(js, "get", r2Result, errorType, kj::mv(body)); @@ -625,8 +625,8 @@ jsg::Promise>> R2Bucket::put(jsg::Lock& auto cancelReader = kj::defer([&] { KJ_IF_SOME(v, value) { KJ_SWITCH_ONEOF(v) { - KJ_CASE_ONEOF(v, jsg::Ref) { - (*v).cancel(js, + KJ_CASE_ONEOF(v, JsReadableStream) { + v.cancel(js, js.error( "Stream cancelled because the associated put operation encountered an error.")); } @@ -850,8 +850,8 @@ jsg::Promise>> R2Bucket::put(jsg::Lock& KJ_IF_SOME(v, value) { KJ_SWITCH_ONEOF(v) { - KJ_CASE_ONEOF(stream, jsg::Ref) { - KJ_IF_SOME(size, stream->tryGetLength(StreamEncoding::IDENTITY)) { + KJ_CASE_ONEOF(stream, JsReadableStream) { + KJ_IF_SOME(size, stream.tryGetLength(js, StreamEncoding::IDENTITY)) { traceContext.setTag("cloudflare.r2.request.size"_kjc, static_cast(size)); } } @@ -875,8 +875,8 @@ jsg::Promise>> R2Bucket::put(jsg::Lock& kj::StringPtr components[1]; auto path = fillR2Path(components, adminBucket); auto client = getHttpClient(context, traceContext); - auto promise = - doR2HTTPPutRequest(kj::mv(client), kj::mv(value), kj::none, kj::mv(requestJson), path, jwt); + auto promise = doR2HTTPPutRequest( + js, kj::mv(client), kj::mv(value), kj::none, kj::mv(requestJson), path, jwt); return context.awaitIo(js, kj::mv(promise), [sentHttpMetadata = kj::mv(sentHttpMetadata), @@ -997,7 +997,7 @@ jsg::Promise> R2Bucket::createMultipartUpload(jsg::L auto path = fillR2Path(components, adminBucket); auto client = getHttpClient(context, traceContext); auto promise = - doR2HTTPPutRequest(kj::mv(client), kj::none, kj::none, kj::mv(requestJson), path, jwt); + doR2HTTPPutRequest(js, kj::mv(client), kj::none, kj::none, kj::mv(requestJson), path, jwt); return context.awaitIo(js, kj::mv(promise), [&errorType, key = kj::mv(key), this, traceContext = kj::mv(traceContext)]( @@ -1070,7 +1070,7 @@ jsg::Promise R2Bucket::delete_(jsg::Lock& js, auto path = fillR2Path(components, adminBucket); auto client = getHttpClient(context, traceContext); auto promise = - doR2HTTPPutRequest(kj::mv(client), kj::none, kj::none, kj::mv(requestJson), path, jwt); + doR2HTTPPutRequest(js, kj::mv(client), kj::none, kj::none, kj::mv(requestJson), path, jwt); return context.awaitIo(js, kj::mv(promise), [&errorType, traceContext = kj::mv(traceContext)]( @@ -1424,35 +1424,24 @@ void R2Bucket::HeadResult::writeHttpMetadata(jsg::Lock& js, Headers& headers) { jsg::Promise> R2Bucket::GetResult::arrayBuffer(jsg::Lock& js) { return js.evalNow([&] { - JSG_REQUIRE(!body->isDisturbed(), TypeError, - "Body has already been used. " - "It can only be used once. Use tee() first if you need to read it twice."); - auto& context = IoContext::current(); - return body->getController().readAllBytes(js, context.getLimitEnforcer().getBufferingLimit()); + return body.arrayBuffer(js, context.getLimitEnforcer().getBufferingLimit()); }); } jsg::Promise> R2Bucket::GetResult::bytes(jsg::Lock& js) { return js.evalNow([&] { - JSG_REQUIRE(!body->isDisturbed(), TypeError, - "Body has already been used. " - "It can only be used once. Use tee() first if you need to read it twice."); - auto& context = IoContext::current(); - return body->getController() - .readAllBytes(js, context.getLimitEnforcer().getBufferingLimit()) - .then(js, [](jsg::Lock& js, jsg::JsRef data) { - auto handle = data.getHandle(js); - return jsg::JsUint8Array::create(js, handle).addRef(js); - }); + return body.bytes(js, context.getLimitEnforcer().getBufferingLimit()); }); } jsg::Promise R2Bucket::GetResult::text(jsg::Lock& js) { // Copy-pasted from http.c++ return js.evalNow([&] { - JSG_REQUIRE(!body->isDisturbed(), TypeError, + // Check for a disturbed body before emitting the non-text warning below. (body.text() + // performs the same check with the same error message; this one just runs first.) + JSG_REQUIRE(!body.isDisturbed(js), TypeError, "Body has already been used. " "It can only be used once. Use tee() first if you need to read it twice."); @@ -1467,7 +1456,7 @@ jsg::Promise R2Bucket::GetResult::text(jsg::Lock& js) { } } - return body->getController().readAllText(js, context.getLimitEnforcer().getBufferingLimit()); + return body.text(js, context.getLimitEnforcer().getBufferingLimit()); }); } diff --git a/src/workerd/api/r2-bucket.h b/src/workerd/api/r2-bucket.h index 04609223b68..a57770965f9 100644 --- a/src/workerd/api/r2-bucket.h +++ b/src/workerd/api/r2-bucket.h @@ -377,7 +377,7 @@ class R2Bucket: public jsg::Object { jsg::Optional range, kj::String storageClass, jsg::Optional ssecKeyMd5, - jsg::Ref body) + JsReadableStream body) : HeadResult(kj::mv(name), kj::mv(version), size, @@ -391,12 +391,12 @@ class R2Bucket: public jsg::Object { kj::mv(ssecKeyMd5)), body(kj::mv(body)) {} - jsg::Ref getBody() { - return body.addRef(); + JsReadableStream getBody(jsg::Lock& js) { + return body.addRef(js); } - bool getBodyUsed() { - return body->isDisturbed(); + bool getBodyUsed(jsg::Lock& js) { + return body.isDisturbed(js); } jsg::Promise> arrayBuffer(jsg::Lock& js); @@ -422,11 +422,11 @@ class R2Bucket: public jsg::Object { } void visitForMemoryInfo(jsg::MemoryTracker& tracker) const { - tracker.trackField("body", body); + body.visitForMemoryInfo(tracker); } private: - jsg::Ref body; + JsReadableStream body; void visitForGc(jsg::GcVisitor& visitor) { visitor.visit(body); diff --git a/src/workerd/api/r2-multipart.c++ b/src/workerd/api/r2-multipart.c++ index fbdf46d02ce..425e9b3dd37 100644 --- a/src/workerd/api/r2-multipart.c++ +++ b/src/workerd/api/r2-multipart.c++ @@ -91,8 +91,8 @@ jsg::Promise R2MultipartUpload::uploadPart(jsg: kj::Maybe requestSize = kj::none; KJ_SWITCH_ONEOF(value) { - KJ_CASE_ONEOF(stream, jsg::Ref) { - KJ_IF_SOME(size, stream->tryGetLength(StreamEncoding::IDENTITY)) { + KJ_CASE_ONEOF(stream, JsReadableStream) { + KJ_IF_SOME(size, stream.tryGetLength(js, StreamEncoding::IDENTITY)) { requestSize = size; } } @@ -117,7 +117,7 @@ jsg::Promise R2MultipartUpload::uploadPart(jsg: auto path = fillR2Path(components, this->bucket->adminBucket); auto client = this->bucket->getHttpClient(context, traceContext); auto promise = doR2HTTPPutRequest( - kj::mv(client), kj::mv(value), kj::none, kj::mv(requestJson), path, kj::none); + js, kj::mv(client), kj::mv(value), kj::none, kj::mv(requestJson), path, kj::none); return context.awaitIo(js, kj::mv(promise), [&errorType, partNumber, traceContext = kj::mv(traceContext)]( @@ -188,8 +188,8 @@ jsg::Promise> R2MultipartUpload::complete(jsg::Lo kj::StringPtr components[1]; auto path = fillR2Path(components, this->bucket->adminBucket); auto client = this->bucket->getHttpClient(context, traceContext); - auto promise = - doR2HTTPPutRequest(kj::mv(client), kj::none, kj::none, kj::mv(requestJson), path, kj::none); + auto promise = doR2HTTPPutRequest( + js, kj::mv(client), kj::none, kj::none, kj::mv(requestJson), path, kj::none); return context.awaitIo(js, kj::mv(promise), [&errorType, traceContext = kj::mv(traceContext)]( @@ -241,8 +241,8 @@ jsg::Promise R2MultipartUpload::abort( kj::StringPtr components[1]; auto path = fillR2Path(components, this->bucket->adminBucket); auto client = this->bucket->getHttpClient(context, traceContext); - auto promise = - doR2HTTPPutRequest(kj::mv(client), kj::none, kj::none, kj::mv(requestJson), path, kj::none); + auto promise = doR2HTTPPutRequest( + js, kj::mv(client), kj::none, kj::none, kj::mv(requestJson), path, kj::none); return context.awaitIo(js, kj::mv(promise), [&errorType, traceContext = kj::mv(traceContext)]( diff --git a/src/workerd/api/r2-rpc.c++ b/src/workerd/api/r2-rpc.c++ index f98b0f75124..cf775556126 100644 --- a/src/workerd/api/r2-rpc.c++ +++ b/src/workerd/api/r2-rpc.c++ @@ -154,60 +154,27 @@ kj::Promise doR2HTTPGetRequest(kj::Own client, } } -kj::Promise doR2HTTPPutRequest(kj::Own client, +namespace { +// The coroutine half of doR2HTTPPutRequest(). Split out of the public function because +// computing the expected body size requires a jsg::Lock, and a jsg::Lock must never be +// captured in a KJ coroutine frame. Everything up to the first co_await runs synchronously +// in the caller's context (in particular, `path` is consumed before any suspension). +kj::Promise doR2HTTPPutRequestImpl(kj::Own client, kj::Maybe supportedBody, - kj::Maybe streamSize, + uint64_t expectedBodySize, kj::String metadataPayload, kj::ArrayPtr path, kj::Maybe jwt) { - // NOTE: A lot of code here is duplicated with kv.c++. Maybe it can be refactored to be more - // reusable? auto& context = IoContext::current(); auto headers = kj::HttpHeaders(context.getHeaderTable()); auto url = getFakeUrl(path); - kj::Maybe expectedBodySize; - - KJ_IF_SOME(b, supportedBody) { - KJ_SWITCH_ONEOF(b) { - KJ_CASE_ONEOF(stream, jsg::Ref) { - expectedBodySize = stream->tryGetLength(StreamEncoding::IDENTITY); - if (expectedBodySize == kj::none) { - expectedBodySize = streamSize; - } - JSG_REQUIRE(expectedBodySize != kj::none, TypeError, - "Provided readable stream must have a known length (request/response body or readable " - "half of FixedLengthStream)"); - JSG_REQUIRE(streamSize.orDefault(KJ_ASSERT_NONNULL(expectedBodySize)) == expectedBodySize, - RangeError, "Provided stream length (", streamSize.orDefault(-1), - ") doesn't match what " - "the stream reports (", - KJ_ASSERT_NONNULL(expectedBodySize), ")"); - } - KJ_CASE_ONEOF(text, jsg::NonCoercible) { - expectedBodySize = text.value.size(); - KJ_REQUIRE(streamSize == kj::none); - } - KJ_CASE_ONEOF(data, kj::Array) { - expectedBodySize = data.size(); - KJ_REQUIRE(streamSize == kj::none); - } - KJ_CASE_ONEOF(data, jsg::Ref) { - expectedBodySize = data->getSize(); - KJ_REQUIRE(streamSize == kj::none); - } - } - } else { - expectedBodySize = static_cast(0); - KJ_REQUIRE(streamSize == kj::none); - } - headers.set(context.getHeaderIds().cfBlobMetadataSize, kj::str(metadataPayload.size())); KJ_IF_SOME(j, jwt) { headers.set(context.getHeaderIds().authorization, kj::str("Bearer ", j)); } - uint64_t combinedSize = metadataPayload.size() + KJ_ASSERT_NONNULL(expectedBodySize); + uint64_t combinedSize = metadataPayload.size() + expectedBodySize; co_await context.waitForOutputLocks(); @@ -227,13 +194,14 @@ kj::Promise doR2HTTPPutRequest(kj::Own client, auto data = blob->getData(); co_await request.body->write(data); } - KJ_CASE_ONEOF(stream, jsg::Ref) { + KJ_CASE_ONEOF(stream, JsReadableStream) { // Because the ReadableStream might be a fully JavaScript-backed stream, we must // start running the pump within the IoContext/isolate lock. co_await context.run( [dest = newSystemStream(kj::mv(request.body), StreamEncoding::IDENTITY, context), stream = kj::mv(stream)](jsg::Lock& js) mutable { - return IoContext::current().waitForDeferredProxy(stream->pumpTo(js, kj::mv(dest), true)); + return IoContext::current().waitForDeferredProxy( + stream.pumpTo(js, kj::mv(dest), EndStream::YES)); }); } } @@ -266,4 +234,54 @@ kj::Promise doR2HTTPPutRequest(kj::Own client, .metadataPayload = responseBody.releaseArray(), }; } +} // namespace + +kj::Promise doR2HTTPPutRequest(jsg::Lock& js, + kj::Own client, + kj::Maybe supportedBody, + kj::Maybe streamSize, + kj::String metadataPayload, + kj::ArrayPtr path, + kj::Maybe jwt) { + // NOTE: A lot of code here is duplicated with kv.c++. Maybe it can be refactored to be more + // reusable? + kj::Maybe expectedBodySize; + + KJ_IF_SOME(b, supportedBody) { + KJ_SWITCH_ONEOF(b) { + KJ_CASE_ONEOF(stream, JsReadableStream) { + expectedBodySize = stream.tryGetLength(js, StreamEncoding::IDENTITY); + if (expectedBodySize == kj::none) { + expectedBodySize = streamSize; + } + JSG_REQUIRE(expectedBodySize != kj::none, TypeError, + "Provided readable stream must have a known length (request/response body or readable " + "half of FixedLengthStream)"); + JSG_REQUIRE(streamSize.orDefault(KJ_ASSERT_NONNULL(expectedBodySize)) == expectedBodySize, + RangeError, "Provided stream length (", streamSize.orDefault(-1), + ") doesn't match what " + "the stream reports (", + KJ_ASSERT_NONNULL(expectedBodySize), ")"); + } + KJ_CASE_ONEOF(text, jsg::NonCoercible) { + expectedBodySize = text.value.size(); + KJ_REQUIRE(streamSize == kj::none); + } + KJ_CASE_ONEOF(data, kj::Array) { + expectedBodySize = data.size(); + KJ_REQUIRE(streamSize == kj::none); + } + KJ_CASE_ONEOF(data, jsg::Ref) { + expectedBodySize = data->getSize(); + KJ_REQUIRE(streamSize == kj::none); + } + } + } else { + expectedBodySize = static_cast(0); + KJ_REQUIRE(streamSize == kj::none); + } + + return doR2HTTPPutRequestImpl(kj::mv(client), kj::mv(supportedBody), + KJ_ASSERT_NONNULL(expectedBodySize), kj::mv(metadataPayload), path, kj::mv(jwt)); +} } // namespace workerd::api diff --git a/src/workerd/api/r2-rpc.h b/src/workerd/api/r2-rpc.h index 288a248d93b..1b6f16b459b 100644 --- a/src/workerd/api/r2-rpc.h +++ b/src/workerd/api/r2-rpc.h @@ -5,6 +5,7 @@ #pragma once #include +#include #include namespace kj { @@ -14,7 +15,6 @@ class HttpClient; namespace workerd::api { class ReadableStreamSource; -class ReadableStream; // NOTE: We don't currently actually use this as a structured object (hence the `kj::Own` // that we see pop up). @@ -64,10 +64,8 @@ class R2Error: public jsg::Object { friend struct R2Result; }; -using R2PutValue = kj::OneOf, - kj::Array, - jsg::NonCoercible, - jsg::Ref>; +using R2PutValue = + kj::OneOf, jsg::NonCoercible, jsg::Ref>; struct R2Result { uint httpStatus; @@ -102,7 +100,10 @@ kj::Promise doR2HTTPGetRequest(kj::Own client, kj::Maybe jwt, CompatibilityFlags::Reader flags); -kj::Promise doR2HTTPPutRequest(kj::Own client, +// Note: takes a jsg::Lock because computing the expected body size of a JsReadableStream value +// requires it. The lock is used synchronously (before any I/O) and is not retained. +kj::Promise doR2HTTPPutRequest(jsg::Lock& js, + kj::Own client, kj::Maybe value, kj::Maybe streamSize, // Deprecated. For internal beta API only. diff --git a/src/workerd/api/sockets.c++ b/src/workerd/api/sockets.c++ index 3fb4cd069ea..80dfda7b0f2 100644 --- a/src/workerd/api/sockets.c++ +++ b/src/workerd/api/sockets.c++ @@ -229,11 +229,11 @@ jsg::Ref setupSocket(jsg::Lock& js, kj::Rc refcountedConnection(kj::mv(connection)); // Initialize the readable/writable streams with the readable/writable sides of an AsyncIoStream. auto sysStreams = newSystemMultiStream(refcountedConnection.addRef(), ioContext); - auto readable = js.alloc(ioContext, kj::mv(sysStreams.readable)); + auto readable = JsReadableStream::create(js, ioContext, kj::mv(sysStreams.readable)); auto allowHalfOpen = getAllowHalfOpen(options); kj::Maybe> eofPromise; if (!allowHalfOpen) { - eofPromise = readable->onEof(js); + eofPromise = readable.onEof(js); } auto openedPrPair = kj::mv(maybeOpenedPrPair).orDefault([&js]() { return js.newPromiseAndResolver(); @@ -373,7 +373,7 @@ jsg::Promise Socket::close(jsg::Lock& js) { isClosing = true; writable->getController().setPendingClosure(); - readable->getController().setPendingClosure(); + readable.setPendingClosure(js); // Wait until the socket connects (successfully or otherwise) // Note: `self` (jsg::Ref) is captured in each continuation to prevent GC from collecting @@ -391,7 +391,7 @@ jsg::Promise Socket::close(jsg::Lock& js) { .then(js, [self = JSG_THIS](jsg::Lock& js) mutable { // Forcibly abort the readable/writable streams. - auto cancelPromise = self->readable->getController().cancel(js, kj::none); + auto cancelPromise = self->readable.forceCancel(js, kj::none); auto abortPromise = self->writable->getController().abort(js, kj::none); // The below is effectively `Promise.all(cancelPromise, abortPromise)` @@ -460,7 +460,7 @@ jsg::Ref Socket::startTls(jsg::Lock& js, jsg::Optional tlsOp auto& context = IoContext::current(); self->writable->detach(js); - self->readable->detach(js, true); + self->readable.detach(js, IgnoreDisturbed::YES); // We should set this before closedResolver.resolve() in order to give the user // the option to check if the closed promise is resolved due to upgrade or not. @@ -604,7 +604,7 @@ void Socket::handleProxyStatus(jsg::Lock& js, kj::PromisegetController().cancel(js, kj::none).markAsHandled(js); + readable.forceCancel(js, kj::none).markAsHandled(js); writable->getController().abort(js, js.error(e.getDescription())).markAsHandled(js); } @@ -666,7 +666,7 @@ kj::Own Socket::takeConnectionStream(jsg::Lock& js) { // We do not care if the socket was disturbed, we require the user to ensure the socket is not // being used. writable->detach(js); - readable->detach(js, true); + readable.detach(js, IgnoreDisturbed::YES); // Move the stream out of the socket, to ensure the stream is properly destroyed when the // caller is done with it. diff --git a/src/workerd/api/sockets.h b/src/workerd/api/sockets.h index 37b8cf3f450..a4166e39511 100644 --- a/src/workerd/api/sockets.h +++ b/src/workerd/api/sockets.h @@ -4,7 +4,7 @@ #pragma once -#include +#include #include #include #include @@ -65,7 +65,7 @@ class Socket: public jsg::Object { kj::Rc connectionStream, kj::Maybe remoteAddress, kj::Maybe localAddress, - jsg::Ref readableParam, + JsReadableStream readableParam, jsg::Ref writable, jsg::PromiseResolverPair closedPrPair, kj::Promise watchForDisconnectTask, @@ -92,8 +92,8 @@ class Socket: public jsg::Object { openedPromiseCopy(openedPrPair.promise.whenResolved(js)), openedPromise(kj::mv(openedPrPair.promise)) {}; - jsg::Ref getReadable() { - return readable.addRef(); + JsReadableStream getReadable(jsg::Lock& js) { + return readable.addRef(js); } jsg::Ref getWritable() { return writable.addRef(); @@ -168,7 +168,7 @@ class Socket: public jsg::Object { void visitForMemoryInfo(jsg::MemoryTracker& tracker) const { tracker.trackFieldWithSize("connectionData", sizeof(IoOwn)); - tracker.trackField("readable", readable); + readable.visitForMemoryInfo(tracker); tracker.trackField("writable", writable); tracker.trackField("closedResolver", closedResolver); tracker.trackField("closedPromiseCopy", closedPromiseCopy); @@ -196,7 +196,7 @@ class Socket: public jsg::Object { }; kj::Maybe> connectionData; - jsg::Ref readable; + JsReadableStream readable; jsg::Ref writable; // This fulfiller is used to resolve the `closedPromise` below. jsg::Promise::Resolver closedResolver; diff --git a/src/workerd/api/streams.h b/src/workerd/api/streams.h index 59b8f131809..21f284ddda0 100644 --- a/src/workerd/api/streams.h +++ b/src/workerd/api/streams.h @@ -7,6 +7,7 @@ // // This is the most over-engineered spec... +#include #include #include #include diff --git a/src/workerd/api/streams/AGENTS.md b/src/workerd/api/streams/AGENTS.md index a4c30a4e7ff..8820fb88469 100644 --- a/src/workerd/api/streams/AGENTS.md +++ b/src/workerd/api/streams/AGENTS.md @@ -6,6 +6,11 @@ See `docs/streams.md` for narrative tutorial. ## ARCHITECTURE +NOTE: C++ code outside this directory does not use `jsg::Ref` directly; it goes +through the `JsReadableStream` abstraction in `src/workerd/api/js-readable-stream.{h,c++}`, which +hides which ReadableStream implementation backs a given stream. New C++ consumers of readable +streams should use that abstraction, not the types defined here. + Dual implementation behind unified API: - **Internal** (`internal.h`): kj-backed, byte-only, single pending read, no JS queue diff --git a/src/workerd/jsg/README.md b/src/workerd/jsg/README.md index 92e7f6d5f7d..f411429af6c 100644 --- a/src/workerd/jsg/README.md +++ b/src/workerd/jsg/README.md @@ -64,7 +64,7 @@ For file map and coding invariants, see [AGENTS.md](AGENTS.md). | `jsg::Identified` | Any | Captures JS object identity + unwrapped value | | `jsg::NonCoercible` | Exact type | No automatic coercion; `T` = `kj::String`, `bool`, `double` | | `jsg::LenientOptional` | Optional | Silently ignores type errors → `undefined` | -| `SelfConvertible` types | Custom | Types defining static `jsgWrap()`/`jsgTryUnwrap()`; no registration needed | +| `SelfConvertible` types | Custom | Types defining static `jsgWrap()`/`jsgTryUnwrap()`; no registration needed. Declare `using JsgRttiDelegate = ...` to describe the type to RTTI (TypeScript generation) as an existing type | ## JsValue Type Mapping diff --git a/src/workerd/jsg/rtti.h b/src/workerd/jsg/rtti.h index 1e726982fe9..b5b0f7fd9d8 100644 --- a/src/workerd/jsg/rtti.h +++ b/src/workerd/jsg/rtti.h @@ -126,6 +126,26 @@ struct TupleRttiBuilder { } }; +// Delegated RTTI +// +// A C++ type that converts to/from an existing JavaScript type (typically a SelfConvertible +// type, see type-wrapper.h) can opt into RTTI by declaring a delegate: +// +// class MyType { +// public: +// using JsgRttiDelegate = jsg::Ref; +// ... +// }; +// +// RTTI (and therefore generated TypeScript) will then describe MyType exactly as it describes +// the delegate type. +template +struct BuildRtti> { + static void build(Type::Builder builder, Builder& rtti) { + BuildRtti::build(builder, rtti); + } +}; + // Primitives template diff --git a/src/workerd/jsg/type-wrapper.h b/src/workerd/jsg/type-wrapper.h index cdcc23e8d39..582d8330d0a 100644 --- a/src/workerd/jsg/type-wrapper.h +++ b/src/workerd/jsg/type-wrapper.h @@ -229,6 +229,11 @@ class UnimplementedWrapper { // v8::Local handle, // kj::Maybe> parentObject); // }; +// +// If a SelfConvertible type appears in JSG-visible signatures (method parameters or return +// types, JSG_STRUCT fields, etc.), it also needs an RTTI representation for TypeScript type +// generation. Declare `using JsgRttiDelegate = ...;` to describe the type to RTTI as some +// existing type; see the delegated-RTTI support in rtti.h. template concept SelfConvertible = requires(Lock& js, v8::Local ctx, diff --git a/src/workerd/tests/bench-response.c++ b/src/workerd/tests/bench-response.c++ index 1ab06096e34..b821ea61f41 100644 --- a/src/workerd/tests/bench-response.c++ +++ b/src/workerd/tests/bench-response.c++ @@ -81,7 +81,7 @@ BENCHMARK_F(Response, arrayBufferBody)(benchmark::State& state) { auto view = bytes.asPtr(); auto rs = api::streams::newMemorySource(view, kj::heap(kj::mv(bytes))); auto body = - api::Body::Initializer(env.js.alloc(env.context, kj::mv(rs))); + api::Body::Initializer(api::JsReadableStream::create(js, env.context, kj::mv(rs))); benchmark::DoNotOptimize(api::Response::constructor(js, kj::mv(body), kj::none)); } }); From 22750983958a234190103e1d12ec8f32fc500cef Mon Sep 17 00:00:00 2001 From: Erik Corry Date: Tue, 7 Jul 2026 14:27:15 +0000 Subject: [PATCH 037/101] Use non-sandbox buffer to read in kv.c++ * Use non-sandbox buffer to read in kv.c++ In case the sandbox array buffers are under MPK protection we provide a temporary buffer to read from in the put operation. See merge request cloudflare/ew/workerd!422 See merge request cloudflare/ew/workerd!431 --- src/workerd/api/kv.c++ | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/workerd/api/kv.c++ b/src/workerd/api/kv.c++ index 50167a57584..aa829eb35e3 100644 --- a/src/workerd/api/kv.c++ +++ b/src/workerd/api/kv.c++ @@ -648,6 +648,11 @@ jsg::Promise KvNamespace::put(jsg::Lock& js, traceContext.setTag("cloudflare.kv.query.value_type"_kjc, "text"_kjc); } KJ_CASE_ONEOF(data, kj::Array) { + // `data` aliases the V8 BackingStore via jsg::asBytes(). The async + // write() below runs after context.waitForOutputLocks() has released + // the isolate lock, relinquishing any MPK keys. Copy the data while + // we have the key. + data = kj::heapArray(data.asPtr()); expectedBodySize = static_cast(data.size()); traceContext.setTag("cloudflare.kv.query.value_type"_kjc, "ArrayBuffer"_kjc); } From abfddc0d7a9ab47b63adbf71e56dce2aae347951 Mon Sep 17 00:00:00 2001 From: James M Snell Date: Tue, 7 Jul 2026 08:28:40 -0700 Subject: [PATCH 038/101] Improve spec compliance and run wpt's for ts-implemented encoding streams --- src/per_isolate/webstreams/encoding.ts | 63 ++++++--- src/wpt/BUILD.bazel | 12 ++ src/wpt/encoding-test-ts.ts | 177 +++++++++++++++++++++++++ 3 files changed, 237 insertions(+), 15 deletions(-) create mode 100644 src/wpt/encoding-test-ts.ts diff --git a/src/per_isolate/webstreams/encoding.ts b/src/per_isolate/webstreams/encoding.ts index cc9955c2da2..8aa5d0b9d78 100644 --- a/src/per_isolate/webstreams/encoding.ts +++ b/src/per_isolate/webstreams/encoding.ts @@ -21,6 +21,7 @@ const { ObjectGetOwnPropertyDescriptor, StringPrototypeCharCodeAt, StringPrototypeSlice, + SymbolToStringTag, textDecoderEncodingGet, TextDecoder, TextEncoder, @@ -57,6 +58,9 @@ const transformStreamWritableGet = getProtoGetter< (stream: object) => WritableStreamType >(TransformStream.prototype, 'writable'); +let assertIsTextEncoderStream: (self: TextEncoderStream) => void; +let assertIsTextDecoderStream: (self: TextDecoderStream) => void; + // --------------------------------------------------------------------------- // TextEncoderStream // @@ -71,6 +75,12 @@ class TextEncoderStream { #transform: InstanceType; #pendingHighSurrogate: string = ''; + static { + assertIsTextEncoderStream = function (self: TextEncoderStream) { + if (!(#transform in self)) throw new TypeError('Illegal invocation'); + }; + } + constructor() { const self = this; const encoder = new TextEncoder(); @@ -114,18 +124,19 @@ class TextEncoderStream { } get encoding(): string { + assertIsTextEncoderStream(this); return 'utf-8'; } get readable(): ReadableStreamType { - if (!(#transform in this)) throw new TypeError('Illegal invocation'); + assertIsTextEncoderStream(this); return transformStreamReadableGet( this.#transform ) as ReadableStreamType; } get writable(): WritableStreamType { - if (!(#transform in this)) throw new TypeError('Illegal invocation'); + assertIsTextEncoderStream(this); return transformStreamWritableGet( this.#transform ) as WritableStreamType; @@ -150,6 +161,12 @@ class TextDecoderStream { #transform: InstanceType; #decoder: TextDecoder; + static { + assertIsTextDecoderStream = function (self: TextDecoderStream) { + if (!(#decoder in self)) throw new TypeError('Illegal invocation'); + }; + } + constructor(label: string = 'utf-8', options?: TextDecoderStreamOptions) { const decoder = new TextDecoder(label, options); this.#decoder = decoder; @@ -180,47 +197,63 @@ class TextDecoderStream { } get encoding(): string { - if (!(#decoder in this)) throw new TypeError('Illegal invocation'); + assertIsTextDecoderStream(this); return textDecoderEncodingGet(this.#decoder); } get fatal(): boolean { - if (!(#decoder in this)) throw new TypeError('Illegal invocation'); + assertIsTextDecoderStream(this); return textDecoderFatalGet(this.#decoder); } get ignoreBOM(): boolean { - if (!(#decoder in this)) throw new TypeError('Illegal invocation'); + assertIsTextDecoderStream(this); return textDecoderIgnoreBOMGet(this.#decoder); } get readable(): ReadableStreamType { - if (!(#transform in this)) throw new TypeError('Illegal invocation'); + assertIsTextDecoderStream(this); return transformStreamReadableGet( this.#transform ) as ReadableStreamType; } get writable(): WritableStreamType { - if (!(#transform in this)) throw new TypeError('Illegal invocation'); + assertIsTextDecoderStream(this); return transformStreamWritableGet( this.#transform ) as WritableStreamType; } } +const kEnumerable = { __proto__: null, enumerable: true }; + ObjectDefineProperties(TextEncoderStream.prototype, { - encoding: { enumerable: true }, - readable: { enumerable: true }, - writable: { enumerable: true }, + encoding: kEnumerable, + readable: kEnumerable, + writable: kEnumerable, + [SymbolToStringTag]: { + __proto__: null, + value: 'TextEncoderStream', + writable: false, + enumerable: false, + configurable: true, + }, }); ObjectDefineProperties(TextDecoderStream.prototype, { - encoding: { enumerable: true }, - fatal: { enumerable: true }, - ignoreBOM: { enumerable: true }, - readable: { enumerable: true }, - writable: { enumerable: true }, + encoding: kEnumerable, + fatal: kEnumerable, + ignoreBOM: kEnumerable, + readable: kEnumerable, + writable: kEnumerable, + [SymbolToStringTag]: { + __proto__: null, + value: 'TextDecoderStream', + writable: false, + enumerable: false, + configurable: true, + }, }); module.exports = { diff --git a/src/wpt/BUILD.bazel b/src/wpt/BUILD.bazel index c7a0dfd1637..71513028c0e 100644 --- a/src/wpt/BUILD.bazel +++ b/src/wpt/BUILD.bazel @@ -125,6 +125,18 @@ wpt_test( wpt_directory = "@wpt//:streams@module", ) +wpt_test( + name = "encoding-ts", + size = "large", + autogates = ["workerd-autogate-per-isolate-javascript-bootstrap"], + compat_flags = [ + "typescript_implemented_streams", + "text_decoder_cjk_decoder", + ], + config = "encoding-test-ts.ts", + wpt_directory = "@wpt//:encoding@module", +) + wpt_test( name = "fs", size = "large", diff --git a/src/wpt/encoding-test-ts.ts b/src/wpt/encoding-test-ts.ts new file mode 100644 index 00000000000..fd1f04a26b2 --- /dev/null +++ b/src/wpt/encoding-test-ts.ts @@ -0,0 +1,177 @@ +// Copyright (c) 2017-2022 Cloudflare, Inc. +// Licensed under the Apache 2.0 license found in the LICENSE file or at: +// https://opensource.org/licenses/Apache-2.0 + +import path from 'node:path'; +import { getBindingPath } from 'harness/common'; +import { type TestRunnerConfig } from 'harness/harness'; + +function loadWptResource(relativePath: string): void { + const bindingPath = getBindingPath( + path.dirname(globalThis.state.testFileName), + relativePath + ); + const code = globalThis.state.env[bindingPath]; + if (typeof code !== 'string') { + throw new Error( + `Test file ${bindingPath} not found. Update wpt_test.bzl to handle this case.` + ); + } + globalThis.state.env.unsafe.eval(code); +} + +export default { + 'api-basics.any.js': {}, + 'api-invalid-label.any.js': {}, + 'api-replacement-encodings.any.js': {}, + 'api-surrogates-utf8.any.js': {}, + 'encodeInto.any.js': { + comment: 'Requires MessageChannel.postMessage transfer list support', + expectedFailures: ['encodeInto() and a detached output buffer'], + }, + 'idlharness.any.js': {}, + 'iso-2022-jp-decoder.any.js': {}, + 'legacy-mb-japanese/euc-jp/eucjp-decoder.js': {}, + 'legacy-mb-japanese/euc-jp/eucjp-encoder.js': { + before: (): void => { + loadWptResource('./jis0208_index.js'); + loadWptResource('./jis0212_index.js'); + }, + }, + 'legacy-mb-japanese/euc-jp/jis0208_index.js': {}, + 'legacy-mb-japanese/euc-jp/jis0212_index.js': {}, + 'legacy-mb-japanese/iso-2022-jp/iso2022jp-decoder.js': {}, + 'legacy-mb-japanese/iso-2022-jp/iso2022jp-encoder.js': { + before: (): void => { + loadWptResource('./jis0208_index.js'); + }, + }, + 'legacy-mb-japanese/iso-2022-jp/jis0208_index.js': {}, + 'legacy-mb-japanese/shift_jis/jis0208_index.js': {}, + 'legacy-mb-japanese/shift_jis/sjis-decoder.js': {}, + 'legacy-mb-japanese/shift_jis/sjis-encoder.js': { + before: (): void => { + loadWptResource('./jis0208_index.js'); + }, + }, + 'legacy-mb-korean/euc-kr/euckr-decoder.js': {}, + 'legacy-mb-korean/euc-kr/euckr-encoder.js': { + before: (): void => { + loadWptResource('./euckr_index.js'); + }, + }, + 'legacy-mb-korean/euc-kr/euckr_index.js': {}, + 'legacy-mb-schinese/gb18030/gb18030-decoder.any.js': {}, + 'legacy-mb-schinese/gbk/gbk-decoder.any.js': {}, + 'legacy-mb-tchinese/big5/big5-decoder.js': {}, + 'legacy-mb-tchinese/big5/big5-encoder.js': { + before: (): void => { + loadWptResource('./big5_index.js'); + }, + }, + 'legacy-mb-tchinese/big5/big5_index.js': {}, + 'replacement-encodings.any.js': { + comment: 'XMLHttpRequest is not defined', + disabledTests: [ + 'csiso2022kr - non-empty input decodes to one replacement character.', + 'csiso2022kr - empty input decodes to empty output.', + 'hz-gb-2312 - non-empty input decodes to one replacement character.', + 'hz-gb-2312 - empty input decodes to empty output.', + 'iso-2022-cn - non-empty input decodes to one replacement character.', + 'iso-2022-cn - empty input decodes to empty output.', + 'iso-2022-cn-ext - non-empty input decodes to one replacement character.', + 'iso-2022-cn-ext - empty input decodes to empty output.', + 'iso-2022-kr - non-empty input decodes to one replacement character.', + 'iso-2022-kr - empty input decodes to empty output.', + 'replacement - non-empty input decodes to one replacement character.', + 'replacement - empty input decodes to empty output.', + ], + }, + 'single-byte-decoder.window.js': { + comment: 'document and XMLHttpRequest are not supported', + disabledTests: true, + }, + 'streams/backpressure.any.js': {}, + 'streams/decode-attributes.any.js': {}, + 'streams/decode-bad-chunks.any.js': {}, + 'streams/decode-ignore-bom.any.js': {}, + 'streams/decode-incomplete-input.any.js': {}, + 'streams/decode-non-utf8.any.js': {}, + 'streams/decode-split-character.any.js': {}, + 'streams/decode-utf8.any.js': { + comment: 'MessageChannel transfer list is not supported', + disabledTests: [ + 'decoding a transferred Uint8Array chunk should give no output', + 'decoding a transferred ArrayBuffer chunk should give no output', + ], + }, + 'streams/encode-bad-chunks.any.js': {}, + 'streams/encode-utf8.any.js': {}, + 'streams/invalid-realm.window.js': { + comment: 'Enable when ShadowRealm is supported', + expectedFailures: [ + 'TextDecoderStream: write in detached realm should succeed', + 'TextEncoderStream: write in detached realm should succeed', + 'TextEncoderStream: close in detached realm should succeed', + 'TextDecoderStream: close in detached realm should succeed', + ], + }, + 'streams/readable-writable-properties.any.js': {}, + 'streams/realms.window.js': { + comment: 'ReferenceError: window is not defined', + omittedTests: true, + }, + 'textdecoder-arguments.any.js': { + comment: + 'TextDecoder does not detach the underlying ArrayBuffer during options argument conversion', + disabledTests: [ + // Behavior differs between platforms: on Windows this test passes while + // it fails on Linux. Use disabledTests to avoid the cross-platform mismatch. + 'TextDecoder decode() with array buffer detached during arg conversion', + ], + }, + 'textdecoder-byte-order-marks.any.js': {}, + 'textdecoder-copy.any.js': {}, + 'textdecoder-eof.any.js': {}, + 'textdecoder-fatal-single-byte.any.js': {}, + 'textdecoder-fatal-streaming.any.js': {}, + 'textdecoder-fatal.any.js': {}, + 'textdecoder-ignorebom.any.js': {}, + 'textdecoder-labels.any.js': {}, + 'textdecoder-mistakes.any.js': { + comment: 'iso-2022-jp fatal stream state not preserved after throw', + disabledTests: [ + // Behavior differs between platforms: on Windows this test passes while + // it fails on Linux. Use disabledTests to avoid the cross-platform mismatch. + 'fatal stream: iso-2022-jp', + ], + }, + 'textdecoder-streaming.any.js': {}, + 'textdecoder-utf16-surrogates.any.js': {}, + 'textencoder-constructor-non-utf.any.js': {}, + 'textencoder-utf16-surrogates.any.js': {}, + 'unsupported-encodings.any.js': { + comment: 'XMLHttpRequest is not defined', + expectedFailures: [ + 'UTF-7 should not be supported', + 'utf-7 should not be supported', + 'UTF-32 with BOM should decode as UTF-16LE', + 'UTF-32 with no BOM should decode as UTF-8', + 'utf-32 with BOM should decode as UTF-16LE', + 'utf-32 with no BOM should decode as UTF-8', + 'UTF-32LE with BOM should decode as UTF-16LE', + 'UTF-32LE with no BOM should decode as UTF-8', + 'utf-32le with BOM should decode as UTF-16LE', + 'utf-32le with no BOM should decode as UTF-8', + 'UTF-32be with no BOM should decode as UTF-8', + 'UTF-32be with BOM should decode as UTF-8', + 'utf-32be with no BOM should decode as UTF-8', + 'utf-32be with BOM should decode as UTF-8', + ], + }, + 'unsupported-labels.window.js': { + comment: + 'Browser-only test that requires document/iframe for HTML charset inheritance detection', + disabledTests: true, + }, +} satisfies TestRunnerConfig; From cfba31b571b3b445ed5570cfe1a844dcaad9faea Mon Sep 17 00:00:00 2001 From: James M Snell Date: Tue, 7 Jul 2026 07:25:46 -0700 Subject: [PATCH 039/101] Minor per_isolate primordials cleanups, notes --- src/per_isolate/AGENTS.md | 181 +++++++++++++++++++------ src/per_isolate/primordials.ts | 20 +-- src/per_isolate/webstreams/AGENTS.md | 49 +++++++ src/per_isolate/webstreams/encoding.ts | 26 ++-- src/per_isolate/webstreams/identity.ts | 4 +- 5 files changed, 214 insertions(+), 66 deletions(-) create mode 100644 src/per_isolate/webstreams/AGENTS.md diff --git a/src/per_isolate/AGENTS.md b/src/per_isolate/AGENTS.md index c4f7b31023c..8c8f69b25f7 100644 --- a/src/per_isolate/AGENTS.md +++ b/src/per_isolate/AGENTS.md @@ -29,7 +29,8 @@ at context creation, before any user code. Gated by the - **Primordials discipline**: no bare prototype lookups on builtins after bootstrap captures; no `for...of` over patchable iterables. Capture - methods/getters at module scope via `uncurryThis`. + methods/getters at module scope via `uncurryThis`. See the + **PRIMORDIALS USAGE** section below for the full guide. - **JSG property capture trap**: readonly JSG properties live on the PROTOTYPE only under modern compat (`workers_api_getters_setters_on_prototype`, default-on 2022-01-31). @@ -47,49 +48,147 @@ at context creation, before any user code. Gated by the `instanceof` (classes reachable from the global have user-reachable `Symbol.hasInstance`). -## WEBSTREAMS ARCHITECTURE (webstreams/) - -TypeScript Streams implementation with a backend-blind reader layer and -two consumer backends behind the `StreamConsumer`/`ByteStreamConsumer` -fence (authoritative docs are IN-SOURCE — read these headers first): - -| File | Role | -| ------------- | -------------------------------------------------------------------------------------------- | -| `queue.ts` | QUEUED backend: single-queue/multi-cursor, JS sources; fence interfaces; invariant list | -| `native.ts` | NATIVE backend: C++-(eventually-)backed pull conduit; **the C++/JS contract** + invariants | -| `readable.ts` | Reader layer + queued controllers + the BACKEND-DISPATCH points (constructor, tee, chains, byte-capable gate, JS-to-C++ extraction) | -| `writable.ts` / `transform.ts` / `strategies.ts` | WHATWG writable/transform/strategies | -| `identity.ts` | IdentityTransformStream and FixedLengthStream (byte-capable identity transforms) | -| `encoding.ts` | TextEncoderStream and TextDecoderStream (pure JS codec transforms) | -| `streams.ts` | Module aggregator and temporary native-source exports | -| `types.d.ts` | TypeScript type definitions for the streams API | - -Key rules: - -- The reader layer must stay backend-blind; backend divergence is - confined to the fence interface and the marked BACKEND-DISPATCH points. -- Do not port logic across the fence without checking BOTH invariant - lists (`queue.ts` and `native.ts` headers). -- The native source contract (marker symbol, standard pull/cancel hooks, - byobRequest discrimination, once-per-pull delivery, per-pull abort - signal for cancellation, under-delivery = fused - `{done: true, value: partial}` EOF, tee hook, `expectedLength` - exact-total byte contract) is specified in the `native.ts` header. - The future C++ integration MUST conform to it; it is currently exercised - by JS mocks only. Key addition: `pull` receives an extension `signal` - argument — the source checks `signal.aborted` before delivery and stashes - bytes for redelivery if aborted (race buffering lives source-side; the JS - conduit is uniformly bufferless). -- `kNativeSource` is TEMPORARILY re-exported via `streams.ts` for tests; - it is removed when the real C++ handshake lands. +## PRIMORDIALS USAGE + +Bootstrap code shares an isolate with user code. After bootstrap runs, +user code can replace or patch any global constructor, prototype method, +or well-known symbol accessor. Primordials are pre-user-code captures of +built-in functions and constructors that remain trustworthy regardless of +subsequent mutation. The captures are in `primordials.ts`; the `primordials` +pseudo-global is injected into every bootstrap script's scope. + +**What primordials guarantee — and what they do not.** Primordials are +an **internal correctness** mechanism, not a security sandbox. The hard +security boundary is the V8 isolate: separate address spaces, no shared +mutable state between tenants. Within a single isolate, primordials +ensure that the runtime's own implementation (bootstrap code, built-in +API polyfills, internal bookkeeping) continues to function correctly even +when user code mutates built-in prototypes — whether by accident (sloppy +polyfills, test mocks) or by design (legitimate metaprogramming). +Without primordials, a user doing `Array.prototype.push = () => {}` or +`Promise.prototype.then = badFn` could break internal runtime code that +calls those methods, leading to silent data corruption, spec-violating +behaviour, or crashes. Primordials prevent that class of failure. + +### When primordials ARE required + +Use captured references whenever bootstrap code operates on **built-in +objects whose prototypes are reachable by user code**: + +- **Method calls on built-in types.** `array.push(v)` dispatches through + `Array.prototype.push` — use `ArrayPrototypePush(array, v)`. + This applies to Map, Set, Promise, ArrayBuffer, TypedArray, DataView, + String, and every other built-in with a mutable prototype. +- **Constructor calls.** `new Map()` uses the global `Map` — use + `new primordials.Map()` (or `new SafeMap()` when you also need safe + method dispatch — see below). +- **Static methods.** `Promise.resolve(v)` uses the global `Promise` — + use `PromiseResolve(v)`. +- **Iteration of internal collections.** `for...of` calls + `[Symbol.iterator]()` on the target, which is patchable on built-in + prototypes. For arrays, use `SafeArrayIterator` or index-based loops. + For Map/Set, use `SafeMap`/`SafeSet` (whose `[Symbol.iterator]` is + overridden) or iterate with captured methods and + `SafeMapIterator`/`SafeSetIterator`. +- **Metadata reads on views at trust boundaries.** `view.buffer`, + `view.byteOffset`, `view.byteLength`, `view.length` are prototype + accessors that user code can shadow or redefine. Use the captured + getters: `TypedArrayPrototypeGetBuffer(view)`, + `TypedArrayPrototypeGetByteLength(view)`, etc. Same for DataView + equivalents. +- **Type identification.** Never use `instanceof` or `.constructor` for + type checks — both are user-controllable. Use + `TypedArrayPrototypeGetSymbolToStringTag(value)` for typed arrays + (returns the internal `[[TypedArrayName]]` or `undefined`) and + private-brand `#field in obj` checks for internal classes. + +### When primordials are NOT required + +Primordials protect bootstrap code from mutations to **built-in +prototypes**. They are **not needed — and actively wrong — when the goal +is to invoke user-defined behaviour**: + +- **User-provided iterables / async iterables.** When an API accepts an + `Iterable` or `AsyncIterable` from user code (e.g., + `ReadableStream.from(userIterable)`), you MUST call the user's + `[Symbol.iterator]()` / `[Symbol.asyncIterator]()` — that is the API + contract. Using `SafeArrayIterator` would bypass the user's iterator + and break semantics. Standard `for...of` / `for await...of` is + correct here. +- **User-provided callbacks.** Callbacks passed by user code (e.g., + `underlyingSource.pull`, `transformer.transform`, strategy `size()`) + are user-defined functions — call them normally. +- **User-provided objects generally.** When reading properties from + user-provided objects (e.g., the `init` bag in `new Request(url, + init)`), normal property access is correct — you are consuming the + user's API surface, not a built-in prototype. +- **Spread / `Array.from` on user iterables.** `[...userIterable]` and + `ArrayFrom(userIterable)` both invoke the user's iteration protocol, + which is correct when the spec requires consuming a user iterable. + +The bright line: **primordials protect built-in prototype chains from +third-party mutation**. If the object is user-provided and you are +intentionally invoking user-defined protocols, use normal JS operations. + +### Captured constructors vs Safe wrappers + +The exports include both raw captured constructors and Safe wrappers. +These are NOT interchangeable: + +- `primordials.Map` (= `MapCtor`): the original `Map` constructor, + captured before user code runs. `new primordials.Map()` creates a + real Map, but **method calls on it** (`.get()`, `.set()`, etc.) still + dispatch through `Map.prototype` — which user code can patch. +- `SafeMap`: extends `MapCtor` with every method overridden to dispatch + through captured primordials. `new SafeMap()` is safe for both + construction and subsequent method calls. Same for `SafeSet`, + `SafeWeakMap`, `SafeWeakSet`. + +Use captured constructors (`primordials.Map`) only when you need `new` +but will access the instance exclusively through captured uncurried +methods (e.g., `MapPrototypeGet(map, key)`). Use Safe wrappers when you +want normal `map.get(key)` call syntax with pollution resistance. + +### Promise patterns + +- **`SafePromise`**: species-protected; `.then()` / `.catch()` / + `.finally()` use captured methods and `Symbol.species` is pinned. + Static helpers (`SafePromise.resolve`, `.reject`, `.all`, etc.) are + bound to `SafePromise`. Use for **internal-only** promise chains + where species hijacking would be a concern. +- **`PromiseResolve` / `PromiseReject` / `PromiseWithResolvers`**: + captured statics bound to the original `Promise` constructor. Use for + **user-facing** promises — the returned promise is a regular + `Promise`, which is what spec algorithms and user code expect. +- **`PromisePrototypeThen` / `PromisePrototypeCatch` / + `PromisePrototypeFinally`**: captured prototype methods. Use when + chaining on an existing promise (internal or external) without + relying on the prototype chain. + +### Never expose primordials to user code + +Primordial captures, Safe\* classes, and captured constructors are +**strictly internal**. Never assign them as properties on any object +reachable by user code. If user code can reach a Safe\* class instance, +it can traverse `Object.getPrototypeOf()` to reach the Safe class and +then the original captured constructor — potentially mutating it and +affecting all code that references it. + +Consequences: +- Never return a `SafePromise` to user code — wrap with + `PromiseResolve()` to return a regular `Promise`. +- Never store a Safe\* wrapper on a user-visible object property. +- Never expose captured constructors (e.g., `primordials.Map`) as API + return values or properties. + +The `module.exports` object from `primordials.ts` is `ObjectFreeze`-d +to prevent accidental mutation by downstream bootstrap scripts. ## ANTI-PATTERNS - **NEVER** assume a JSG readonly property has a capturable getter (see the capture trap above). - **NEVER** add a runtime require cycle between bootstrap scripts. -- **NEVER** expose internals on user-visible exports (the temporary - `kNativeSource` exception is tracked for removal). -- **NEVER** rely on `readable-source-adapter.h` as a reference for the - native streams contract — it belongs to the original (non-enabled) - implementation; `native.ts` is authoritative. + +See also `webstreams/AGENTS.md` for streams-specific architecture and +anti-patterns. diff --git a/src/per_isolate/primordials.ts b/src/per_isolate/primordials.ts index 6d50b1c98e4..a210de46ea7 100644 --- a/src/per_isolate/primordials.ts +++ b/src/per_isolate/primordials.ts @@ -633,12 +633,12 @@ const EventTargetRemoveEventListener = uncurryThis( const TextDecoderCtor = globalThis.TextDecoder; const TextEncoderCtor = globalThis.TextEncoder; -const textEncoderEncode = uncurryThis(TextEncoderCtor.prototype.encode) as ( +const TextEncoderEncode = uncurryThis(TextEncoderCtor.prototype.encode) as ( encoder: TextEncoder, input: string ) => Uint8Array; -const textDecoderDecode = uncurryThis(TextDecoderCtor.prototype.decode) as ( +const TextDecoderDecode = uncurryThis(TextDecoderCtor.prototype.decode) as ( decoder: TextDecoder, input?: BufferSource, options?: { stream?: boolean } @@ -646,14 +646,14 @@ const textDecoderDecode = uncurryThis(TextDecoderCtor.prototype.decode) as ( // TextDecoder readonly properties use the JSG layout trap: prototype // accessors under modern compat, own data properties under old dates. -const textDecoderEncodingGet = captureJsgGetter< +const TextDecoderEncodingGet = captureJsgGetter< (decoder: TextDecoder) => string >(TextDecoderCtor.prototype, 'encoding'); -const textDecoderFatalGet = captureJsgGetter<(decoder: TextDecoder) => boolean>( +const TextDecoderFatalGet = captureJsgGetter<(decoder: TextDecoder) => boolean>( TextDecoderCtor.prototype, 'fatal' ); -const textDecoderIgnoreBOMGet = captureJsgGetter< +const TextDecoderIgnoreBOMGet = captureJsgGetter< (decoder: TextDecoder) => boolean >(TextDecoderCtor.prototype, 'ignoreBOM'); @@ -825,11 +825,11 @@ module.exports = ObjectFreeze({ AbortSignalReasonGet, // TextDecoder/TextEncoder - textDecoderEncodingGet, - textDecoderFatalGet, - textDecoderIgnoreBOMGet, - textEncoderEncode, - textDecoderDecode, + TextDecoderEncodingGet, + TextDecoderFatalGet, + TextDecoderIgnoreBOMGet, + TextEncoderEncode, + TextDecoderDecode, // Safe types — use normal method syntax, resistant to prototype pollution SafeMap, diff --git a/src/per_isolate/webstreams/AGENTS.md b/src/per_isolate/webstreams/AGENTS.md new file mode 100644 index 00000000000..6c32d6161a7 --- /dev/null +++ b/src/per_isolate/webstreams/AGENTS.md @@ -0,0 +1,49 @@ +# src/per_isolate/webstreams/ + +TypeScript Streams implementation with a backend-blind reader layer and +two consumer backends behind the `StreamConsumer`/`ByteStreamConsumer` +fence. Authoritative docs are IN-SOURCE — read the file headers first. + +Parent directory conventions (primordials discipline, JSG capture trap, +private-brand dispatch, no `instanceof`) apply here — see +`src/per_isolate/AGENTS.md`. + +## FILE MAP + +| File | Role | +| ------------- | -------------------------------------------------------------------------------------------- | +| `queue.ts` | QUEUED backend: single-queue/multi-cursor, JS sources; fence interfaces; invariant list | +| `native.ts` | NATIVE backend: C++-(eventually-)backed pull conduit; **the C++/JS contract** + invariants | +| `readable.ts` | Reader layer + queued controllers + the BACKEND-DISPATCH points (constructor, tee, chains, byte-capable gate, JS-to-C++ extraction) | +| `writable.ts` / `transform.ts` / `strategies.ts` | WHATWG writable/transform/strategies | +| `identity.ts` | IdentityTransformStream and FixedLengthStream (byte-capable identity transforms) | +| `encoding.ts` | TextEncoderStream and TextDecoderStream (pure JS codec transforms) | +| `streams.ts` | Module aggregator and temporary native-source exports | +| `types.d.ts` | TypeScript type definitions for the streams API | + +## KEY RULES + +- The reader layer must stay backend-blind; backend divergence is + confined to the fence interface and the marked BACKEND-DISPATCH points. +- Do not port logic across the fence without checking BOTH invariant + lists (`queue.ts` and `native.ts` headers). +- The native source contract (marker symbol, standard pull/cancel hooks, + byobRequest discrimination, once-per-pull delivery, per-pull abort + signal for cancellation, under-delivery = fused + `{done: true, value: partial}` EOF, tee hook, `expectedLength` + exact-total byte contract) is specified in the `native.ts` header. + The future C++ integration MUST conform to it; it is currently exercised + by JS mocks only. Key addition: `pull` receives an extension `signal` + argument — the source checks `signal.aborted` before delivery and stashes + bytes for redelivery if aborted (race buffering lives source-side; the JS + conduit is uniformly bufferless). +- `kNativeSource` is TEMPORARILY re-exported via `streams.ts` for tests; + it is removed when the real C++ handshake lands. + +## ANTI-PATTERNS + +- **NEVER** expose internals on user-visible exports (the temporary + `kNativeSource` exception is tracked for removal). +- **NEVER** rely on `readable-source-adapter.h` as a reference for the + native streams contract — it belongs to the original (non-enabled) + implementation; `native.ts` is authoritative. diff --git a/src/per_isolate/webstreams/encoding.ts b/src/per_isolate/webstreams/encoding.ts index 8aa5d0b9d78..c41ecc09c0a 100644 --- a/src/per_isolate/webstreams/encoding.ts +++ b/src/per_isolate/webstreams/encoding.ts @@ -22,13 +22,13 @@ const { StringPrototypeCharCodeAt, StringPrototypeSlice, SymbolToStringTag, - textDecoderEncodingGet, + TextDecoderEncodingGet, TextDecoder, TextEncoder, - textDecoderFatalGet, - textDecoderIgnoreBOMGet, - textEncoderEncode, - textDecoderDecode, + TextDecoderFatalGet, + TextDecoderIgnoreBOMGet, + TextEncoderEncode, + TextDecoderDecode, TypeError, uncurryThis, } = primordials; @@ -105,18 +105,18 @@ class TextEncoderStream { ) as string; const prefix = StringPrototypeSlice(text, 0, len - 1) as string; if (prefix.length > 0) { - controller.enqueue(textEncoderEncode(encoder, prefix)); + controller.enqueue(TextEncoderEncode(encoder, prefix)); } } else { self.#pendingHighSurrogate = ''; - controller.enqueue(textEncoderEncode(encoder, text)); + controller.enqueue(TextEncoderEncode(encoder, text)); } }, flush(controller: { enqueue: (c: Uint8Array) => void }) { // A pending high surrogate at end-of-stream is replaced with // U+FFFD (the replacement character). if (self.#pendingHighSurrogate !== '') { - controller.enqueue(textEncoderEncode(encoder, '\uFFFD')); + controller.enqueue(TextEncoderEncode(encoder, '\uFFFD')); self.#pendingHighSurrogate = ''; } }, @@ -178,7 +178,7 @@ class TextDecoderStream { 'TextDecoderStream: chunk must be a BufferSource' ); } - const decoded = textDecoderDecode(decoder, chunk as BufferSource, { + const decoded = TextDecoderDecode(decoder, chunk as BufferSource, { stream: true, }); if (decoded.length > 0) { @@ -188,7 +188,7 @@ class TextDecoderStream { flush(controller: { enqueue: (c: string) => void }) { // Final decode: flushes any incomplete multi-byte sequences. // In fatal mode this throws if the sequence is incomplete. - const decoded = textDecoderDecode(decoder); + const decoded = TextDecoderDecode(decoder); if (decoded.length > 0) { controller.enqueue(decoded); } @@ -198,17 +198,17 @@ class TextDecoderStream { get encoding(): string { assertIsTextDecoderStream(this); - return textDecoderEncodingGet(this.#decoder); + return TextDecoderEncodingGet(this.#decoder); } get fatal(): boolean { assertIsTextDecoderStream(this); - return textDecoderFatalGet(this.#decoder); + return TextDecoderFatalGet(this.#decoder); } get ignoreBOM(): boolean { assertIsTextDecoderStream(this); - return textDecoderIgnoreBOMGet(this.#decoder); + return TextDecoderIgnoreBOMGet(this.#decoder); } get readable(): ReadableStreamType { diff --git a/src/per_isolate/webstreams/identity.ts b/src/per_isolate/webstreams/identity.ts index 3fe93c1aa3e..36a7a271d81 100644 --- a/src/per_isolate/webstreams/identity.ts +++ b/src/per_isolate/webstreams/identity.ts @@ -43,7 +43,7 @@ const { PromiseWithResolvers, Symbol, TextEncoder, - textEncoderEncode, + TextEncoderEncode, TypeError, TypedArrayPrototypeGetBuffer, TypedArrayPrototypeGetByteLength, @@ -112,7 +112,7 @@ const textEncoderInstance = new TextEncoder(); function validateAndCopyChunk(chunk: unknown): Uint8Array | undefined { if (typeof chunk === 'string') { if (chunk.length === 0) return undefined; - return textEncoderEncode(textEncoderInstance, chunk); + return TextEncoderEncode(textEncoderInstance, chunk); } if (isArrayBuffer(chunk) || isSharedArrayBuffer(chunk)) { // Wrap in Uint8Array for a uniform code path — Uint8Array accepts From 73e94d8b6e852e666fa858c469640425e673d19c Mon Sep 17 00:00:00 2001 From: Aaron Loyd Date: Tue, 23 Jun 2026 20:56:05 -0500 Subject: [PATCH 040/101] Update unsafe-continuation-capture path filter to be based on file path Originally it was based on bazel packages, but I failed to realize that a lot of workerd stuff isn't in separate bazel packages. It's one big package. So //src/workerd/api would miss a lot of files in that folder. This also moves the captures test file from edgeworker and updates more documentation. --- AGENTS.md | 5 + build/AGENTS.md | 10 +- build/tools/clang_tidy/check_path_filters.bzl | 18 +- build/tools/clang_tidy/clang_tidy.bzl | 23 +- docs/reference/detail/async-patterns.md | 47 ++ tools/clang-tidy/BUILD.bazel | 1 + .../unsafe-continuation-capture-test.c++ | 530 ++++++++++++++++++ 7 files changed, 609 insertions(+), 25 deletions(-) create mode 100644 tools/clang-tidy/tests/unsafe-continuation-capture-test.c++ diff --git a/AGENTS.md b/AGENTS.md index 5156e364565..281bfaca64a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -246,6 +246,11 @@ C++ classes are exposed to JavaScript via JSG macros in `src/workerd/jsg/`. See - The `jsg-visit-for-gc` clang-tidy check (`//tools/clang-tidy:workerd-lint`) validates that GC-visitable fields are traced in `visitForGc()`. Run via `just clang-tidy `. See `build/AGENTS.md` for details. +- The `workerd-unsafe-continuation-capture` clang-tidy check flags lambdas + passed to async sinks (`kj/jsg::Promise::then`, `IoContext::run/awaitIo/...`, + `kj::evalLater`, ...) that capture bare references, `[this]`, or non-owning + views. See `docs/reference/detail/async-patterns.md` §Continuation Captures + for the safe-capture pattern catalog. ### Feature Management diff --git a/build/AGENTS.md b/build/AGENTS.md index d7c6ff641fe..1bffa304036 100644 --- a/build/AGENTS.md +++ b/build/AGENTS.md @@ -74,7 +74,7 @@ supports this: 1. Add the check to `.clang-tidy` Checks list 2. Add an entry to `CHECK_PATH_FILTERS` with an empty list (runs nowhere) -3. Add packages as they are cleaned up +3. Add file paths as they are cleaned up 4. Remove the entry once fully rolled out (runs everywhere) Example: @@ -82,14 +82,14 @@ Example: ```python CHECK_PATH_FILTERS = { "workerd-unsafe-continuation-capture": [ - "//src/workerd/io", - "//src/workerd/api", + "src/workerd/io", + "src/workerd/api", ], } ``` -Package prefixes match themselves and all subpackages (`//src/workerd/io` -matches `//src/workerd/io:*` and `//src/workerd/io/subdir:*`). +Path prefixes match all files under that directory (`src/workerd/io` +matches `src/workerd/io/foo.c++` and `src/workerd/io/subdir/bar.c++`). To run a filtered check everywhere during development: diff --git a/build/tools/clang_tidy/check_path_filters.bzl b/build/tools/clang_tidy/check_path_filters.bzl index 904e46de6b7..fd915f060b4 100644 --- a/build/tools/clang_tidy/check_path_filters.bzl +++ b/build/tools/clang_tidy/check_path_filters.bzl @@ -1,25 +1,25 @@ """Per-check path filtering for incremental clang-tidy rollout. -Checks listed here are only enabled in specific Bazel packages. This allows +Checks listed here are only enabled for files under specific paths. This allows incremental rollout of new checks: add the check to .clang-tidy, add an entry -here with an empty list, then add packages as they are cleaned up. +here with an empty list, then add paths as they are cleaned up. -Keys are clang-tidy check names. Values are lists of Bazel package prefixes. -A package prefix matches itself and all subpackages: - - "//src/workerd/io" matches //src/workerd/io:* and //src/workerd/io/foo:* +Keys are clang-tidy check names. Values are lists of file path prefixes. +A path prefix matches all files under that directory: + - "src/workerd/io" matches src/workerd/io/foo.c++ and src/workerd/io/bar/baz.c++ Behavior: - Check in .clang-tidy but NOT here: runs everywhere (normal behavior) - Check here with empty list: runs nowhere - - Check here with packages: runs only in listed packages + - Check here with paths: runs only for files under listed paths Once a check is fully rolled out, remove its entry to run everywhere. """ CHECK_PATH_FILTERS = { "workerd-unsafe-continuation-capture": [ - # Add packages here as they are cleaned up: - # "//src/workerd/io", - # "//src/workerd/api", + # Add paths here as they are cleaned up: + # "src/workerd/io", + # "src/workerd/api", ], } diff --git a/build/tools/clang_tidy/clang_tidy.bzl b/build/tools/clang_tidy/clang_tidy.bzl index 298a737f847..ee15bd794f5 100644 --- a/build/tools/clang_tidy/clang_tidy.bzl +++ b/build/tools/clang_tidy/clang_tidy.bzl @@ -9,23 +9,23 @@ load("@rules_cc//cc/common:cc_common.bzl", "cc_common") load("@rules_cc//cc/common:cc_info.bzl", "CcInfo") load("//build/tools/clang_tidy:check_path_filters.bzl", "CHECK_PATH_FILTERS") -def _get_disabled_checks_for_package(package): - """Returns checks that should be disabled for this package. +def _get_disabled_checks_for_file(file_path): + """Returns checks that should be disabled for this file. - Checks listed in CHECK_PATH_FILTERS are only enabled in their allowed - packages. For packages not in the allowed list, the check is disabled. + Checks listed in CHECK_PATH_FILTERS are only enabled for files under their + allowed paths. For files not under any allowed path, the check is disabled. """ disabled = [] - for check, allowed_packages in CHECK_PATH_FILTERS.items(): + for check, allowed_paths in CHECK_PATH_FILTERS.items(): enabled = False - for allowed in allowed_packages: - # Normalize: remove leading "//" for comparison + for allowed in allowed_paths: + # Normalize: remove leading "//" or "src/" for comparison allowed_path = allowed if allowed_path.startswith("//"): allowed_path = allowed_path[2:] - # Prefix match: "src/foo" matches "src/foo" and "src/foo/bar" - if package == allowed_path or package.startswith(allowed_path + "/"): + # Prefix match: "src/foo" matches "src/foo/bar.c++" and "src/foo/baz/qux.c++" + if file_path.startswith(allowed_path + "/") or file_path.startswith(allowed_path): enabled = True break if not enabled: @@ -148,8 +148,9 @@ def _clang_tidy_aspect_impl(target, ctx): args.add("--experimental-custom-checks") args.add("--config-file=" + clang_tidy_config.path) - # Disable checks that are path-filtered and not enabled for this package - disabled_checks = _get_disabled_checks_for_package(ctx.label.package) + # Disable checks that are path-filtered and not enabled for this file + real_path = src.path.removeprefix(src.owner.workspace_root + "/") + disabled_checks = _get_disabled_checks_for_file(real_path) # Parse clang_tidy_args to separate --checks from other args. # We must combine all --checks into a single argument because multiple diff --git a/docs/reference/detail/async-patterns.md b/docs/reference/detail/async-patterns.md index 34db1e36719..152ee40447e 100644 --- a/docs/reference/detail/async-patterns.md +++ b/docs/reference/detail/async-patterns.md @@ -45,6 +45,53 @@ return kj::evalNow([&]() { --- +### Continuation Captures + +`workerd-unsafe-continuation-capture` flags lambdas passed to async sinks +(`kj/jsg::Promise::then/catch_`, `IoContext::run/addTask/awaitIo/addFunctor`, +`kj::evalLater`, ...) that capture bare references, `[this]`, or non-owning views. +Use one of these patterns: + +| Site | Capture | +|---|---| +| JSG resource | `[self = JSG_THIS]` | +| `kj::Refcounted` | `[self = addRefToThis()]` | +| IoContext, in JS-lock scope (`Worker::Lock&`/`jsg::Lock&` available) | `auto& context = IoContext::current();` inside the lambda (but see caveat below) | +| IoContext, outside JS-lock scope | `[weakRef = context.getWeakRef()]` + `KJ_ASSERT_NONNULL(weakRef->tryGet())` (alive by invariant) or `weakRef->runIfAlive(...)` (may be gone) | +| Chain feeds opaque wrapper (`oomCanceler.wrap`, `gate.lockWhile`, `.fork()`) | IILE coroutine — pass `this`/`&context` as a coroutine parameter, not a lambda capture | +| Chain is `co_await`ed / `.wait()`ed / joined from a local container | already safe; no change | + +Do **not** strong-ref an IoContext (`kj::addRef(context)`) to satisfy the check — +chains owned transitively by the IoContext form a refcount cycle. + +**`IoContext::current()` caveat.** Re-deriving the IoContext from inside the +lambda is safe only when the continuation is guaranteed to run under the same +IoContext that scheduled it. `jsg::Promise::then()` does **not** make that +guarantee in general: if the application returns a foreign promise that +resolves from a different context, the continuation can run with a different +IoContext active, and `IoContext::current()` will return the wrong one (or +throw if no context is active). Each use needs to be evaluated for this +possibility. When it can happen, capture the originating context's +`getWeakRef()` instead (or use `IoContext::addFunctor`) so the continuation +either runs under the correct context or fails loudly. + +**IoContext WeakRef.** `IoContext::WeakRef` (returned by `context.getWeakRef()`) +is a refcounted, non-owning handle to an `IoContext`. The WeakRef itself is +safe to hold past the IoContext's destruction; `tryGet()` returns a `Maybe` +that becomes `kj::none` once the context is gone, and `runIfAlive(func)` runs +`func` with the live `IoContext&` and returns whether it did. Use the WeakRef +pattern any time a continuation may outlive its originating context, or when +the continuation runs outside a JS-lock scope where `IoContext::current()` may +not be the right context (or may not exist at all). + +For invariants the analyzer fundamentally can't see (capnp `thisCap()`, +fiber-blocked stacks, constructor-time `*this`, `CantOutliveIncomingRequest`-style +structural guarantees), add +`// NOLINTNEXTLINE(workerd-unsafe-continuation-capture)` with a one-line +justification. + +--- + ### Mutex Patterns `kj::MutexGuarded` ties locking to access — you can't touch the data without a lock: diff --git a/tools/clang-tidy/BUILD.bazel b/tools/clang-tidy/BUILD.bazel index 0e37a519398..f28371941f7 100644 --- a/tools/clang-tidy/BUILD.bazel +++ b/tools/clang-tidy/BUILD.bazel @@ -37,6 +37,7 @@ exports_files([ "visit-for-gc.h", "unsafe-continuation-capture.c++", "unsafe-continuation-capture.h", + "tests/unsafe-continuation-capture-test.c++", ]) filegroup( diff --git a/tools/clang-tidy/tests/unsafe-continuation-capture-test.c++ b/tools/clang-tidy/tests/unsafe-continuation-capture-test.c++ new file mode 100644 index 00000000000..779463bf597 --- /dev/null +++ b/tools/clang-tidy/tests/unsafe-continuation-capture-test.c++ @@ -0,0 +1,530 @@ +// Test cases for the workerd-unsafe-continuation-capture check. +// +// This file is *not* part of the regular build. To run the check against +// it manually: +// +// bazel build //tools/clang-tidy:workerd-lint +// clang-tidy --load=bazel-bin/tools/clang-tidy/libworkerd-lint.so \ +// --checks=-*,workerd-unsafe-continuation-capture \ +// tools/clang-tidy/tests/unsafe-continuation-capture-test.c++ -- -std=c++23 +// +// Comments of the form `// expected: ...` describe the diagnostic the +// check should (or should not) emit on the following lambda. + +// --------------------------------------------------------------------------- +// Minimal stand-ins for kj / jsg / workerd types. These only need enough +// shape for the AST matcher to recognize the sink callees and capture +// types -- no real semantics. + +namespace kj { +template +class Own { +public: + T *operator->() const; + T &operator*() const; +}; +template +class Rc {}; +template +class Arc {}; +template +class WeakRef {}; +template +class Maybe {}; +template +class Array {}; +template +class Vector { +public: + template + void add(U &&); + Array releaseAsArray(); + bool empty() const; +}; +template +class ArrayBuilder { +public: + template + void add(U &&); + Array finish(); +}; +class TaskSet { +public: + template + void add(U &&); +}; +template +class ArrayPtr { +public: + ArrayPtr() = default; + ArrayPtr(T *, unsigned long); +}; +class StringPtr { +public: + StringPtr() = default; + StringPtr(const char *); +}; +class String {}; +class Date {}; +template +class Promise { +public: + template + Promise then(F &&); + template + Promise catch_(F &&); +}; +template +auto evalLater(F &&f) -> Promise; +template +T mv(T &&v) { + return static_cast(v); +} +template +ArrayBuilder heapArrayBuilder(unsigned long n); +template +Array arr(T &&, U &&...); +template +Promise joinPromisesFailFast(Array>); +template +Promise joinPromises(Array>); +} // namespace kj + +namespace workerd { +namespace jsg { +class Lock {}; +template +class Ref {}; +template +class V8Ref {}; +template +class JsRef {}; +template +class WeakRef {}; +template +class Promise { +public: + template + Promise then(Lock &, F &&); +}; +template +Ref _jsgThis(Self *); +} // namespace jsg + +class IoContext { +public: + template + auto run(Func &&) -> kj::Promise; + void addTask(kj::Promise); + template + auto addFunctor(Func &&) -> Func &&; + template + jsg::Promise awaitIo(jsg::Lock &, kj::Promise, Func &&); +}; + +template +class IoOwn {}; +template +class IoPtr {}; + +// Minimal stand-in for the workerd test harness fixture. Real +// definition lives in src/workerd/tests/test-fixture.h. Its callback is +// invoked synchronously and the returned promise is `.wait()`-ed before +// `runInIoContext` returns -- so lambdas passed to it (and any nested +// continuations whose escape route is `return` from the callback) are +// treated as locally consumed. +class TestFixture { +public: + template + auto runInIoContext(Func &&) -> int; +}; +} // namespace workerd + +#define JSG_THIS (::workerd::jsg::_jsgThis(this)) + +// --------------------------------------------------------------------------- +// Example sink we expect the check to *not* recognize. + +template +auto KJ_MAP(Container &&c, F &&f) -> int { + return 0; +} + +// --------------------------------------------------------------------------- +// POSITIVE CASES: should trigger diagnostics. + +struct Resource; + +kj::Promise bareReferenceCapture(Resource &r) { + kj::Promise p; + // expected: unsafe by-reference capture of `r` + return p.then([&r](int x) { return x; }); +} + +kj::Promise bareReferenceDefault(Resource &r) { + kj::Promise p; + // expected: unsafe by-reference capture (implicit `[&]`) + return p.then([&](int x) { (void)r; return x; }); +} + +kj::Promise rawPointerCapture(Resource *r) { + kj::Promise p; + // expected: unsafe raw pointer capture of `r` + return p.then([r](int x) { (void)r; return x; }); +} + +kj::Promise arrayPtrCapture(kj::ArrayPtr data) { + kj::Promise p; + // expected: unsafe non-owning view capture of `data` + return p.then([data](int x) { (void)data; return x; }); +} + +kj::Promise stringPtrCapture(kj::StringPtr s) { + kj::Promise p; + // expected: unsafe non-owning view capture of `s` + return p.then([s](int x) { (void)s; return x; }); +} + +struct JsgThing { + jsg::Promise doIt(jsg::Lock &js, jsg::Promise p) { + // expected: unsafe `this` capture; use JSG_THIS + return p.then(js, [this](jsg::Lock &, int x) { (void)this; return x; }); + } +}; + +kj::Promise ioContextRun(workerd::IoContext &ctx, kj::Own r) { + // Mirror of streams/standard.c++:3789. Expected diagnostic on `&r`. + return ctx.run([&r](workerd::jsg::Lock &) -> int { (void)r; return 0; }); +} + +// --------------------------------------------------------------------------- +// NEGATIVE CASES: should NOT trigger diagnostics. + +kj::Promise ownedCapture(kj::Own r) { + kj::Promise p; + // safe: kj::Own transfers ownership. + return p.then([r = kj::mv(r)](int x) { (void)r; return x; }); +} + +kj::Promise jsgRefCapture(workerd::jsg::Ref ref, + kj::Promise p) { + // safe: jsg::Ref is an owning JS heap root. + return p.then([ref = kj::mv(ref)](int x) { (void)ref; return x; }); +} + +jsg::Promise jsgThisOk(JsgThing *t, jsg::Lock &js, jsg::Promise p) { + // safe: JSG_THIS is a jsg::Ref. + return p.then(js, [self = JSG_THIS](jsg::Lock &, int x) { + (void)self; + return x; + }); +} + +int immediateLambda(int x, int y) { + int &refY = y; + // safe: invoked immediately, not stored. The matcher will see this is + // not passed to an async-sink callee. + return [&]() { return x + refY; }(); +} + +// Escape-analysis cases: the same lambda capturing bare references is +// safe when the continuation's result is consumed locally, unsafe when +// it escapes. + +int promiseWaitedSynchronously(Resource &r) { + kj::Promise p; + kj::WaitScope *ws = nullptr; + // safe: .then's promise is .wait()'d immediately. `r` cannot dangle. + return p.then([&r](int x) { (void)r; return x; }).wait(*ws); +} + +kj::Promise promiseAwaitedInCoroutine(Resource &r, kj::Promise p) { + // safe: the .then's promise is co_await-ed; the coroutine frame keeps + // `r` alive through the suspension point. + int v = co_await p.then([&r](int x) { (void)r; return x; }); + co_return v; +} + +kj::Promise promiseChainWaitedLater(Resource &r, kj::Promise p, + kj::WaitScope &ws) { + // safe: result is bound to a local, then .wait()-ed. + auto chained = p.then([&r](int x) { (void)r; return x; }); + return chained.wait(ws); +} + +kj::Promise promiseChainedThenReturned(Resource &r, kj::Promise p) { + // unsafe: outer .then's result is returned; the captures of *both* + // lambdas escape. expected diagnostic on `&r`. + return p.then([&r](int x) { (void)r; return x; }) + .then([](int y) { return y + 1; }); +} + +kj::Promise promiseDiscarded(kj::Promise p, Resource &r) { + // The .then result is discarded as a full-expression. The promise's + // destructor runs in this scope; the captures don't outlive the + // function. Safe. + p.then([&r](int x) { (void)r; return x; }); + return kj::Promise(); +} + +// Escape-analysis through a local promise *container* that is finalized, +// joined, and co_await-ed in the same coroutine: the continuation cannot +// outlive the container, so by-reference captures are safe. + +kj::Promise addedToArrayBuilderThenJoined(Resource &r, kj::Promise p) { + // safe: continuation is added to a local kj::ArrayBuilder, finished, joined, + // and co_await-ed in this coroutine. + auto promises = kj::heapArrayBuilder>(1); + promises.add(p.then([&r](int x) { (void)r; return x; })); + co_await kj::joinPromisesFailFast(promises.finish()); +} + +kj::Promise addedToVectorThenJoined(Resource &r, kj::Promise p) { + // safe: continuation is added to a local kj::Vector, released, joined, and + // co_await-ed in this coroutine. + kj::Vector> promises; + promises.add(p.then([&r](int x) { (void)r; return x; })); + co_await kj::joinPromisesFailFast(promises.releaseAsArray()); +} + +kj::Promise addedToVectorWithEmptyCheck(Resource &r, kj::Promise p) { + // safe: the container is queried via .empty() (a benign query) and then + // released, joined, and co_await-ed -- the continuation still cannot escape. + kj::Vector> promises; + promises.add(p.then([&r](int x) { (void)r; return x; })); + if (!promises.empty()) { + co_await kj::joinPromisesFailFast(promises.releaseAsArray()); + } +} + +kj::Promise localTaskIntoArrThenJoined(Resource &r, kj::Promise p) { + // safe: continuation bound to a local, moved into kj::arr(...), joined, and + // co_await-ed in this coroutine. + auto task = p.then([&r](int x) { (void)r; return x; }); + co_await kj::joinPromisesFailFast(kj::arr(kj::mv(task))); +} + +kj::Array> addedToArrayThenReturned(Resource &r, + kj::Promise p) { + // unsafe: the promise array is returned to the caller, so the continuation + // escapes this function. expected diagnostic on `&r`. + auto promises = kj::heapArrayBuilder>(1); + promises.add(p.then([&r](int x) { (void)r; return x; })); + return promises.finish(); +} + +// A continuation added to a *member* promise container of `*this` (e.g. a +// `kj::TaskSet` field) is owned by that member; its destructor cancels the +// continuation before any other field of `*this` can dangle, so capturing +// `this` is safe (other captures still escape). + +struct MemberTaskHolder { + kj::TaskSet tasks; + + void addThisToMemberTaskSet(kj::Promise p) { + // safe: continuation is stored in the member `tasks`; capturing `this` is + // safe (StoredAsSelfMember). + tasks.add(p.catch_([this]() { (void)this; })); + } + + void addRefToMemberTaskSet(kj::Promise p, Resource &r) { + // unsafe: storing in a member container does not save a by-reference + // capture of a stack reference -- it dangles once this method returns. + // expected diagnostic on `&r`. + tasks.add(p.then([&r]() { (void)r; })); + } +}; + +int synchronousCallback(int x) { + int &refX = x; + // safe: KJ_MAP is not in the async-sinks list. + return KJ_MAP(refX, [&](int) { return 1; }); +} + +kj::Promise intCapture(int n, kj::Promise p) { + // safe: int is a trivially-safe value. + return p.then([n](int x) { return x + n; }); +} + +kj::Promise stringCapture(kj::String s, kj::Promise p) { + // safe: kj::String owns its buffer. + return p.then([s = kj::mv(s)](int x) { (void)s; return x; }); +} + +kj::Promise maybeOwnCapture(kj::Maybe> maybe, + kj::Promise p) { + // safe: kj::Maybe> -- transparent container around an + // owning type. + return p.then([maybe = kj::mv(maybe)](int x) { (void)maybe; return x; }); +} + +// --------------------------------------------------------------------------- +// `.attach(...)` binding suppression: a continuation chain +// can bind the lifetime of arbitrary referents to itself via .attach. +// Captures of those bound names are safe even when the chain escapes. + +namespace kj { +template +class Promise2 { +public: + template + Promise2 then(F &&); + template + Promise2 attach(Attachments &&...); +}; +} // namespace kj + +kj::Promise2 attachOwnSuppresses(kj::Own r, kj::Promise2 p) { + // safe: r is bound to the chain via .attach(kj::mv(r)), so the + // by-reference capture of r in the inner lambda cannot dangle. + return p.then([&r](int x) { (void)r; return x; }).attach(kj::mv(r)); +} + +struct PromiseAndFulfiller { + kj::Own fulfiller; +}; + +kj::Promise2 attachMemberInitCapture(kj::Promise2 p) { + PromiseAndFulfiller paf; + // safe: init-capture binds a reference to `paf.fulfiller`, and the + // chain attaches `paf.fulfiller` -- the bound (base, member) pair + // matches, so the capture is treated as safe. + return p.then([&f = *paf.fulfiller](int x) { (void)f; return x; }) + .attach(kj::mv(paf.fulfiller)); +} + +kj::Promise2 attachThisSuppresses(kj::Promise2 p); +struct AttachThisHolder { + kj::Promise2 p; + kj::Promise2 doIt() { + // safe: `*this` is attached to the chain. + return p.then([this](int x) { (void)this; return x; }).attach(*this); + } +}; + +// --------------------------------------------------------------------------- +// StoredAsSelfMember: the continuation is assigned to a member of the +// current class. The field's destructor cancels the chain before any +// other field can be torn down, so `[this]` is safe -- but other +// captures still escape. + +struct StoresChain { + kj::Promise task; + void start() { + kj::Promise p; + // safe: [this] is OK because `task` (the storing field) will be + // destroyed and cancel the chain before *this dies. + task = p.then([this](int x) { (void)this; return x; }); + } + void startBadRef(Resource &r) { + kj::Promise p; + // expected: unsafe by-reference capture of `r` -- the chain may + // outlive r even though it's stored on *this. + task = p.then([&r](int x) { (void)r; return x; }); + } +}; + +// --------------------------------------------------------------------------- +// Free-function async sinks (`kj::evalLater`, `kj::evalLast`). + +namespace kj { +template +auto evalLast(F &&) -> Promise; +} // namespace kj + +kj::Promise evalLaterBadCapture(Resource &r) { + // expected: unsafe by-reference capture passed to kj::evalLater. + return kj::evalLater([&r]() { (void)r; return 0; }); +} + +kj::Promise evalLastBadCapture(Resource &r) { + // expected: unsafe by-reference capture passed to kj::evalLast. + return kj::evalLast([&r]() { (void)r; return 0; }); +} + +// --------------------------------------------------------------------------- +// ExtraSinks / ExtraOwningTypes: user-configurable extensions. To +// exercise these the file should be compiled with the corresponding +// CheckOptions in a project-local .clang-tidy: +// +// CheckOptions: +// - key: workerd-unsafe-continuation-capture.AsyncSinks +// value: 'myproject::mySink' +// - key: workerd-unsafe-continuation-capture.OwningCaptureTypes +// value: 'myproject::MyOwn' +// +// Both cases below are exercised through that mechanism. + +namespace myproject { +template +auto mySink(F &&) -> kj::Promise; + +template +class MyOwn {}; +} // namespace myproject + +kj::Promise extraSinkBadCapture(Resource &r) { + // expected (when ExtraSinks contains "myproject::mySink"): unsafe + // by-reference capture passed to myproject::mySink. + return myproject::mySink([&r]() { (void)r; return 0; }); +} + +kj::Promise extraOwningTypeOk(myproject::MyOwn own, + kj::Promise p) { + // safe (when OwningCaptureTypes contains "myproject::MyOwn"): + // capturing a MyOwn by value transfers ownership. + return p.then([own = kj::mv(own)](int x) { (void)own; return x; }); +} + +// --------------------------------------------------------------------------- +// Synchronous sinks: lambdas passed to a callee on `kBuiltinSynchronousSinks` +// (e.g. `workerd::TestFixture::runInIoContext`) are invoked synchronously, +// and the promise they return is `.wait()`-ed before the sink returns. +// So even nested `.then()` continuations whose escape route is `return` +// from the outer lambda are safely consumed within the caller's +// activation -- their captures of stack-local references cannot dangle. + +int syncSinkDirectLambdaIsExempt(workerd::TestFixture &fx, Resource &r) { + // safe: the lambda runs synchronously; capturing `&r` cannot dangle + // because `runInIoContext` is not an async sink at all (and a fortiori + // not flagged for this capture). + return fx.runInIoContext([&r]() { (void)r; return 0; }); +} + +int syncSinkNestedThenIsExempt(workerd::TestFixture &fx, Resource &r, + kj::Promise p) { + // safe: the inner .then's promise is returned from a lambda that is + // passed directly to a synchronous sink. The sink will .wait() on the + // returned promise before returning, so `&r` cannot dangle. + return fx.runInIoContext([&r, p = kj::mv(p)]() mutable { + return p.then([&r](int x) { (void)r; return x; }); + }); +} + +int syncSinkNestedThenChainIsExempt(workerd::TestFixture &fx, Resource &r, + kj::Promise p) { + // safe: same as above but with a longer .then chain. Every link + // ultimately escapes via `return` from the outer sync-sink lambda. + return fx.runInIoContext([&r, p = kj::mv(p)]() mutable { + return p.then([&r](int x) { (void)r; return x; }) + .then([&r](int y) { (void)r; return y + 1; }); + }); +} + +// And the converse: when a nested .then's promise escapes via something +// other than the sync-sink lambda's `return`, the capture is still +// flagged. The promise here is stored in a member -- not returned to +// runInIoContext -- so `&r` can outlive the sink call. +struct StoresChainInSink { + kj::Promise task; + void start(workerd::TestFixture &fx, Resource &r, kj::Promise p) { + fx.runInIoContext([&]() mutable { + // expected: unsafe by-reference capture of `r` -- the chain is + // assigned to `task` (a non-local destination), not returned to + // the synchronous sink, so the capture's lifetime is not bounded + // by runInIoContext. + task = p.then([&r](int x) { (void)r; return x; }); + return 0; + }); + } +}; From 78ce78923937ddece178ccb580f5c856833046d5 Mon Sep 17 00:00:00 2001 From: Kira Kaviani Date: Tue, 7 Jul 2026 20:59:07 +0000 Subject: [PATCH 041/101] Remove .js module extension limitation when main module is .py * Formatting * Remove unused variable * Rename test * Fix comments & remove unnecessary test * Remove .js module extension limitation when main module is .py Some python modules may include .js files (for example, FastAPI does this) See merge request cloudflare/ew/workerd!405 --- src/workerd/api/tests/worker-loader-test.js | 17 ----------------- src/workerd/api/worker-loader.c++ | 10 +++------- 2 files changed, 3 insertions(+), 24 deletions(-) diff --git a/src/workerd/api/tests/worker-loader-test.js b/src/workerd/api/tests/worker-loader-test.js index 98befd9a6cb..be2f5f1469c 100644 --- a/src/workerd/api/tests/worker-loader-test.js +++ b/src/workerd/api/tests/worker-loader-test.js @@ -876,23 +876,6 @@ return "Hello, " + name export let noMixedJsPythonModules = { async test(ctrl, env, ctx) { let worker = env.loader.get('noMixedJsPythonModules', () => { - return { - ...mixedModules, - mainModule: 'foo.py', - }; - }); - - await assert.rejects(worker.getEntrypoint().greet('Alice'), { - name: 'TypeError', - message: - 'Module "foo.js" is a JS module, but the main module is a Python module.', - }); - }, -}; - -export let noMixedJsPythonModules2 = { - async test(ctrl, env, ctx) { - let worker = env.loader.get('noMixedJsPythonModules2', () => { return { ...mixedModules, mainModule: 'foo.js', diff --git a/src/workerd/api/worker-loader.c++ b/src/workerd/api/worker-loader.c++ index e4136a232c6..70d1e4a9738 100644 --- a/src/workerd/api/worker-loader.c++ +++ b/src/workerd/api/worker-loader.c++ @@ -306,16 +306,12 @@ Worker::Script::Source WorkerLoader::extractSource(jsg::Lock& js, WorkerCode& co }; bool isPython = code.mainModule.endsWith(".py"_kj); - // Disallow Python modules when the main module is a JS module, and vice versa. Also tally up the + // Disallow Python modules when the main module is a JS module. Also tally up the // total size of all module bodies so we can enforce the worker code size limit. + // This behavior is deliberately not replicated for Python main modules since Python packages + // can contain arbitrary .js files. size_t totalCodeSize = 0; for (auto& module: modules) { - auto isJsModule = module.content.is() || - module.content.is(); - if (isPython && isJsModule) { - JSG_FAIL_REQUIRE(TypeError, "Module \"", module.name, - "\" is a JS module, but the main module is a Python module."); - } auto isPythonModule = module.content.is(); if (!isPython && isPythonModule) { JSG_FAIL_REQUIRE(TypeError, "Module \"", module.name, From 265f4370da9428340c8abeb5332e207bbf01e006 Mon Sep 17 00:00:00 2001 From: James M Snell Date: Mon, 6 Jul 2026 17:54:22 -0700 Subject: [PATCH 042/101] Improve handling of serialization errors for diag channel tracing --- src/workerd/api/node/diagnostics-channel.c++ | 31 ++++++++-- src/workerd/api/tests/BUILD.bazel | 9 +++ .../diagnostics-channel-tail-test.wd-test | 26 ++++++++ .../tests/diagnostics-channel-tail-trigger.js | 27 ++++++++ .../api/tests/diagnostics-channel-tail.js | 62 +++++++++++++++++++ 5 files changed, 149 insertions(+), 6 deletions(-) create mode 100644 src/workerd/api/tests/diagnostics-channel-tail-test.wd-test create mode 100644 src/workerd/api/tests/diagnostics-channel-tail-trigger.js create mode 100644 src/workerd/api/tests/diagnostics-channel-tail.js diff --git a/src/workerd/api/node/diagnostics-channel.c++ b/src/workerd/api/node/diagnostics-channel.c++ index 86a5e16ef71..bb72b0f3b58 100644 --- a/src/workerd/api/node/diagnostics-channel.c++ +++ b/src/workerd/api/node/diagnostics-channel.c++ @@ -29,7 +29,8 @@ void Channel::publish(jsg::Lock& js, jsg::Value message) { auto& context = IoContext::current(); KJ_IF_SOME(tracer, context.getWorkerTracer()) { - js.tryCatch([&]() { + // clang-format off + JSG_TRY(js) { jsg::Serializer ser(js, jsg::Serializer::Options{ .omitHeader = false, @@ -42,11 +43,29 @@ void Channel::publish(jsg::Lock& js, jsg::Value message) { "transferred ArrayBuffer instances"); tracer.addDiagnosticChannelEvent( context.getInvocationSpanContext(), context.now(), name.toString(js), kj::mv(tmp.data)); - }, [&](jsg::Value&& exception) { - jsg::JsValue jsException(exception.getHandle(js)); - tracer.addException(context.getInvocationSpanContext(), context.now(), kj::str("Error"), - kj::str("Failed to publish diagnostics channel message: ", jsException), kj::none); - }); + } JSG_CATCH(exception KJ_UNUSED) { + // There was an error serializing the message. This is most likely due to + // the message containing a non-serializable value. That's ok, we'll just + // use a fallback message so there's at least a record that an event was + // published. + JSG_TRY(js) { + jsg::Serializer ser(js, + jsg::Serializer::Options{ + .omitHeader = false, + }); + ser.write(js, js.str("[[ Diagnostic event was not serializable ]]"_kj)); + auto tmp = ser.release(); + tracer.addDiagnosticChannelEvent( + context.getInvocationSpanContext(), context.now(), name.toString(js), kj::mv(tmp.data)); + } JSG_CATCH(exception) { + // This should not happen, but if it does, we don't want to crash. We'll + // log the error and continue. + LOG_PERIODICALLY(WARNING, "Diagnostic event fallback serialization failed: ", exception.getHandle(js)); + tracer.addException(context.getInvocationSpanContext(), context.now(), kj::str("Error"), + kj::str("Failed to publish diagnostics channel message"), kj::none); + } + } + // clang-format on } } diff --git a/src/workerd/api/tests/BUILD.bazel b/src/workerd/api/tests/BUILD.bazel index 303947a8b01..2d74a60909c 100644 --- a/src/workerd/api/tests/BUILD.bazel +++ b/src/workerd/api/tests/BUILD.bazel @@ -505,6 +505,15 @@ wd_test( ], ) +wd_test( + src = "diagnostics-channel-tail-test.wd-test", + args = ["--experimental"], + data = [ + "diagnostics-channel-tail.js", + "diagnostics-channel-tail-trigger.js", + ], +) + wd_test( size = "large", src = "global-scope-test.wd-test", diff --git a/src/workerd/api/tests/diagnostics-channel-tail-test.wd-test b/src/workerd/api/tests/diagnostics-channel-tail-test.wd-test new file mode 100644 index 00000000000..37c62e56b49 --- /dev/null +++ b/src/workerd/api/tests/diagnostics-channel-tail-test.wd-test @@ -0,0 +1,26 @@ +using Workerd = import "/workerd/workerd.capnp"; + +const unitTests :Workerd.Config = ( + services = [ + ( name = "trigger", + worker = ( + modules = [ + ( name = "worker", esModule = embed "diagnostics-channel-tail-trigger.js" ) + ], + bindings = [ + ( name = "SELF", service = "trigger" ), + ], + compatibilityFlags = ["nodejs_compat", "nodejs_compat_v2"], + streamingTails = ["tail"], + ), + ), + ( name = "tail", + worker = ( + modules = [ + ( name = "worker", esModule = embed "diagnostics-channel-tail.js" ) + ], + compatibilityFlags = ["nodejs_compat", "nodejs_compat_v2"], + ), + ), + ], +); diff --git a/src/workerd/api/tests/diagnostics-channel-tail-trigger.js b/src/workerd/api/tests/diagnostics-channel-tail-trigger.js new file mode 100644 index 00000000000..7deb9098424 --- /dev/null +++ b/src/workerd/api/tests/diagnostics-channel-tail-trigger.js @@ -0,0 +1,27 @@ +// Copyright (c) 2026 Cloudflare, Inc. +// Licensed under the Apache 2.0 license found in the LICENSE file or at: +// https://opensource.org/licenses/Apache-2.0 + +import { channel } from 'node:diagnostics_channel'; + +export default { + async fetch(request) { + // Serializable payload: should be forwarded as-is to the tail worker. + channel('test:serializable').publish({ key: 'value' }); + + // Non-serializable payload: a function cannot be structured-cloned. + // The tail worker should receive a fallback placeholder instead of an + // exception event. + channel('test:non-serializable').publish(function notCloneable() {}); + + return new Response('ok'); + }, +}; + +export const test = { + async test(ctrl, env) { + // Invoke via service binding so the fetch() handler runs in a traced + // invocation (test() handlers are not traced by the test runner). + await env.SELF.fetch('http://dummy'); + }, +}; diff --git a/src/workerd/api/tests/diagnostics-channel-tail.js b/src/workerd/api/tests/diagnostics-channel-tail.js new file mode 100644 index 00000000000..28e4ca4e3a1 --- /dev/null +++ b/src/workerd/api/tests/diagnostics-channel-tail.js @@ -0,0 +1,62 @@ +// Copyright (c) 2026 Cloudflare, Inc. +// Licensed under the Apache 2.0 license found in the LICENSE file or at: +// https://opensource.org/licenses/Apache-2.0 + +// Streaming tail worker that collects events from the trigger worker and +// verifies that non-serializable diagnostics_channel payloads produce a +// fallback placeholder message instead of a trace exception. + +import * as assert from 'node:assert'; + +let allEvents = []; + +export default { + tailStream(event) { + allEvents.push(event.event); + return (event) => { + allEvents.push(event.event); + }; + }, +}; + +export const test = { + async test() { + // Brief wait for streaming tail events to arrive (see tail-worker-test.js). + await scheduler.wait(50); + + const diagEvents = allEvents.filter((e) => e.type === 'diagnosticChannel'); + + // 1. Serializable message is forwarded as-is. + const serEvent = diagEvents.find((e) => e.channel === 'test:serializable'); + assert.ok(serEvent, 'expected test:serializable diagnostic channel event'); + assert.deepStrictEqual(serEvent.message, { key: 'value' }); + + // 2. Non-serializable message gets a fallback placeholder string. + const nonSerEvent = diagEvents.find( + (e) => e.channel === 'test:non-serializable' + ); + assert.ok( + nonSerEvent, + 'expected test:non-serializable diagnostic channel event' + ); + assert.strictEqual( + nonSerEvent.message, + '[[ Diagnostic event was not serializable ]]' + ); + + // 3. No trace exceptions from serialization failures. + // Before the fix, the runtime would call tracer.addException() with + // "Failed to publish diagnostics channel message: ..." for every + // non-serializable payload. + const serFailureExceptions = allEvents.filter( + (e) => + e.type === 'exception' && + e.message?.includes?.('Failed to publish diagnostics channel message') + ); + assert.strictEqual( + serFailureExceptions.length, + 0, + 'non-serializable diagnostic channel messages must not produce trace exceptions' + ); + }, +}; From c34e603dd3b476c4b3cb963afee1babc4d4b160b Mon Sep 17 00:00:00 2001 From: Felix Hanau Date: Fri, 3 Jul 2026 20:10:40 -0400 Subject: [PATCH 043/101] [build] Use -Wl,--lto-O0 when linking tests As part of keeping linker options in one place to avoid duplication --- build/config/BUILD.bazel | 11 ------- build/deps/BUILD | 58 ++++++++++++++++++++++++++++++++++++ build/kj_test.bzl | 9 +----- build/wd_cc_benchmark.bzl | 8 ++--- build/wd_cc_binary.bzl | 18 +---------- build/wd_rust_binary.bzl | 19 ++++-------- build/wd_rust_crate.bzl | 8 ++--- build/wd_rust_proc_macro.bzl | 2 ++ 8 files changed, 73 insertions(+), 60 deletions(-) diff --git a/build/config/BUILD.bazel b/build/config/BUILD.bazel index 8e340e64719..a9125185cbc 100644 --- a/build/config/BUILD.bazel +++ b/build/config/BUILD.bazel @@ -20,14 +20,3 @@ config_setting( values = {"define": "never_match=true"}, # This will never match visibility = ["//visibility:public"], ) - -# Whether rules_rust binaries/tests should link via cc_common.link (see the -# wd_rust_* macros). Always false standalone; an embedder may override this -# when it needs these targets to link through the cc toolchain (e.g. to pick -# up a custom allocator selected via --custom_malloc). Kept off here because -# standalone workerd builds with non-hermetic system clang. -config_setting( - name = "rust_cc_common_link", - values = {"define": "never_match=true"}, # This will never match - visibility = ["//visibility:public"], -) diff --git a/build/deps/BUILD b/build/deps/BUILD index e69de29bb2d..8b9751d77df 100644 --- a/build/deps/BUILD +++ b/build/deps/BUILD @@ -0,0 +1,58 @@ +# Bazel makes it difficult to define link options so that they can be set to different values for +# tool/test binaries and across different C++ and Rust binaries – define a common configuration for +# linker flags for workerd and downstream builds. + +load("@rules_cc//cc:cc_library.bzl", "cc_library") + +cc_library( + name = "linkopts_common", + # -dead_strip is the macOS equivalent of -ffunction-sections, -Wl,--gc-sections. + # -no_exported_symbols is used to not include the exports trie, which significantly reduces + # binary sizes. Unfortunately, the flag and the exports trie are poorly documented. Based + # on analyzing the binary sections with and without the flag, the information being removed + # consists of weak binding info, export binding info and stub bindings as described in + # http://www.newosxbook.com/articles/DYLD.html. The flag itself is described in + # https://www.wwdcnotes.com/notes/wwdc22/110362/. + # The affected sections appear to not be needed for debugging and are only used when + # external code needs to look up bindings in a binary, e.g. when loading a plugin + # (mac-specific feature). In particular, the symbol table is not affected (the name of the + # flag is misleading here). + linkopts = select({ + "@//:use_dead_strip": [ + "-Wl,-dead_strip", + "-Wl,-no_exported_symbols", + ], + "//conditions:default": [""], + }), + visibility = ["//visibility:public"], +) + +cc_library( + name = "linkopts_default", + # For test binaries, reduce thinLTO optimizations and inlining to speed up linking. This + # only has an effect if thinLTO is enabled. -Wl,--lto-O0 results in disabling cross-optimization + # entirely without affecting codegen optimizations within a translation unit. Also limit thinLTO + # parallelism since our many test binaries may lead to resource exhaustion, but are unlikely to + # be on the critical path of a build step. + # TODO(someday): We used to turn down thinLTO optimization slightly less – maybe there's a use + # case for these settings somewhere still? + # linkopts = ["-Wl,--lto-O1", "-Wl,-mllvm,-import-instr-limit=5"], + linkopts = select({ + "@platforms//os:linux": [ + "-Wl,--lto-O0", + "-Wl,--thinlto-jobs=4", + ], + "//conditions:default": [""], + }), + visibility = ["//visibility:public"], + deps = [":linkopts_common"], +) + +# link options to use with tools where we don't want to compromise performance, e.g. workerd. At +# present, this is identical to linkopts_common, but we may want to move some linker optimizations +# here in the future. +cc_library( + name = "linkopts_tool", + visibility = ["//visibility:public"], + deps = [":linkopts_common"], +) diff --git a/build/kj_test.bzl b/build/kj_test.bzl index 5e6789d295d..6d4ad136cbd 100644 --- a/build/kj_test.bzl +++ b/build/kj_test.bzl @@ -16,19 +16,12 @@ def kj_test( srcs = [src], deps = [ "@capnp-cpp//src/kj:kj-test", + "//build/deps:linkopts_default", ] + deps, linkstatic = select({ "@platforms//os:linux": 0, "//conditions:default": 1, }), - # For test binaries, reduce thinLTO optimizations and inlining to speed up linking. This - # only has an effect if thinLTO is enabled. Also apply dead_strip on macOS to manage binary - # sizes. - linkopts = select({ - "@platforms//os:linux": ["-Wl,--lto-O1", "-Wl,-mllvm,-import-instr-limit=5"], - "@//:use_dead_strip": ["-Wl,-dead_strip", "-Wl,-no_exported_symbols"], - "//conditions:default": [""], - }), data = data, tags = tags, target_compatible_with = select({ diff --git a/build/wd_cc_benchmark.bzl b/build/wd_cc_benchmark.bzl index a1caa295a9c..9600ccd02c6 100644 --- a/build/wd_cc_benchmark.bzl +++ b/build/wd_cc_benchmark.bzl @@ -4,7 +4,6 @@ load("@rules_cc//cc:cc_test.bzl", "cc_test") def wd_cc_benchmark( name, - linkopts = [], deps = [], tags = [], visibility = None, @@ -21,14 +20,13 @@ def wd_cc_benchmark( "@platforms//os:linux": 0, "//conditions:default": 1, }), - linkopts = linkopts + select({ - "@//:use_dead_strip": ["-Wl,-dead_strip", "-Wl,-no_exported_symbols"], - "//conditions:default": [""], - }), visibility = visibility, deps = deps + [ "@google_benchmark//:benchmark_main", "//src/workerd/tests:bench-tools", + # Use same linker flags as with test binaries – wd_cc_benchmark is used with + # microbenchmarks, which will produce relatively accurate results without thinLTO. + "//build/deps:linkopts_default", ], # use the same malloc we use for server malloc = "//src/workerd/server:malloc", diff --git a/build/wd_cc_binary.bzl b/build/wd_cc_binary.bzl index 3f8890855f4..aedc68edf95 100644 --- a/build/wd_cc_binary.bzl +++ b/build/wd_cc_binary.bzl @@ -4,7 +4,6 @@ load("@rules_cc//cc:cc_binary.bzl", "cc_binary") def wd_cc_binary( name, - linkopts = [], visibility = None, deps = [], target_compatible_with = [], @@ -13,27 +12,12 @@ def wd_cc_binary( """ cc_binary( name = name, - # -dead_strip is the macOS equivalent of -ffunction-sections, -Wl,--gc-sections. - # -no_exported_symbols is used to not include the exports trie, which significantly reduces - # binary sizes. Unfortunately, the flag and the exports trie are poorly documented. Based - # on analyzing the binary sections with and without the flag, the information being removed - # consists of weak binding info, export binding info and stub bindings as described in - # http://www.newosxbook.com/articles/DYLD.html. The flag itself is described in - # https://www.wwdcnotes.com/notes/wwdc22/110362/. - # The affected sections appear to not be needed for debugging and are only used when - # external code needs to look up bindings in a binary, e.g. when loading a plugin - # (mac-specific feature). In particular, the symbol table is not affected (the name of the - # flag is misleading here). - linkopts = linkopts + select({ - "@//:use_dead_strip": ["-Wl,-dead_strip", "-Wl,-no_exported_symbols"], - "//conditions:default": [""], - }), target_compatible_with = select({ "@//build/config:no_build": ["@platforms//:incompatible"], "//conditions:default": [], }) + target_compatible_with, visibility = visibility, - deps = deps, + deps = deps + ["//build/deps:linkopts_tool"], **kwargs ) diff --git a/build/wd_rust_binary.bzl b/build/wd_rust_binary.bzl index 7f58b635969..395fe6970bd 100644 --- a/build/wd_rust_binary.bzl +++ b/build/wd_rust_binary.bzl @@ -53,20 +53,17 @@ def wd_rust_binary( srcs = srcs, rustc_env = rustc_env, deps = deps, - link_deps = link_deps, + # wd_rust_binary is not used for the workerd production binary so far – apply default + # optimization instead of linkopts_tool + link_deps = link_deps + ["//build/deps:linkopts_default"], visibility = visibility, data = data, + experimental_use_cc_common_link = 1, proc_macro_deps = proc_macro_deps, target_compatible_with = select({ "@//build/config:no_build": ["@platforms//:incompatible"], "//conditions:default": [], }), - # Optionally link via cc_common.link so embedder-selected cc link settings - # (e.g. --custom_malloc) take effect. Off standalone (see //build/config). - experimental_use_cc_common_link = select({ - "@//build/config:rust_cc_common_link": 1, - "//conditions:default": -1, - }), ) rust_test( @@ -83,12 +80,8 @@ def wd_rust_binary( "@//build/config:no_build": ["@platforms//:incompatible"], "//conditions:default": [], }), + experimental_use_cc_common_link = 1, + link_deps = ["//build/deps:linkopts_default"], size = test_size, tags = ["no-coverage"], - # Optionally link via cc_common.link so embedder-selected cc link settings - # (e.g. --custom_malloc) take effect. Off standalone (see //build/config). - experimental_use_cc_common_link = select({ - "@//build/config:rust_cc_common_link": 1, - "//conditions:default": -1, - }), ) diff --git a/build/wd_rust_crate.bzl b/build/wd_rust_crate.bzl index a4c53eaa221..dcdeabde48f 100644 --- a/build/wd_rust_crate.bzl +++ b/build/wd_rust_crate.bzl @@ -163,15 +163,11 @@ def wd_rust_crate( } | test_env, size = test_size, tags = test_tags + ["no-coverage"], + experimental_use_cc_common_link = 1, crate_features = crate_features, deps = test_deps, + link_deps = ["//build/deps:linkopts_default"], proc_macro_deps = test_proc_macro_deps, - # Optionally link via cc_common.link so embedder-selected cc link settings - # (e.g. --custom_malloc) take effect. Off standalone (see //build/config). - experimental_use_cc_common_link = select({ - "@//build/config:rust_cc_common_link": 1, - "//conditions:default": -1, - }), target_compatible_with = select({ "@//build/config:no_build": ["@platforms//:incompatible"], "//conditions:default": [], diff --git a/build/wd_rust_proc_macro.bzl b/build/wd_rust_proc_macro.bzl index 1b5f2c80e52..8eefccca362 100644 --- a/build/wd_rust_proc_macro.bzl +++ b/build/wd_rust_proc_macro.bzl @@ -45,8 +45,10 @@ def wd_rust_proc_macro( # our tests are usually very heavy and do not support concurrent invocation "RUST_TEST_THREADS": "1", } | test_env, + experimental_use_cc_common_link = 1, tags = test_tags + ["no-coverage"], deps = test_deps, + link_deps = ["@@//deps:rust_runtime", "//build/deps:linkopts_default"], target_compatible_with = select({ "@//build/config:no_build": ["@platforms//:incompatible"], "//conditions:default": [], From 478c944bb4235ed03b9561772ef8e4d8d7345f74 Mon Sep 17 00:00:00 2001 From: Gyeongjae Choi Date: Thu, 25 Jun 2026 09:19:34 +0000 Subject: [PATCH 044/101] Enable socket operations in Python Workers --- build/deps/python.MODULE.bazel | 14 +- build/python_metadata.bzl | 18 +- src/pyodide/create_vendor_zip.py | 2 +- src/pyodide/helpers.bzl | 2 +- src/pyodide/internal/const.ts | 2 +- src/pyodide/python-entrypoint-helper.ts | 16 + src/pyodide/types/Pyodide.d.ts | 1 + src/pyodide/types/emscripten.d.ts | 4 +- src/workerd/api/pyodide/pyodide-test.c++ | 4 +- src/workerd/io/compatibility-date.capnp | 2 +- src/workerd/server/tests/python/BUILD.bazel | 19 ++ .../tests/python/sockets/socket-server.js | 81 +++++ .../tests/python/sockets/sockets.wd-test | 23 ++ .../server/tests/python/sockets/worker.py | 287 ++++++++++++++++++ .../tests/python/vendor_pkg_tests/BUILD | 2 +- 15 files changed, 453 insertions(+), 24 deletions(-) create mode 100644 src/workerd/server/tests/python/sockets/socket-server.js create mode 100644 src/workerd/server/tests/python/sockets/sockets.wd-test create mode 100644 src/workerd/server/tests/python/sockets/worker.py diff --git a/build/deps/python.MODULE.bazel b/build/deps/python.MODULE.bazel index da8ade0f564..8cee2eb6a0d 100644 --- a/build/deps/python.MODULE.bazel +++ b/build/deps/python.MODULE.bazel @@ -24,31 +24,31 @@ use_repo( "all_pyodide_wheels_20250808", "beautifulsoup4_src_0.26.0a2", "beautifulsoup4_src_0.28.2", - "beautifulsoup4_src_314.0.0", + "beautifulsoup4_src_314.0.2", "beautifulsoup4_src_development", "fastapi_src_0.26.0a2", "fastapi_src_0.28.2", "numpy_src_0.28.2", - "numpy_src_314.0.0", + "numpy_src_314.0.2", "numpy_src_development", "pyodide-0.26.0a2", "pyodide-0.28.2", - "pyodide-314.0.0", + "pyodide-314.0.2", "pyodide-snapshot-baseline-4569679fb.bin", + "pyodide-snapshot-baseline-484e61538.bin", "pyodide-snapshot-baseline-61eedf943.bin", - "pyodide-snapshot-baseline-8816cf608.bin", "pyodide-snapshot-snapshot_a6b652a95810783f5078b9a5dbd4a07c30718acb4ff724e82c25db7353dd7f2d.bin", "pyodide_0.26.0a2_2024-03-01_83.capnp.bin", "pyodide_0.28.2_2025-01-16_14.capnp.bin", - "pyodide_314.0.0_2026-06-10_6.capnp.bin", + "pyodide_314.0.2_2026-06-10_12.capnp.bin", "pyodide_dev.capnp.bin", "pytest-asyncio_src_0.26.0a2", "pytest-asyncio_src_0.28.2", - "pytest-asyncio_src_314.0.0", + "pytest-asyncio_src_314.0.2", "pytest-asyncio_src_development", "python-workers-runtime-sdk_src_0.26.0a2", "python-workers-runtime-sdk_src_0.28.2", - "python-workers-runtime-sdk_src_314.0.0", + "python-workers-runtime-sdk_src_314.0.2", "python-workers-runtime-sdk_src_development", "scipy_src_0.26.0a2", "shapely_src_0.28.2", diff --git a/build/python_metadata.bzl b/build/python_metadata.bzl index f5435ff9e4a..fd2c51045d0 100644 --- a/build/python_metadata.bzl +++ b/build/python_metadata.bzl @@ -21,8 +21,8 @@ PYODIDE_VERSIONS = [ "sha256": "c9f6dd067d119e50850849f7428e3c636ecbc2684a0d2ff992f3bd48a1062b6c", }, { - "version": "314.0.0", - "sha256": "a4a568ca2ee0c8dcaaff00a714b42613a3298a7efb85b9e682a27bba0fce81f9", + "version": "314.0.2", + "sha256": "86e3d5e0cbd39b1def1e424b3f1abdcc9edc66ae200fa5280ae8825bf71799ec", }, ] @@ -170,17 +170,17 @@ BUNDLE_VERSION_INFO = _make_bundle_version_info([ ], }, { - "name": "314.0.0", - "pyodide_version": "314.0.0", + "name": "314.0.2", + "pyodide_version": "314.0.2", "pyodide_date": "2026-06-10", - "backport": "6", - "integrity": "sha256-TlRjur4ijCdSOVGHvZT7/m7SAEb2lc2cQqADSvKvzJY=", + "backport": "12", + "integrity": "sha256-Q8miBQHo7Gqys4rDVoCAIuHKRPwUaCcnrLnbplfgWRI=", "flag": "pythonWorkers20260610", "enable_flag_name": "python_workers_20260610", "emscripten_version": "5.0.3", "python_version": "3.14.2", - "baseline_snapshot": "baseline-8816cf608.bin", - "baseline_snapshot_hash": "8816cf608779af2529000ff21292019d387591abe9b86c1287b120cb25447cb0", + "baseline_snapshot": "baseline-484e61538.bin", + "baseline_snapshot_hash": "484e6153873eea75f6e63476bc8972b62bb374d8113531ad9782505be454137a", "vendored_packages_for_tests": VENDORED_VERSION_INDEPENDENT + [ { "name": "numpy", @@ -190,7 +190,7 @@ BUNDLE_VERSION_INFO = _make_bundle_version_info([ ], }, { - "real_pyodide_version": "314.0.0", + "real_pyodide_version": "314.0.2", "name": "development", "pyodide_version": "dev", "pyodide_date": "dev", diff --git a/src/pyodide/create_vendor_zip.py b/src/pyodide/create_vendor_zip.py index d19abb60dd9..314c60fbe46 100755 --- a/src/pyodide/create_vendor_zip.py +++ b/src/pyodide/create_vendor_zip.py @@ -35,7 +35,7 @@ PYODIDE_INDEX_VERSIONS: dict[PyVer, str] = { "3.12": "0.27.7", "3.13": "0.28.3", - "3.14": "314.0.0", + "3.14": "314.0.2", } diff --git a/src/pyodide/helpers.bzl b/src/pyodide/helpers.bzl index d13d9065d01..fc096ab7df5 100644 --- a/src/pyodide/helpers.bzl +++ b/src/pyodide/helpers.bzl @@ -272,7 +272,7 @@ _REPLACEMENTS_COMMON_0_26_0_28 = [ _REPLACEMENTS = { "0.26.0a2": _REPLACEMENTS_COMMON + _REPLACEMENTS_COMMON_0_26_0_28, "0.28.2": _REPLACEMENTS_COMMON + _REPLACEMENTS_COMMON_0_26_0_28, - "314.0.0": _REPLACEMENTS_COMMON + [ + "314.0.2": _REPLACEMENTS_COMMON + [ # for 314 or later, pyodide.asm.mjs is es6 module [ "export default _createPyodideModule;", diff --git a/src/pyodide/internal/const.ts b/src/pyodide/internal/const.ts index 6b657c0a49d..19d3e1c8cfc 100644 --- a/src/pyodide/internal/const.ts +++ b/src/pyodide/internal/const.ts @@ -5,5 +5,5 @@ export const PyodideVersion = { V0_26_0a2: '0.26.0a2', V0_28_2: '0.28.2', - V314_0_0: '314.0.0', + V314_0_2: '314.0.2', } as const; diff --git a/src/pyodide/python-entrypoint-helper.ts b/src/pyodide/python-entrypoint-helper.ts index dcc3d33e73e..0c9dd274936 100644 --- a/src/pyodide/python-entrypoint-helper.ts +++ b/src/pyodide/python-entrypoint-helper.ts @@ -246,6 +246,20 @@ function disabledLoadPackage(): never { throw new PythonWorkersInternalError('pyodide.loadPackage is disabled'); } +/** + * Initialize socket operation for Pyodide. + */ +async function maybeInitializeNodeSockFS(pyodide: Pyodide): Promise { + if (!pyodide._api.initializeNodeSockFS) { + // This API exists only in Pyodide 314.0.0 and later. + return; + } + + await pyodide._api.initializeNodeSockFS( + get_pyodide_entrypoint_helper().cloudflareSocketsModule.connect + ); +} + async function setupPatches(pyodide: Pyodide): Promise { await enterJaegerSpan('setup_patches', async () => { pyodide.loadPackage = disabledLoadPackage; @@ -259,6 +273,8 @@ async function setupPatches(pyodide: Pyodide): Promise { pyodide.registerJsModule('_cloudflare_compat_flags', COMPATIBILITY_FLAGS); + await maybeInitializeNodeSockFS(pyodide); + // Inject modules that enable JS features to be used idiomatically from Python. await injectWorkersApi(pyodide); }); diff --git a/src/pyodide/types/Pyodide.d.ts b/src/pyodide/types/Pyodide.d.ts index db113a74582..f63900828a7 100644 --- a/src/pyodide/types/Pyodide.d.ts +++ b/src/pyodide/types/Pyodide.d.ts @@ -26,6 +26,7 @@ interface PyModule { interface Pyodide { _module: Module; + _api: API; runPython: ( code: string, opts?: { globals?: PyDict; filename?: string } diff --git a/src/pyodide/types/emscripten.d.ts b/src/pyodide/types/emscripten.d.ts index a91ddc9822e..a085d00ebcb 100644 --- a/src/pyodide/types/emscripten.d.ts +++ b/src/pyodide/types/emscripten.d.ts @@ -37,7 +37,7 @@ interface API { stdout?: (a: string) => void, stderr?: (a: string) => void ) => void; - version: '0.26.0a2' | '0.28.2' | '314.0.0'; + version: '0.26.0a2' | '0.28.2' | '314.0.2'; pyodide_base: { pyimport_impl: PyCallable; }; @@ -47,6 +47,8 @@ interface API { // Callback invoked when Pyodide encounters a fatal error. Setting this allows // the runtime to handle fatal errors (e.g., by condemning the isolate). on_fatal?: (error: any) => void; + // exists from 314.0.0 + initializeNodeSockFS?: (connect: unknown) => void | Promise; } interface LDSO { diff --git a/src/workerd/api/pyodide/pyodide-test.c++ b/src/workerd/api/pyodide/pyodide-test.c++ index 33cef23862e..0c5d644cf19 100644 --- a/src/workerd/api/pyodide/pyodide-test.c++ +++ b/src/workerd/api/pyodide/pyodide-test.c++ @@ -51,14 +51,14 @@ KJ_TEST("getPythonSnapshotRelease") { featureFlags.setPythonWorkers20260610(true); { auto res = KJ_ASSERT_NONNULL(getPythonSnapshotRelease(featureFlags)); - KJ_ASSERT(res.getPyodide() == "314.0.0"); + KJ_ASSERT(res.getPyodide() == "314.0.2"); KJ_ASSERT(res.getFlagName() == "pythonWorkers20260610"); } featureFlags.setPythonWorkersDevPyodide(false); { auto res = KJ_ASSERT_NONNULL(getPythonSnapshotRelease(featureFlags)); - KJ_ASSERT(res.getPyodide() == "314.0.0"); + KJ_ASSERT(res.getPyodide() == "314.0.2"); KJ_ASSERT(res.getFlagName() == "pythonWorkers20260610"); } diff --git a/src/workerd/io/compatibility-date.capnp b/src/workerd/io/compatibility-date.capnp index fcfe4079550..caf148fecd9 100644 --- a/src/workerd/io/compatibility-date.capnp +++ b/src/workerd/io/compatibility-date.capnp @@ -1572,8 +1572,8 @@ struct CompatibilityFlags @0x8f8c1b68151b6cef { $compatDisableFlag("no_python_workers_20260610") $pythonSnapshotRelease $experimental; + # Enables Python Workers using Pyodide 314.0.2 (CPython 3.14.2, Emscripten 5.0.3). - # Enables Python Workers using Pyodide 314.0.0 (CPython 3.14.2, Emscripten 5.0.3). enableNodeJsInspectorLocalDev @180 :Bool $compatEnableFlag("enable_nodejs_inspector_local_dev") $experimental; diff --git a/src/workerd/server/tests/python/BUILD.bazel b/src/workerd/server/tests/python/BUILD.bazel index ea36bce71ed..d9fd1ce028d 100644 --- a/src/workerd/server/tests/python/BUILD.bazel +++ b/src/workerd/server/tests/python/BUILD.bazel @@ -1,3 +1,4 @@ +load("@aspect_rules_js//js:defs.bzl", "js_binary") load("@rules_shell//shell:sh_test.bzl", "sh_test") load("//src/workerd/server/tests/python:py_wd_test.bzl", "py_wd_test", "python_test_setup") load("//src/workerd/server/tests/python/vendor_pkg_tests:vendor_test.bzl", "vendored_py_wd_test") @@ -70,6 +71,24 @@ py_wd_test("pth-file") py_wd_test("metadata-read-overflow") +js_binary( + name = "socket-server", + entry_point = "sockets/socket-server.js", +) + +py_wd_test( + "sockets", + # sidecar and python_snapshot_test currently cannot be used together + make_snapshot = False, + sidecar = ":socket-server", + sidecar_port_bindings = ["PYTHON_SOCKET_SERVER_PORT"], + skip_python_flags = [ + # Older versions does not support sockets + "0.26.0a2", + "0.28.2", + ], +) + # Shell-driven test for aborting the isolate on a Python fatal error. The Python worker # triggers a fatal error which calls abortIsolate(), terminating the workerd process. The shell # script verifies the expected fatal output and non-zero exit. diff --git a/src/workerd/server/tests/python/sockets/socket-server.js b/src/workerd/server/tests/python/sockets/socket-server.js new file mode 100644 index 00000000000..cfcba06c6e8 --- /dev/null +++ b/src/workerd/server/tests/python/sockets/socket-server.js @@ -0,0 +1,81 @@ +// Copyright (c) 2026 Cloudflare, Inc. +// Licensed under the Apache 2.0 license found in the LICENSE file or at: +// https://opensource.org/licenses/Apache-2.0 +import net from 'node:net'; + +const server = net.createServer((socket) => { + let buffer = Buffer.alloc(0); + let pendingCount = null; + + socket.on('data', (data) => { + buffer = Buffer.concat([buffer, data]); + + while (true) { + if (pendingCount !== null) { + if (buffer.length < pendingCount) { + return; + } + + buffer = buffer.subarray(pendingCount); + socket.write(`COUNTED ${pendingCount}\n`); + socket.end(); + return; + } + + const newline = buffer.indexOf(0x0a); + if (newline === -1) { + return; + } + + const line = buffer.subarray(0, newline).toString('utf8'); + buffer = buffer.subarray(newline + 1); + + if (line.startsWith('BASIC ')) { + socket.write(`echo: ${line.slice('BASIC '.length)}\n`); + socket.end(); + return; + } + + if (line.startsWith('ECHO ')) { + socket.write(`${line.slice('ECHO '.length)}\n`); + continue; + } + + if (line === 'BYE') { + socket.end(); + return; + } + + if (line === 'FINAL') { + socket.write('final message\n'); + socket.end(); + return; + } + + if (line === 'LINES') { + socket.write('line1\nline2\nline3\n'); + socket.end(); + return; + } + + if (line.startsWith('COUNT ')) { + pendingCount = Number(line.slice('COUNT '.length)); + continue; + } + + if (line.startsWith('SEND ')) { + const size = Number(line.slice('SEND '.length)); + socket.write(Buffer.alloc(size, 'y')); + socket.end(); + return; + } + + socket.destroy(new Error(`unknown command: ${line}`)); + return; + } + }); +}); + +server.listen(0, process.env.SIDECAR_HOSTNAME, () => { + console.log(`PYTHON_SOCKET_SERVER_PORT=${server.address().port}`); +}); diff --git a/src/workerd/server/tests/python/sockets/sockets.wd-test b/src/workerd/server/tests/python/sockets/sockets.wd-test new file mode 100644 index 00000000000..eebac80e4e7 --- /dev/null +++ b/src/workerd/server/tests/python/sockets/sockets.wd-test @@ -0,0 +1,23 @@ +using Workerd = import "/workerd/workerd.capnp"; + +const unitTests :Workerd.Config = ( + services = [ + ( name = "python-sockets", + worker = ( + modules = [ + (name = "worker.py", pythonModule = embed "worker.py") + ], + compatibilityFlags = [ + %PYTHON_FEATURE_FLAGS, + ], + bindings = [ + (name = "SIDECAR_HOSTNAME", fromEnvironment = "SIDECAR_HOSTNAME"), + (name = "PYTHON_SOCKET_SERVER_PORT", fromEnvironment = "PYTHON_SOCKET_SERVER_PORT"), + ], + ) + ), + ( name = "internet", + network = ( allow = ["private"] ) + ), + ], +); diff --git a/src/workerd/server/tests/python/sockets/worker.py b/src/workerd/server/tests/python/sockets/worker.py new file mode 100644 index 00000000000..e7f5e5db09e --- /dev/null +++ b/src/workerd/server/tests/python/sockets/worker.py @@ -0,0 +1,287 @@ +import asyncio +import socket + +from workers import WorkerEntrypoint + + +def recv_all(sock): + chunks = [] + while True: + chunk = sock.recv(4096) + if chunk == b"": + break + chunks.append(chunk) + return b"".join(chunks) + + +def recv_line(sock): + data = bytearray() + while not data.endswith(b"\n"): + chunk = sock.recv(1) + if chunk == b"": + break + data.extend(chunk) + return bytes(data) + + +class Default(WorkerEntrypoint): + def connect(self): + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.connect( + (self.env.SIDECAR_HOSTNAME, int(self.env.PYTHON_SOCKET_SERVER_PORT)) + ) + return sock + + def test_basic_send_recv(self): + message = b"BASIC hello from python\n" + expected = b"echo: hello from python\n" + + sock = self.connect() + try: + sock.sendall(message) + response = recv_line(sock) + finally: + sock.close() + + assert response == expected + + def test_multiple_send_recv(self): + sock = self.connect() + try: + for message in [b"first", b"second", b"third"]: + sock.sendall(b"ECHO " + message + b"\n") + assert recv_line(sock) == message + b"\n" + sock.sendall(b"BYE\n") + finally: + sock.close() + + def test_large_send(self): + data = b"x" * (64 * 1024) + sock = self.connect() + try: + sock.sendall(f"COUNT {len(data)}\n".encode()) + sock.sendall(data) + assert recv_line(sock) == f"COUNTED {len(data)}\n".encode() + finally: + sock.close() + + def test_large_recv(self): + size = 64 * 1024 + sock = self.connect() + try: + sock.sendall(f"SEND {size}\n".encode()) + data = recv_all(sock) + finally: + sock.close() + + assert len(data) == size + assert data == b"y" * size + + def test_recv_backpressure(self): + size = 1024 * 1024 + sock = self.connect() + try: + sock.sendall(f"SEND {size}\n".encode()) + data = recv_all(sock) + finally: + sock.close() + + assert len(data) == size + assert data == b"y" * size + + def test_partial_recv(self): + size = 1000 + sock = self.connect() + try: + sock.sendall(f"SEND {size}\n".encode()) + received = bytearray() + while len(received) < size: + chunk = sock.recv(100) + if chunk == b"": + break + assert len(chunk) <= 100 + received.extend(chunk) + finally: + sock.close() + + assert len(received) == size + assert bytes(received) == b"y" * size + + def test_socket_metadata(self): + sock = self.connect() + try: + fd_before = sock.fileno() + peer = sock.getpeername() + local = sock.getsockname() + sock.sendall(b"BASIC metadata\n") + assert recv_line(sock) == b"echo: metadata\n" + finally: + sock.close() + + assert isinstance(fd_before, int) + assert fd_before > 0 + assert peer[0] == self.env.SIDECAR_HOSTNAME + assert peer[1] == int(self.env.PYTHON_SOCKET_SERVER_PORT) + assert isinstance(local[0], str) + assert isinstance(local[1], int) + + def test_create_multiple(self): + sock1 = self.connect() + sock2 = self.connect() + try: + sock1.sendall(b"BASIC one\n") + sock2.sendall(b"BASIC two\n") + assert recv_line(sock1) == b"echo: one\n" + assert recv_line(sock2) == b"echo: two\n" + finally: + sock1.close() + sock2.close() + + def test_double_close_after_eof(self): + sock = self.connect() + sock.sendall(b"BASIC close twice\n") + assert recv_line(sock) == b"echo: close twice\n" + sock.close() + sock.close() + + def test_recv_after_remote_close(self): + sock = self.connect() + try: + sock.sendall(b"FINAL\n") + assert recv_line(sock) == b"final message\n" + finally: + sock.close() + + def test_makefile(self): + sock = self.connect() + try: + sock.sendall(b"LINES\n") + f = sock.makefile("r") + try: + assert [line.strip() for line in f.readlines()] == [ + "line1", + "line2", + "line3", + ] + finally: + f.close() + finally: + sock.close() + + def test_connection_refused(self): + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + try: + try: + sock.connect((self.env.SIDECAR_HOSTNAME, 1)) + except OSError: + pass + else: + raise AssertionError("expected connection to closed port to fail") + finally: + sock.close() + + def test_settimeout_nonblocking_and_restore(self): + sock = self.connect() + try: + sock.settimeout(0) + try: + sock.recv(1024) + except OSError: + pass + else: + raise AssertionError("expected nonblocking recv without data to fail") + + sock.settimeout(None) + sock.sendall(b"BASIC asyncio\n") + assert recv_line(sock) == b"echo: asyncio\n" + finally: + sock.close() + + def test_shutdown_after_eof(self): + for how in [socket.SHUT_RD, socket.SHUT_WR, socket.SHUT_RDWR]: + sock = self.connect() + sock.sendall(b"BASIC shutdown\n") + assert recv_line(sock) == b"echo: shutdown\n" + sock.shutdown(how) + sock.close() + + async def test_asyncio_sock_connect_recv_sendall(self): + loop = asyncio.get_event_loop() + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.setblocking(False) + try: + await loop.sock_connect( + sock, + (self.env.SIDECAR_HOSTNAME, int(self.env.PYTHON_SOCKET_SERVER_PORT)), + ) + await loop.sock_sendall(sock, b"BASIC asyncio\n") + assert await loop.sock_recv(sock, 1024) == b"echo: asyncio\n" + assert await loop.sock_recv(sock, 1024) == b"" + finally: + sock.close() + + async def test_asyncio_sock_recv_into(self): + loop = asyncio.get_event_loop() + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.setblocking(False) + try: + await loop.sock_connect( + sock, + (self.env.SIDECAR_HOSTNAME, int(self.env.PYTHON_SOCKET_SERVER_PORT)), + ) + await loop.sock_sendall(sock, b"BASIC recv into\n") + buffer = bytearray(1024) + nbytes = await loop.sock_recv_into(sock, buffer) + assert buffer[:nbytes] == b"echo: recv into\n" + assert await loop.sock_recv(sock, 1024) == b"" + finally: + sock.close() + + async def test_asyncio_open_connection(self): + reader, writer = await asyncio.open_connection( + self.env.SIDECAR_HOSTNAME, + int(self.env.PYTHON_SOCKET_SERVER_PORT), + ) + writer.write(b"BASIC stream\n") + await writer.drain() + assert await reader.readline() == b"echo: stream\n" + assert await reader.read() == b"" + writer.close() + + async def test_asyncio_concurrent_connections(self): + async def echo(message): + reader, writer = await asyncio.open_connection( + self.env.SIDECAR_HOSTNAME, + int(self.env.PYTHON_SOCKET_SERVER_PORT), + ) + writer.write(b"BASIC " + message + b"\n") + await writer.drain() + response = await reader.readline() + assert await reader.read() == b"" + writer.close() + return response + + assert await asyncio.gather(echo(b"one"), echo(b"two")) == [ + b"echo: one\n", + b"echo: two\n", + ] + + async def test(self): + self.test_basic_send_recv() + self.test_multiple_send_recv() + self.test_large_send() + self.test_large_recv() + self.test_recv_backpressure() + self.test_partial_recv() + self.test_socket_metadata() + self.test_create_multiple() + self.test_double_close_after_eof() + self.test_recv_after_remote_close() + self.test_makefile() + self.test_connection_refused() + self.test_settimeout_nonblocking_and_restore() + self.test_shutdown_after_eof() + await self.test_asyncio_sock_connect_recv_sendall() + await self.test_asyncio_sock_recv_into() + await self.test_asyncio_open_connection() + await self.test_asyncio_concurrent_connections() diff --git a/src/workerd/server/tests/python/vendor_pkg_tests/BUILD b/src/workerd/server/tests/python/vendor_pkg_tests/BUILD index 4a74647ae1b..85c31d26920 100644 --- a/src/workerd/server/tests/python/vendor_pkg_tests/BUILD +++ b/src/workerd/server/tests/python/vendor_pkg_tests/BUILD @@ -30,7 +30,7 @@ vendored_py_wd_test( make_snapshot = True, python_flags = [ "0.28.2", - "314.0.0", + "314.0.2", ], use_snapshot = "baseline", ) From 82f97da628bbff5a09ab6dc0ee0ff60f6da60ca5 Mon Sep 17 00:00:00 2001 From: Aaron Loyd Date: Tue, 7 Jul 2026 23:13:24 -0500 Subject: [PATCH 045/101] Remove cases for deprecated promise reject events These are getting removed next V8 update, so let's remove them now. --- src/workerd/jsg/promise.c++ | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/workerd/jsg/promise.c++ b/src/workerd/jsg/promise.c++ index 8c54fafc872..045a24e3a70 100644 --- a/src/workerd/jsg/promise.c++ +++ b/src/workerd/jsg/promise.c++ @@ -49,10 +49,7 @@ void UnhandledRejectionHandler::report( handledAfterRejection(js, kj::mv(promise)); return; } - case v8::PromiseRejectEvent::kPromiseRejectAfterResolved: { - break; - } - case v8::PromiseRejectEvent::kPromiseResolveAfterResolved: { + default: { break; } } From e55b437bf9dae1ba320a6fc974805b992c92b3f6 Mon Sep 17 00:00:00 2001 From: Thomas Ankcorn Date: Wed, 8 Jul 2026 12:38:48 +0000 Subject: [PATCH 046/101] feat(WO-1475): expose Durable Object id on RPC exceptions * feat(WO-1475): expose durable object id on rpc exceptions See merge request cloudflare/ew/workerd!389 --- src/workerd/api/tests/js-rpc-test.js | 26 +++++++++++++++ src/workerd/api/worker-rpc.c++ | 48 +++++++++++++++++++++++++--- src/workerd/jsg/util.c++ | 29 +++++++++++++++++ src/workerd/jsg/util.h | 10 ++++++ 4 files changed, 109 insertions(+), 4 deletions(-) diff --git a/src/workerd/api/tests/js-rpc-test.js b/src/workerd/api/tests/js-rpc-test.js index a5f237dc121..6d01fa145d0 100644 --- a/src/workerd/api/tests/js-rpc-test.js +++ b/src/workerd/api/tests/js-rpc-test.js @@ -574,6 +574,10 @@ export class MyActor extends DurableObject { return this.#counter; } + async throwingMethod() { + throw new Error("ACTOR METHOD THREW"); + } + async doCallbackBlockingConcurrency() { // Check that we can receive RPC callbacks during blockConcurrencyWhile(), if they are from // an RPC running inside the block. This verifies that the critical section is captured @@ -1819,6 +1823,28 @@ export let testExceptionProperties = { }, }; +export let testDurableObjectExceptionProperties = { + async test(controller, env, ctx) { + let id = env.MyActor.idFromName("exception-properties"); + try { + await env.MyActor.get(id).throwingMethod(); + assert.fail("expected actor RPC to throw"); + } catch (e) { + assert.strictEqual(e.remote, true); + assert.strictEqual(e.message, "ACTOR METHOD THREW"); + assert.strictEqual(e.durableObjectId, id.toString()); + } + + try { + await env.MyService.throwingMethod(); + assert.fail("expected service RPC to throw"); + } catch (e) { + assert.strictEqual(e.remote, true); + assert.strictEqual(e.durableObjectId, undefined); + } + }, +}; + // Test that get(), put(), and delete() are valid RPC method names, not hijacked by Fetcher. export let canUseGetPutDelete = { async test(controller, env, ctx) { diff --git a/src/workerd/api/worker-rpc.c++ b/src/workerd/api/worker-rpc.c++ index fade4977092..21bb51f99b1 100644 --- a/src/workerd/api/worker-rpc.c++ +++ b/src/workerd/api/worker-rpc.c++ @@ -16,6 +16,39 @@ namespace workerd::api { +namespace { + +kj::Maybe getCurrentDurableObjectId() { + KJ_IF_SOME(context, IoContext::tryCurrent()) { + KJ_IF_SOME(actor, context.getActor()) { + KJ_IF_SOME(id, actor.getId().tryGet>()) { + return id->toString(); + } + } + } + return kj::none; +} + +void maybeAddDurableObjectId(kj::Exception& exception, kj::Maybe durableObjectId) { + if (jsg::isExceptionFromInputGateBroken(exception.getDescription())) return; + KJ_IF_SOME(id, durableObjectId) { + jsg::addDurableObjectId(exception, id); + } +} + +// callPipeline rejects from the JS error path before js.exceptionToKj() creates a KJ exception, +// so this overload annotates the live Error object directly. +void maybeAddDurableObjectId( + jsg::Lock& js, jsg::Value& error, kj::Maybe durableObjectId) { + if (!error.getHandle(js)->IsObject()) return; + KJ_IF_SOME(id, durableObjectId) { + jsg::check(error.getHandle(js).As()->Set(js.v8Context(), + jsg::v8StrIntern(js.v8Isolate, "durableObjectId"_kj), jsg::v8Str(js.v8Isolate, id))); + } +} + +} // namespace + capnp::Orphan> RpcSerializerExternalHandler::build( capnp::Orphanage orphanage) { auto result = orphanage.newOrphan>(externals.size()); @@ -976,7 +1009,8 @@ class JsRpcTargetBase: public rpc::JsRpcTarget::Server { // makeReentryCallback() to guard against the possibility that the IoContext is canceled before // or during a call. JsRpcTargetBase(IoContext& ctx, MayOutliveIncomingRequest) - : enterIsolateAndCall(ctx.makeReentryCallback( + : durableObjectId(getCurrentDurableObjectId()), + enterIsolateAndCall(ctx.makeReentryCallback( [this, &ctx](Worker::Lock& lock, CallContext callContext) { return callImpl(lock, ctx, callContext); })), @@ -985,7 +1019,8 @@ class JsRpcTargetBase: public rpc::JsRpcTarget::Server { // Constructor use by EntrypointJsRpcTarget, which is revoked and destroyed before the IoContext // can possibly be canceled. It can just use ctx.run(). JsRpcTargetBase(IoContext& ctx, CantOutliveIncomingRequest) - : enterIsolateAndCall([this, &ctx](CallContext callContext) { + : durableObjectId(getCurrentDurableObjectId()), + enterIsolateAndCall([this, &ctx](CallContext callContext) { // Note: No need to topUpActor() since this is the start of a top-level request, so the // actor will already have been topped up by IncomingRequest::delivered(). return ctx.run([this, &ctx, callContext](Worker::Lock& lock) mutable { @@ -1019,7 +1054,8 @@ class JsRpcTargetBase: public rpc::JsRpcTarget::Server { co_await kj::yield(); // Try to execute the requested method. - co_return co_await enterIsolateAndCall(callContext).catch_([](kj::Exception&& e) { + co_return co_await enterIsolateAndCall(callContext).catch_([this](kj::Exception&& e) { + maybeAddDurableObjectId(e, durableObjectId.map([](const kj::String& id) { return id.asPtr(); })); if (jsg::isTunneledException(e.getDescription())) { // Annotate exceptions in RPC worker calls as remote exceptions. auto description = jsg::stripRemoteExceptionPrefix(e.getDescription()); @@ -1053,6 +1089,8 @@ class JsRpcTargetBase: public rpc::JsRpcTarget::Server { private: virtual void maybeSetJsRpcInfo(IoContext& ctx, const kj::ConstString& methodNameForTrace) = 0; + kj::Maybe durableObjectId; + // Function which enters the isolate lock and IoContext and then invokes callImpl(). Created // using IoContext::makeReentryCallback(). kj::Function(CallContext callContext)> enterIsolateAndCall; @@ -1191,7 +1229,9 @@ class JsRpcTargetBase: public rpc::JsRpcTarget::Server { // as a result of the promise being rejected). This will implicitly dispose the param // stubs. }), - ctx.addFunctor([callPipelineFulfillerRef](jsg::Lock& js, jsg::Value&& error) { + ctx.addFunctor([callPipelineFulfillerRef, durableObjectId = getCurrentDurableObjectId()]( + jsg::Lock& js, jsg::Value&& error) { + maybeAddDurableObjectId(js, error, durableObjectId.map([](const kj::String& id) { return id.asPtr(); })); // If we set up a `callPipeline` early, we have to make sure it propagates the error. // (Otherwise we get a PromiseFulfiller error instead, which is pretty useless...) KJ_IF_SOME(cpf, callPipelineFulfillerRef) { diff --git a/src/workerd/jsg/util.c++ b/src/workerd/jsg/util.c++ index cf998174b57..2ff326c8f47 100644 --- a/src/workerd/jsg/util.c++ +++ b/src/workerd/jsg/util.c++ @@ -155,6 +155,15 @@ bool setDurableObjectResetError(v8::Isolate* isolate, v8::Local& exce return jsg::check(obj->Set(isolate->GetCurrentContext(), jsg::v8StrIntern(isolate, "durableObjectReset"_kj), v8::True(isolate))); } + +bool setDurableObjectIdError( + v8::Isolate* isolate, v8::Local& exception, kj::StringPtr durableObjectId) { + KJ_ASSERT(exception->IsObject()); + auto obj = exception.As(); + return jsg::check(obj->Set(isolate->GetCurrentContext(), + jsg::v8StrIntern(isolate, "durableObjectId"_kj), jsg::v8Str(isolate, durableObjectId))); +} + struct DecodedException { v8::Local handle; bool isInternal; @@ -221,6 +230,10 @@ DecodedException decodeTunneledException( if (result.isDurableObjectReset) { setDurableObjectResetError(isolate, result.handle); } + + KJ_IF_SOME(durableObjectId, getDurableObjectId(exception)) { + setDurableObjectIdError(isolate, result.handle, durableObjectId); + } }; if (tunneledInfo.isJsgError) { @@ -325,6 +338,10 @@ DecodedException decodeTunneledException( if (tunneledInfo.isDurableObjectReset) { setDurableObjectResetError(isolate, result.handle); } + + KJ_IF_SOME(durableObjectId, getDurableObjectId(exception)) { + setDurableObjectIdError(isolate, result.handle, durableObjectId); + } } else { // For everything return a generic error with an internal error id. result.internalErrorId = makeInternalErrorId(); @@ -436,6 +453,18 @@ void addExceptionDetail(Lock& js, kj::Exception& exception, v8::Local } } +void addDurableObjectId(kj::Exception& exception, kj::StringPtr durableObjectId) { + exception.setDetail(DURABLE_OBJECT_EXCEPTION_METADATA_DETAIL_ID, + kj::heapArray(durableObjectId.asBytes())); +} + +kj::Maybe getDurableObjectId(const kj::Exception& exception) { + KJ_IF_SOME(detail, exception.getDetail(DURABLE_OBJECT_EXCEPTION_METADATA_DETAIL_ID)) { + return kj::str(detail.asChars()); + } + return kj::none; +} + void addJsExceptionMetadata(Lock& js, kj::Exception& exception, v8::Local handle) { // Extract JavaScript error type and stack trace if (!handle->IsObject()) { diff --git a/src/workerd/jsg/util.h b/src/workerd/jsg/util.h index 87fe907a928..8a3f00ded48 100644 --- a/src/workerd/jsg/util.h +++ b/src/workerd/jsg/util.h @@ -109,6 +109,10 @@ constexpr kj::Exception::DetailTypeId REQUEST_NOT_DELIVERED_TO_ACTOR_DETAIL_ID = constexpr kj::Exception::DetailTypeId REQUEST_DELIVERED_TO_ACTOR_DETAIL_ID = 0x7f6e0bece261e8eeull; +// Detail type for Durable Object metadata on exceptions propagated out of actor execution. +constexpr kj::Exception::DetailTypeId DURABLE_OBJECT_EXCEPTION_METADATA_DETAIL_ID = + 0xb6e3c4156a4f9d22ull; + // Add a serialized copy of the exception value to the KJ exception, as a "detail". void addExceptionDetail(Lock& js, kj::Exception& exception, v8::Local handle); @@ -116,6 +120,12 @@ void addExceptionDetail(Lock& js, kj::Exception& exception, v8::Local // Serializes using Cap'n Proto schema defined in exception-metadata.capnp. void addJsExceptionMetadata(Lock& js, kj::Exception& exception, v8::Local handle); +// Attach the Durable Object id that produced an exception before it is propagated over RPC. +void addDurableObjectId(kj::Exception& exception, kj::StringPtr durableObjectId); + +// Read Durable Object id metadata attached by addDurableObjectId(), if present. +kj::Maybe getDurableObjectId(const kj::Exception& exception); + struct TypeErrorContext { enum Kind : uint8_t { METHOD_ARGUMENT, // has type name, member (method) name, and argument index From 9d971fd951ff62c497efad2af30db0ae6e6a38e3 Mon Sep 17 00:00:00 2001 From: Dan Carney Date: Wed, 8 Jul 2026 14:03:15 +0000 Subject: [PATCH 047/101] pyodide: disable implicit __init__.py creation for unit-test-introspection * pyodide: also disable implicit __init__.py for pack_python_packages The MR targets the gitlab branch, where pack_python_packages exists. Apply legacy_create_init = False there too for consistency. * pyodide: also disable implicit __init__.py for unit-test-frozen-sdk Address AI review: apply legacy_create_init = False to the unit-test-frozen-sdk py_test as well so it does not emit the same rules_python deprecation warning. * pyodide: disable implicit __init__.py creation for unit-test-introspection Set legacy_create_init = False on the unit-test-introspection py_test target to silence the rules_python deprecation warning about implicit __init__.py creation. See https://github.com/bazel-contrib/rules_python/issues/2945 See merge request cloudflare/ew/workerd!447 --- src/pyodide/BUILD.bazel | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/pyodide/BUILD.bazel b/src/pyodide/BUILD.bazel index 4f0fa134c20..8847f737936 100644 --- a/src/pyodide/BUILD.bazel +++ b/src/pyodide/BUILD.bazel @@ -16,6 +16,7 @@ wd_cc_capnp_library( py_binary( name = "pack_python_packages", srcs = ["pack_python_packages.py"], + legacy_create_init = False, visibility = ["//visibility:public"], ) @@ -81,6 +82,7 @@ py_test( "internal/introspection.py", "internal/test_introspection.py", ] + glob(["internal/workers-api/src/workers/*"]), + legacy_create_init = False, main = "internal/test_introspection.py", ) @@ -88,6 +90,7 @@ py_test( name = "unit-test-frozen-sdk", srcs = ["internal/test_frozen_sdk.py"], data = glob(["internal/workers-api/**"]), + legacy_create_init = False, main = "internal/test_frozen_sdk.py", target_compatible_with = ["@platforms//os:linux"], ) From 941444637f5bf5c62f6a4d8c2b33973aec5fa1d7 Mon Sep 17 00:00:00 2001 From: Sebastien Pahl Date: Wed, 8 Jul 2026 16:00:11 +0200 Subject: [PATCH 048/101] Propagate trace context to container snapshots Snapshot RPCs currently omit the active Workers span, causing backend snapshot work to start a disconnected trace that is often dropped by root sampling. Carry the current span context on both snapshot request types so Cloudchamber spans remain attached to the originating request. --- src/workerd/api/BUILD.bazel | 8 ++ src/workerd/api/container-test.c++ | 125 +++++++++++++++++++++++++++++ src/workerd/api/container.c++ | 6 ++ src/workerd/io/container.capnp | 6 ++ 4 files changed, 145 insertions(+) create mode 100644 src/workerd/api/container-test.c++ diff --git a/src/workerd/api/BUILD.bazel b/src/workerd/api/BUILD.bazel index c0daafc3668..84a87af13e5 100644 --- a/src/workerd/api/BUILD.bazel +++ b/src/workerd/api/BUILD.bazel @@ -633,6 +633,14 @@ kj_test( ], ) +kj_test( + src = "container-test.c++", + deps = [ + "//src/workerd/io", + "//src/workerd/tests:test-fixture", + ], +) + kj_test( src = "sql-toarray-oom-test.c++", deps = [ diff --git a/src/workerd/api/container-test.c++ b/src/workerd/api/container-test.c++ new file mode 100644 index 00000000000..8a7bb2db8c0 --- /dev/null +++ b/src/workerd/api/container-test.c++ @@ -0,0 +1,125 @@ +// Copyright (c) 2026 Cloudflare, Inc. +// Licensed under the Apache 2.0 license found in the LICENSE file or at: +// https://opensource.org/licenses/Apache-2.0 + +#include "container.h" + +#include +#include + +#include + +namespace workerd::api { +namespace { + +constexpr tracing::TraceId EXPECTED_TRACE_ID(0x0123456789abcdef, 0xfedcba9876543210); +constexpr tracing::SpanId EXPECTED_PARENT_SPAN_ID(0x123456789abcdef0); +constexpr tracing::TraceFlags EXPECTED_TRACE_FLAGS(0x01); + +void expectSpanContext(rpc::SpanContext::Reader reader) { + auto spanContext = tracing::SpanContext::fromCapnp(reader); + KJ_EXPECT(spanContext.getTraceId() == EXPECTED_TRACE_ID); + KJ_EXPECT(KJ_ASSERT_NONNULL(spanContext.getSpanId()) == EXPECTED_PARENT_SPAN_ID); + KJ_EXPECT(KJ_ASSERT_NONNULL(spanContext.getTraceFlags()) == EXPECTED_TRACE_FLAGS); +} + +class TracingRequestObserver final: public RequestObserver { + public: + SpanParent getSpan() override { + return SpanParent::fromSpanContext( + tracing::SpanContext(EXPECTED_TRACE_ID, EXPECTED_PARENT_SPAN_ID, EXPECTED_TRACE_FLAGS)); + } +}; + +class MockContainerServer final: public rpc::Container::Server { + public: + MockContainerServer(bool& directoryCalled, bool& containerCalled) + : directoryCalled(directoryCalled), + containerCalled(containerCalled) {} + + kj::Promise snapshotDirectory(SnapshotDirectoryContext context) override { + auto params = context.getParams(); + KJ_EXPECT(params.hasSpanContext()); + expectSpanContext(params.getSpanContext()); + KJ_EXPECT(params.getDir() == "/data"); + KJ_EXPECT(params.getName() == "directory-snapshot"); + directoryCalled = true; + + auto snapshot = context.getResults().initSnapshot(); + snapshot.setId("directory-snapshot-id"); + snapshot.setSize(123); + snapshot.setDir(params.getDir()); + snapshot.setName(params.getName()); + return kj::READY_NOW; + } + + kj::Promise snapshotContainer(SnapshotContainerContext context) override { + auto params = context.getParams(); + KJ_EXPECT(params.hasSpanContext()); + expectSpanContext(params.getSpanContext()); + KJ_EXPECT(params.getName() == "container-snapshot"); + containerCalled = true; + + auto snapshot = context.getResults().initSnapshot(); + snapshot.setId("container-snapshot-id"); + snapshot.setSize(456); + snapshot.setName(params.getName()); + return kj::READY_NOW; + } + + private: + bool& directoryCalled; + bool& containerCalled; +}; + +TestFixture makeFixture() { + return TestFixture(TestFixture::SetupParams{ + .useRealTimers = false, + .requestObserverFactory = kj::Function()>( + []() -> kj::Own { return kj::refcounted(); }), + }); +} + +KJ_TEST("Container::snapshotDirectory propagates the current span context") { + bool directoryCalled = false; + bool containerCalled = false; + auto fixture = makeFixture(); + + fixture.runInIoContext([&](const TestFixture::Environment& env) { + auto container = kj::heap( + rpc::Container::Client(kj::heap(directoryCalled, containerCalled)), + true); + auto promise = container->snapshotDirectory(env.js, + Container::DirectorySnapshotOptions{ + .dir = kj::str("/data"), + .name = kj::str("directory-snapshot"), + }); + return env.context.awaitJs(env.js, kj::mv(promise)).attach(kj::mv(container)); + }); + + KJ_EXPECT(directoryCalled); + KJ_EXPECT(!containerCalled); +} + +KJ_TEST("Container::snapshotContainer propagates the current span context") { + bool directoryCalled = false; + bool containerCalled = false; + auto fixture = makeFixture(); + + fixture.runInIoContext([&](const TestFixture::Environment& env) { + auto container = kj::heap( + rpc::Container::Client(kj::heap(directoryCalled, containerCalled)), + true); + auto promise = container->snapshotContainer(env.js, + Container::SnapshotOptions{ + .name = kj::str("container-snapshot"), + }); + return env.context.awaitJs(env.js, kj::mv(promise)).attach(kj::mv(container)); + }); + + KJ_EXPECT(!directoryCalled); + KJ_EXPECT(containerCalled); +} + +} // namespace +} // namespace workerd::api diff --git a/src/workerd/api/container.c++ b/src/workerd/api/container.c++ index 9000910ba83..79029d2cb9c 100644 --- a/src/workerd/api/container.c++ +++ b/src/workerd/api/container.c++ @@ -350,6 +350,9 @@ jsg::Promise Container::snapshotDirectory( "snapshotDirectory() requires an absolute directory path (starting with '/')."); auto req = rpcClient->snapshotDirectoryRequest(); + KJ_IF_SOME(spanContext, IoContext::current().getCurrentTraceSpan().toSpanContext()) { + spanContext.toCapnp(req.initSpanContext()); + } req.setDir(options.dir); KJ_IF_SOME(name, options.name) { @@ -380,6 +383,9 @@ jsg::Promise Container::snapshotContainer( running, Error, "snapshotContainer() cannot be called on a container that is not running."); auto req = rpcClient->snapshotContainerRequest(); + KJ_IF_SOME(spanContext, IoContext::current().getCurrentTraceSpan().toSpanContext()) { + spanContext.toCapnp(req.initSpanContext()); + } KJ_IF_SOME(name, options.name) { req.setName(name); diff --git a/src/workerd/io/container.capnp b/src/workerd/io/container.capnp index 12d53d31f61..89f0fafd221 100644 --- a/src/workerd/io/container.capnp +++ b/src/workerd/io/container.capnp @@ -93,6 +93,9 @@ interface Container @0x9aaceefc06523bca { name @1 :Text; # Optional human-friendly name. Empty string means not set. + + spanContext @2 :SpanContext; + # Trace context propagated from the Workers request. } struct ContainerSnapshot { @@ -111,6 +114,9 @@ interface Container @0x9aaceefc06523bca { struct SnapshotContainerParams { name @0 :Text; # Optional human-friendly name. Empty string means not set. + + spanContext @1 :SpanContext; + # Trace context propagated from the Workers request. } struct ExecOptions { From ded415e53245831879129d5f8e047ad3ba648988 Mon Sep 17 00:00:00 2001 From: Dan Carney Date: Wed, 8 Jul 2026 20:42:11 +0000 Subject: [PATCH 049/101] Revert "Use non-sandbox buffer to read in kv.c++" This reverts merge request !431 --- src/workerd/api/kv.c++ | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/workerd/api/kv.c++ b/src/workerd/api/kv.c++ index f421bcd5a2e..6906feb515d 100644 --- a/src/workerd/api/kv.c++ +++ b/src/workerd/api/kv.c++ @@ -648,11 +648,6 @@ jsg::Promise KvNamespace::put(jsg::Lock& js, traceContext.setTag("cloudflare.kv.query.value_type"_kjc, "text"_kjc); } KJ_CASE_ONEOF(data, kj::Array) { - // `data` aliases the V8 BackingStore via jsg::asBytes(). The async - // write() below runs after context.waitForOutputLocks() has released - // the isolate lock, relinquishing any MPK keys. Copy the data while - // we have the key. - data = kj::heapArray(data.asPtr()); expectedBodySize = static_cast(data.size()); traceContext.setTag("cloudflare.kv.query.value_type"_kjc, "ArrayBuffer"_kjc); } From d9ce147dcc4114a40a6334b14c68e4b8c78cf9e0 Mon Sep 17 00:00:00 2001 From: James M Snell Date: Thu, 9 Jul 2026 08:41:58 -0700 Subject: [PATCH 050/101] More general improvements to ts streams --- src/per_isolate/webstreams/encoding.ts | 12 +- src/per_isolate/webstreams/readable.ts | 235 +++++++++++++++-------- src/per_isolate/webstreams/strategies.ts | 59 ++++-- src/per_isolate/webstreams/transform.ts | 65 +++++-- src/per_isolate/webstreams/writable.ts | 115 +++++++---- 5 files changed, 336 insertions(+), 150 deletions(-) diff --git a/src/per_isolate/webstreams/encoding.ts b/src/per_isolate/webstreams/encoding.ts index c41ecc09c0a..3455f5bfd6a 100644 --- a/src/per_isolate/webstreams/encoding.ts +++ b/src/per_isolate/webstreams/encoding.ts @@ -40,6 +40,10 @@ const StringCoerce = String; const { TransformStream } = require('webstreams/transform'); +function isActualObject(value: unknown) { + return value != null && typeof value === 'object'; +} + // Capture TransformStream.prototype accessors at bootstrap time so // internal reads do not go through the (user-patchable) prototype chain. // Uses the same validation guard pattern as getProtoGetter in primordials.ts. @@ -77,7 +81,8 @@ class TextEncoderStream { static { assertIsTextEncoderStream = function (self: TextEncoderStream) { - if (!(#transform in self)) throw new TypeError('Illegal invocation'); + if (!isActualObject(self) || !(#transform in self)) + throw new TypeError('Illegal invocation'); }; } @@ -163,7 +168,8 @@ class TextDecoderStream { static { assertIsTextDecoderStream = function (self: TextDecoderStream) { - if (!(#decoder in self)) throw new TypeError('Illegal invocation'); + if (!isActualObject(self) || !(#decoder in self)) + throw new TypeError('Illegal invocation'); }; } @@ -229,6 +235,7 @@ class TextDecoderStream { const kEnumerable = { __proto__: null, enumerable: true }; ObjectDefineProperties(TextEncoderStream.prototype, { + __proto__: null, encoding: kEnumerable, readable: kEnumerable, writable: kEnumerable, @@ -242,6 +249,7 @@ ObjectDefineProperties(TextEncoderStream.prototype, { }); ObjectDefineProperties(TextDecoderStream.prototype, { + __proto__: null, encoding: kEnumerable, fatal: kEnumerable, ignoreBOM: kEnumerable, diff --git a/src/per_isolate/webstreams/readable.ts b/src/per_isolate/webstreams/readable.ts index 17cc9b44b2a..eb597b3a607 100644 --- a/src/per_isolate/webstreams/readable.ts +++ b/src/per_isolate/webstreams/readable.ts @@ -866,8 +866,6 @@ class ReadableStreamDefaultReader< if (!isReaderBoundToStream(this)) return; readableStreamReaderGenericRelease(this); } - - [SymbolToStringTag] = 'ReadableStreamDefaultReader'; } class ReadableStreamBYOBReader implements ReadableStreamBYOBReaderType { @@ -1030,8 +1028,6 @@ class ReadableStreamBYOBReader implements ReadableStreamBYOBReaderType { if (!isReaderBoundToStream(this)) return; readableStreamReaderGenericRelease(this); } - - [SymbolToStringTag] = 'ReadableStreamBYOBReader'; } // Stream-level state transitions shared by the controllers and (in later @@ -1133,6 +1129,10 @@ function validateAndTransferView(view: ArrayBufferView): ByteQueueEntry { }; } +let assertIsReadableStreamDefaultController: ( + self: ReadableStreamDefaultController +) => void; + class ReadableStreamDefaultController< R = unknown, > implements ReadableStreamDefaultControllerType { @@ -1150,6 +1150,13 @@ class ReadableStreamDefaultController< #cancelPromise: Promise | undefined; static { + assertIsReadableStreamDefaultController = function ( + self: ReadableStreamDefaultController + ): void { + if (!isActualObject(self) || !(#queue in self)) + throw new TypeError('Illegal invocation'); + }; + // BACKEND-DISPATCH: the chained controller helpers (one of the five // sanctioned dispatch points). Each backend wraps the previous // implementation behind its own brand check — private-brand `in` for @@ -1283,7 +1290,7 @@ class ReadableStreamDefaultController< } get desiredSize(): number | null { - if (!(#queue in this)) throw new TypeError('Illegal invocation'); + assertIsReadableStreamDefaultController(this); // null when ERRORED, 0 when CLOSED, computed while readable — including // while close-requested-but-still-draining (spec GetDesiredSize). switch (getReadableStreamGetState(this.#stream)) { @@ -1297,7 +1304,7 @@ class ReadableStreamDefaultController< } enqueue(chunk: R = undefined as R): void { - if (!(#queue in this)) throw new TypeError('Illegal invocation'); + assertIsReadableStreamDefaultController(this); if (!this.#canCloseOrEnqueue()) { throw new TypeError( 'Cannot enqueue a chunk into a stream that is closed or closing' @@ -1351,7 +1358,7 @@ class ReadableStreamDefaultController< } close(): void { - if (!(#queue in this)) throw new TypeError('Illegal invocation'); + assertIsReadableStreamDefaultController(this); if (!this.#canCloseOrEnqueue()) { throw new TypeError( 'Cannot close a stream that is already closed or closing' @@ -1366,7 +1373,7 @@ class ReadableStreamDefaultController< } error(reason: unknown = undefined): void { - if (!(#queue in this)) throw new TypeError('Illegal invocation'); + assertIsReadableStreamDefaultController(this); if (getReadableStreamGetState(this.#stream) !== 'readable') return; // Propagate to every live consumer stream (tee branches) via the // cursors' weak owner refs — no strong retention of branches. The @@ -1483,8 +1490,6 @@ class ReadableStreamDefaultController< this.#cancelAlgorithm = undefined; this.#sizeAlgorithm = undefined; } - - [SymbolToStringTag] = 'ReadableStreamDefaultController'; } // Cross-class accessors for the BYOB request/controller pairing, assigned @@ -1510,6 +1515,10 @@ let getByteControllerAutoAllocateChunkSize: ( controller: ReadableByteStreamController ) => number | undefined; +let assertIsReadableStreamBYOBRequest: ( + self: ReadableStreamBYOBRequest +) => void; + class ReadableStreamBYOBRequest implements ReadableStreamBYOBRequestType { // All null once invalidated. Per spec, EVERY respond()/ // respondWithNewView()/enqueue() invalidates the outstanding request; @@ -1519,6 +1528,13 @@ class ReadableStreamBYOBRequest implements ReadableStreamBYOBRequestType { #atLeast: number | null = null; static { + assertIsReadableStreamBYOBRequest = function ( + self: ReadableStreamBYOBRequest + ): void { + if (!isActualObject(self) || !(#view in self)) + throw new TypeError('Illegal invocation'); + }; + initializeByobRequest = (request, controller, view, atLeast) => { request.#controller = controller; request.#view = view; @@ -1537,7 +1553,7 @@ class ReadableStreamBYOBRequest implements ReadableStreamBYOBRequestType { } get view(): Uint8Array | null { - if (!(#view in this)) throw new TypeError('Illegal invocation'); + assertIsReadableStreamBYOBRequest(this); return this.#view; } @@ -1552,12 +1568,12 @@ class ReadableStreamBYOBRequest implements ReadableStreamBYOBRequestType { // old implementation's max(elementSize, atLeast) floor. null once // invalidated, mirroring the old kj::Maybe behavior. get atLeast(): number | null { - if (!(#view in this)) throw new TypeError('Illegal invocation'); + assertIsReadableStreamBYOBRequest(this); return this.#atLeast; } respond(bytesWritten: number): void { - if (!(#view in this)) throw new TypeError('Illegal invocation'); + assertIsReadableStreamBYOBRequest(this); if (this.#controller === null) { throw new TypeError('This BYOB request has been invalidated'); } @@ -1565,7 +1581,7 @@ class ReadableStreamBYOBRequest implements ReadableStreamBYOBRequestType { } respondWithNewView(view: ArrayBufferView): void { - if (!(#view in this)) throw new TypeError('Illegal invocation'); + assertIsReadableStreamBYOBRequest(this); if (this.#controller === null) { throw new TypeError('This BYOB request has been invalidated'); } @@ -1579,10 +1595,12 @@ class ReadableStreamBYOBRequest implements ReadableStreamBYOBRequestType { } byteControllerRespondWithNewView(this.#controller, view); } - - [SymbolToStringTag] = 'ReadableStreamBYOBRequest'; } +let assertIsReadableByteStreamController: ( + self: ReadableByteStreamController +) => void; + class ReadableByteStreamController implements ReadableByteStreamControllerType { #stream: ReadableStream; #queue: StreamQueueType; @@ -1608,6 +1626,13 @@ class ReadableByteStreamController implements ReadableByteStreamControllerType { return isActualObject(value) && #queue in value; }; + assertIsReadableByteStreamController = function ( + self: ReadableByteStreamController + ): void { + if (!isByteStreamController(self)) + throw new TypeError('Illegal invocation'); + }; + // Chain the controller dispatch helpers. The default controller's // static block (which runs earlier in module evaluation) assigned the // initial implementations; we wrap them. Note the two classes' #queue @@ -1773,7 +1798,7 @@ class ReadableByteStreamController implements ReadableByteStreamControllerType { } get desiredSize(): number | null { - if (!(#queue in this)) throw new TypeError('Illegal invocation'); + assertIsReadableByteStreamController(this); switch (getReadableStreamGetState(this.#stream)) { case 'errored': return null; @@ -1785,7 +1810,7 @@ class ReadableByteStreamController implements ReadableByteStreamControllerType { } get byobRequest(): ReadableStreamBYOBRequestType | null { - if (!(#queue in this)) throw new TypeError('Illegal invocation'); + assertIsReadableByteStreamController(this); if (this.#byobRequest === null) { // Zero-copy is only unambiguous with exactly one consumer, and only // when it has a head pull-into descriptor (from a BYOB read, or @@ -1815,7 +1840,7 @@ class ReadableByteStreamController implements ReadableByteStreamControllerType { } enqueue(chunk: ArrayBufferView): void { - if (!(#queue in this)) throw new TypeError('Illegal invocation'); + assertIsReadableByteStreamController(this); if (!this.#canCloseOrEnqueue()) { throw new TypeError( 'Cannot enqueue a chunk into a stream that is closed or closing' @@ -1869,7 +1894,7 @@ class ReadableByteStreamController implements ReadableByteStreamControllerType { } close(): void { - if (!(#queue in this)) throw new TypeError('Illegal invocation'); + assertIsReadableByteStreamController(this); if (!this.#canCloseOrEnqueue()) { throw new TypeError( 'Cannot close a stream that is already closed or closing' @@ -1913,7 +1938,7 @@ class ReadableByteStreamController implements ReadableByteStreamControllerType { } error(reason: unknown = undefined): void { - if (!(#queue in this)) throw new TypeError('Illegal invocation'); + assertIsReadableByteStreamController(this); if (getReadableStreamGetState(this.#stream) !== 'readable') return; this.#invalidateByobRequest(); // Branch propagation — see the default controller's error() for why. @@ -2149,8 +2174,6 @@ class ReadableByteStreamController implements ReadableByteStreamControllerType { this.#pullAlgorithm = undefined; this.#cancelAlgorithm = undefined; } - - [SymbolToStringTag] = 'ReadableByteStreamController'; } function setupReadableByteStreamControllerFromUnderlyingSource( @@ -2330,8 +2353,6 @@ class ReadableStreamDrainingReader { if (!isReaderBoundToStream(this)) return; readableStreamReaderGenericRelease(this); } - - [SymbolToStringTag] = 'ReadableStreamDrainingReader'; } // The pipe (spec ReadableStreamPipeTo). Internal operations only on both @@ -2689,6 +2710,8 @@ function pipeToInternal( return promise; } +let assertIsReadableStream: (self: ReadableStream) => void; + class ReadableStream { #controller?: | ReadableStreamDefaultControllerType @@ -2724,6 +2747,10 @@ class ReadableStream { return isActualObject(value) && #state in value; }; + assertIsReadableStream = function (self: ReadableStream): void { + if (!isReadableStream(self)) throw new TypeError('Illegal invocation'); + }; + isReadableStreamLocked = (stream: ReadableStream) => { // The spec definition (a reader is attached) works unchanged for this // implementation: tee() keeps the parent locked permanently via a @@ -3124,9 +3151,7 @@ class ReadableStream { // to extract the native underlying source; absent means "queued, // use DrainingReader". One-shot: subsequent calls throw. extractNativeSource = function (this: ReadableStream): object { - if (!isReadableStream(this)) { - throw new TypeError('Illegal invocation'); - } + assertIsReadableStream(this); if (isReadableStreamLocked(this)) { throw new TypeError( 'Cannot extract a native source from a locked stream' @@ -3272,17 +3297,13 @@ class ReadableStream { } get locked(): boolean { - if (!isReadableStream(this)) { - throw new TypeError('Illegal invocation'); - } + assertIsReadableStream(this); return isReadableStreamLocked(this); } cancel(reason: unknown = undefined): Promise { try { - if (!isReadableStream(this)) { - throw new TypeError('Illegal invocation'); - } + assertIsReadableStream(this); if (isReadableStreamLocked(this)) { throw new TypeError('Cannot cancel a stream that is locked'); } @@ -3295,9 +3316,7 @@ class ReadableStream { getReader( options: { mode?: 'byob' } | null = {} ): ReadableStreamReaderType { - if (!isReadableStream(this)) { - throw new TypeError('Illegal invocation'); - } + assertIsReadableStream(this); // WebIDL dictionary conversion: null and undefined become {}, // objects have their properties read, primitives are rejected. if (options != null && !isActualObject(options)) { @@ -3323,9 +3342,7 @@ class ReadableStream { // Default = {} preserves Function.length = 1 (IDL harness check). options: StreamPipeOptions = {} ): ReadableStreamType { - if (!isReadableStream(this)) { - throw new TypeError('Illegal invocation'); - } + assertIsReadableStream(this); if (isReadableStreamLocked(this)) { throw new TypeError('Cannot pipe a stream that is locked'); } @@ -3360,9 +3377,7 @@ class ReadableStream { options: StreamPipeOptions = {} ): Promise { try { - if (!isReadableStream(this)) { - throw new TypeError('Illegal invocation'); - } + assertIsReadableStream(this); if (isReadableStreamLocked(this)) { throw new TypeError('Cannot pipe a stream that is locked'); } @@ -3415,9 +3430,7 @@ class ReadableStream { } tee(): [ReadableStream, ReadableStream] { - if (!isReadableStream(this)) { - throw new TypeError('Illegal invocation'); - } + assertIsReadableStream(this); if (isReadableStreamLocked(this)) { throw new TypeError('Cannot tee a stream that is locked'); } @@ -3555,9 +3568,7 @@ class ReadableStream { } values(options: { preventCancel?: boolean } = {}): AsyncIterableIterator { - if (!isReadableStream(this)) { - throw new TypeError('Illegal invocation'); - } + assertIsReadableStream(this); if (!isActualObject(options)) { throw new TypeError('Options must be an object'); } @@ -3583,67 +3594,121 @@ class ReadableStream { }): AsyncIterableIterator { return this.values(options); } - - [SymbolToStringTag] = 'ReadableStream'; } +const kEnumerable = { __proto__: null, enumerable: true }; + ObjectDefineProperties(ReadableStreamDefaultReader.prototype, { - closed: { enumerable: true }, - cancel: { enumerable: true }, - read: { enumerable: true }, - releaseLock: { enumerable: true }, + __proto__: null, + closed: kEnumerable, + cancel: kEnumerable, + read: kEnumerable, + releaseLock: kEnumerable, + [SymbolToStringTag]: { + __proto__: null, + value: 'ReadableStreamDefaultReader', + writable: false, + enumerable: false, + configurable: true, + }, }); ObjectDefineProperties(ReadableStreamBYOBReader.prototype, { - closed: { enumerable: true }, - cancel: { enumerable: true }, - read: { enumerable: true }, - releaseLock: { enumerable: true }, + __proto__: null, + closed: kEnumerable, + cancel: kEnumerable, + read: kEnumerable, + releaseLock: kEnumerable, + [SymbolToStringTag]: { + __proto__: null, + value: 'ReadableStreamBYOBReader', + writable: false, + enumerable: false, + configurable: true, + }, }); ObjectDefineProperties(ReadableStream, { - from: { enumerable: true }, + __proto__: null, + from: kEnumerable, }); ObjectDefineProperties(ReadableStream.prototype, { - locked: { enumerable: true }, - cancel: { enumerable: true }, - getReader: { enumerable: true }, - pipeThrough: { enumerable: true }, - pipeTo: { enumerable: true }, - tee: { enumerable: true }, - values: { enumerable: true }, - [SymbolAsyncIterator]: { enumerable: true }, + __proto__: null, + locked: kEnumerable, + cancel: kEnumerable, + getReader: kEnumerable, + pipeThrough: kEnumerable, + pipeTo: kEnumerable, + tee: kEnumerable, + values: kEnumerable, + [SymbolAsyncIterator]: kEnumerable, + [SymbolToStringTag]: { + __proto__: null, + value: 'ReadableStream', + writable: false, + enumerable: false, + configurable: true, + }, }); ObjectDefineProperties(ReadableStreamDefaultController, { - length: { value: 0 }, + __proto__: null, + length: { __proto__: null, value: 0 }, }); ObjectDefineProperties(ReadableByteStreamController, { - length: { value: 0 }, + __proto__: null, + length: { __proto__: null, value: 0 }, }); ObjectDefineProperties(ReadableStreamBYOBRequest, { - length: { value: 0 }, + __proto__: null, + length: { __proto__: null, value: 0 }, }); ObjectDefineProperties(ReadableStreamDefaultController.prototype, { - close: { enumerable: true }, - enqueue: { enumerable: true }, - error: { enumerable: true }, - desiredSize: { enumerable: true }, + __proto__: null, + close: kEnumerable, + enqueue: kEnumerable, + error: kEnumerable, + desiredSize: kEnumerable, + [SymbolToStringTag]: { + __proto__: null, + value: 'ReadableStreamDefaultController', + writable: false, + enumerable: false, + configurable: true, + }, }); ObjectDefineProperties(ReadableByteStreamController.prototype, { - close: { enumerable: true }, - enqueue: { enumerable: true }, - error: { enumerable: true }, - byobRequest: { enumerable: true }, - desiredSize: { enumerable: true }, + __proto__: null, + close: kEnumerable, + enqueue: kEnumerable, + error: kEnumerable, + byobRequest: kEnumerable, + desiredSize: kEnumerable, + [SymbolToStringTag]: { + __proto__: null, + value: 'ReadableByteStreamController', + writable: false, + enumerable: false, + configurable: true, + }, }); ObjectDefineProperties(ReadableStreamBYOBRequest.prototype, { - respond: { enumerable: true }, - respondWithNewView: { enumerable: true }, - view: { enumerable: true }, + __proto__: null, + respond: kEnumerable, + respondWithNewView: kEnumerable, + view: kEnumerable, + [SymbolToStringTag]: { + __proto__: null, + value: 'ReadableStreamBYOBRequest', + writable: false, + enumerable: false, + configurable: true, + }, }); ObjectDefineProperties(ReadableStreamDefaultController.prototype.enqueue, { - length: { value: 0 }, + __proto__: null, + length: { __proto__: null, value: 0 }, }); ObjectDefineProperties(ReadableByteStreamController.prototype.enqueue, { - length: { value: 1 }, + __proto__: null, + length: { __proto__: null, value: 1 }, }); module.exports = { diff --git a/src/per_isolate/webstreams/strategies.ts b/src/per_isolate/webstreams/strategies.ts index 56d638eab38..16e7e07aaec 100644 --- a/src/per_isolate/webstreams/strategies.ts +++ b/src/per_isolate/webstreams/strategies.ts @@ -25,9 +25,23 @@ const countSize = { }, }.size; +let assertIsByteLengthQueuingStrategy: ( + value: ByteLengthQueuingStrategy +) => void; +let assertIsCountQueuingStrategy: (value: CountQueuingStrategy) => void; + class ByteLengthQueuingStrategy { #highWaterMark: number; + static { + assertIsByteLengthQueuingStrategy = function ( + self: ByteLengthQueuingStrategy + ) { + if (!isActualObject(self) || !(#highWaterMark in self)) + throw new TypeError('Illegal invocation'); + }; + } + // The init type is optional-shaped because user input is arbitrary; the // required-member check is the explicit TypeError below. constructor(init: { highWaterMark?: number }) { @@ -44,21 +58,26 @@ class ByteLengthQueuingStrategy { } get highWaterMark(): number { - if (!(#highWaterMark in this)) throw new TypeError('Illegal invocation'); + assertIsByteLengthQueuingStrategy(this); return this.#highWaterMark; } get size(): (chunk: ArrayBufferView) => number { - if (!(#highWaterMark in this)) throw new TypeError('Illegal invocation'); + assertIsByteLengthQueuingStrategy(this); return byteLengthSize; } - - [SymbolToStringTag] = 'ByteLengthQueuingStrategy'; } class CountQueuingStrategy { #highWaterMark: number; + static { + assertIsCountQueuingStrategy = function (self: CountQueuingStrategy) { + if (!isActualObject(self) || !(#highWaterMark in self)) + throw new TypeError('Illegal invocation'); + }; + } + constructor(init: { highWaterMark?: number }) { if (!isActualObject(init)) { throw new TypeError('init must be an object'); @@ -70,26 +89,42 @@ class CountQueuingStrategy { } get highWaterMark(): number { - if (!(#highWaterMark in this)) throw new TypeError('Illegal invocation'); + assertIsCountQueuingStrategy(this); return this.#highWaterMark; } get size(): () => number { - if (!(#highWaterMark in this)) throw new TypeError('Illegal invocation'); + assertIsCountQueuingStrategy(this); return countSize; } - - [SymbolToStringTag] = 'CountQueuingStrategy'; } +const kEnumerable = { __proto__: null, enumerable: true }; + ObjectDefineProperties(ByteLengthQueuingStrategy.prototype, { - highWaterMark: { enumerable: true }, - size: { enumerable: true }, + __proto__: null, + highWaterMark: kEnumerable, + size: kEnumerable, + [SymbolToStringTag]: { + __proto__: null, + value: 'ByteLengthQueuingStrategy', + writable: false, + enumerable: false, + configurable: true, + }, }); ObjectDefineProperties(CountQueuingStrategy.prototype, { - highWaterMark: { enumerable: true }, - size: { enumerable: true }, + __proto__: null, + highWaterMark: kEnumerable, + size: kEnumerable, + [SymbolToStringTag]: { + __proto__: null, + value: 'CountQueuingStrategy', + writable: false, + enumerable: false, + configurable: true, + }, }); module.exports = { diff --git a/src/per_isolate/webstreams/transform.ts b/src/per_isolate/webstreams/transform.ts index b1a0822a28f..36684e42cfd 100644 --- a/src/per_isolate/webstreams/transform.ts +++ b/src/per_isolate/webstreams/transform.ts @@ -109,6 +109,10 @@ let transformStreamDesiredSize: ( stream: TransformStream ) => number | null; +let assertIsTransformStreamDefaultController: ( + self: TransformStreamDefaultController +) => void; + class TransformStreamDefaultController< I = unknown, O = unknown, @@ -116,6 +120,13 @@ class TransformStreamDefaultController< #stream: TransformStream | undefined; static { + assertIsTransformStreamDefaultController = function ( + self: TransformStreamDefaultController + ): void { + if (!isActualObject(self) || !(#stream in self)) + throw new TypeError('Illegal invocation'); + }; + transformStreamDefaultControllerInit = (controller, stream) => { controller.#stream = stream; }; @@ -126,7 +137,7 @@ class TransformStreamDefaultController< } get desiredSize(): number | null { - if (!(#stream in this)) throw new TypeError('Illegal invocation'); + assertIsTransformStreamDefaultController(this); const stream = this.#stream; if (stream === undefined) { throw new TypeError('Controller is not attached to a stream'); @@ -135,7 +146,7 @@ class TransformStreamDefaultController< } enqueue(chunk: O = undefined as O): void { - if (!(#stream in this)) throw new TypeError('Illegal invocation'); + assertIsTransformStreamDefaultController(this); const stream = this.#stream; if (stream === undefined) { throw new TypeError('Controller is not attached to a stream'); @@ -144,7 +155,7 @@ class TransformStreamDefaultController< } error(reason: unknown = undefined): void { - if (!(#stream in this)) throw new TypeError('Illegal invocation'); + assertIsTransformStreamDefaultController(this); const stream = this.#stream; if (stream === undefined) { throw new TypeError('Controller is not attached to a stream'); @@ -153,15 +164,13 @@ class TransformStreamDefaultController< } terminate(): void { - if (!(#stream in this)) throw new TypeError('Illegal invocation'); + assertIsTransformStreamDefaultController(this); const stream = this.#stream; if (stream === undefined) { throw new TypeError('Controller is not attached to a stream'); } transformStreamTerminate(stream); } - - [SymbolToStringTag] = 'TransformStreamDefaultController'; } class TransformStream { @@ -677,33 +686,53 @@ class TransformStream { } get readable(): ReadableStreamType { - if (!(#readable in this)) throw new TypeError('Illegal invocation'); + if (!isActualObject(this) || !(#readable in this)) + throw new TypeError('Illegal invocation'); return this.#readable; } get writable(): WritableStreamType { - if (!(#writable in this)) throw new TypeError('Illegal invocation'); + if (!isActualObject(this) || !(#writable in this)) + throw new TypeError('Illegal invocation'); return this.#writable; } - - [SymbolToStringTag] = 'TransformStream'; } +const kEnumerable = { __proto__: null, enumerable: true }; + ObjectDefineProperties(TransformStream, { - length: { value: 0 }, + __proto__: null, + length: { __proto__: null, value: 0 }, }); ObjectDefineProperties(TransformStreamDefaultController, { - length: { value: 0 }, + __proto__: null, + length: { __proto__: null, value: 0 }, }); ObjectDefineProperties(TransformStream.prototype, { - readable: { enumerable: true }, - writable: { enumerable: true }, + __proto__: null, + readable: kEnumerable, + writable: kEnumerable, + [SymbolToStringTag]: { + __proto__: null, + value: 'TransformStream', + writable: false, + enumerable: false, + configurable: true, + }, }); ObjectDefineProperties(TransformStreamDefaultController.prototype, { - desiredSize: { enumerable: true }, - enqueue: { enumerable: true }, - error: { enumerable: true }, - terminate: { enumerable: true }, + __proto__: null, + desiredSize: kEnumerable, + enqueue: kEnumerable, + error: kEnumerable, + terminate: kEnumerable, + [SymbolToStringTag]: { + __proto__: null, + value: 'TransformStreamDefaultController', + writable: false, + enumerable: false, + configurable: true, + }, }); module.exports = { diff --git a/src/per_isolate/webstreams/writable.ts b/src/per_isolate/webstreams/writable.ts index 1e570d60e0e..e737820bc86 100644 --- a/src/per_isolate/webstreams/writable.ts +++ b/src/per_isolate/webstreams/writable.ts @@ -209,6 +209,7 @@ let getWriterClosedPromiseInternal: ( // readable side's kExtractNativeSource calling convention so the C++ // TypeWrapper uses one uniform protocol for both directions. let extractNativeSink: (this: WritableStream) => object; +let assertIsWritableStream: (self: WritableStream) => void; class WritableStream { #controller?: WritableStreamDefaultController | undefined; @@ -227,6 +228,10 @@ class WritableStream { #nativeSink?: object | undefined; static { + assertIsWritableStream = function (self: WritableStream): void { + if (!isActualObject(self) || !(#state in self)) + throw new TypeError('Illegal invocation'); + }; getWritableStreamState = (stream) => stream.#state; getWritableStreamStoredError = (stream) => stream.#storedError; isWritableStreamLocked = (stream) => stream.#writer !== undefined; @@ -240,9 +245,7 @@ class WritableStream { // Mirrors extractNativeSource on the readable side: atomic // validate → extract → lock. No "disturbed" concept on writable. extractNativeSink = function (this: WritableStream): object { - if (!(#state in this)) { - throw new TypeError('Illegal invocation'); - } + assertIsWritableStream(this); if (isWritableStreamLocked(this)) { throw new TypeError( 'Cannot extract a native sink from a locked stream' @@ -591,13 +594,13 @@ class WritableStream { } get locked(): boolean { - if (!(#state in this)) throw new TypeError('Illegal invocation'); + assertIsWritableStream(this); return isWritableStreamLocked(this); } abort(reason: unknown = undefined): Promise { try { - if (!(#state in this)) throw new TypeError('Illegal invocation'); + assertIsWritableStream(this); if (isWritableStreamLocked(this)) { throw new TypeError('Cannot abort a stream that is locked'); } @@ -609,7 +612,7 @@ class WritableStream { close(): Promise { try { - if (!(#state in this)) throw new TypeError('Illegal invocation'); + assertIsWritableStream(this); if (isWritableStreamLocked(this)) { throw new TypeError('Cannot close a stream that is locked'); } @@ -623,11 +626,9 @@ class WritableStream { } getWriter(): WritableStreamDefaultWriterType { - if (!(#state in this)) throw new TypeError('Illegal invocation'); + assertIsWritableStream(this); return new WritableStreamDefaultWriter(this); } - - [SymbolToStringTag] = 'WritableStream'; } // Started-flag peek used by the stream's erroring machinery; assigned in @@ -644,6 +645,10 @@ interface QueuedWrite { size: number; } +let assertIsWritableStreamDefaultController: ( + self: WritableStreamDefaultController +) => void; + class WritableStreamDefaultController< W = unknown, > implements WritableStreamDefaultControllerType { @@ -659,6 +664,12 @@ class WritableStreamDefaultController< #abortController = new AbortController(); static { + assertIsWritableStreamDefaultController = function ( + self: WritableStreamDefaultController + ): void { + if (!isActualObject(self) || !(#stream in self)) + throw new TypeError('Illegal invocation'); + }; controllerStartedOf = (controller) => controller.#started; controllerGetDesiredSize = (controller) => { @@ -829,12 +840,12 @@ class WritableStreamDefaultController< } get signal(): AbortSignal { - if (!(#stream in this)) throw new TypeError('Illegal invocation'); + assertIsWritableStreamDefaultController(this); return AbortControllerSignalGet(this.#abortController); } error(reason: unknown = undefined): void { - if (!(#stream in this)) throw new TypeError('Illegal invocation'); + assertIsWritableStreamDefaultController(this); if (getWritableStreamState(this.#stream) !== 'writable') return; this.#errorStream(reason); } @@ -938,12 +949,13 @@ class WritableStreamDefaultController< } ); } - - [SymbolToStringTag] = 'WritableStreamDefaultController'; } // --------------------------------------------------------------------------- // WritableStreamDefaultWriter +let assertIsWritableStreamDefaultWriter: ( + self: WritableStreamDefaultWriter +) => void; class WritableStreamDefaultWriter< W = unknown, @@ -955,6 +967,13 @@ class WritableStreamDefaultWriter< static { getWriterStream = (writer) => writer.#stream; + assertIsWritableStreamDefaultWriter = function ( + self: WritableStreamDefaultWriter + ): void { + if (!isActualObject(self) || !(#stream in self)) + throw new TypeError('Illegal invocation'); + }; + getWriterReadyPromiseInternal = (writer) => { const promise = writer.#readyPromise; if (isPromise(promise)) return promise as Promise; @@ -1169,6 +1188,7 @@ class WritableStreamDefaultWriter< get closed(): Promise { try { + assertIsWritableStreamDefaultWriter(this); const promise = this.#closedPromise; if (isPromise(promise)) return promise as Promise; return (promise as PromiseWithResolversType).promise; @@ -1179,6 +1199,7 @@ class WritableStreamDefaultWriter< get ready(): Promise { try { + assertIsWritableStreamDefaultWriter(this); const promise = this.#readyPromise; if (isPromise(promise)) return promise as Promise; return (promise as PromiseWithResolversType).promise; @@ -1188,6 +1209,7 @@ class WritableStreamDefaultWriter< } get desiredSize(): number | null { + assertIsWritableStreamDefaultWriter(this); const stream = this.#stream; if (stream === undefined) { throw new TypeError('This writer has been released'); @@ -1201,6 +1223,7 @@ class WritableStreamDefaultWriter< abort(reason: unknown = undefined): Promise { try { + assertIsWritableStreamDefaultWriter(this); const stream = this.#stream; if (stream === undefined) { throw new TypeError('This writer has been released'); @@ -1213,7 +1236,7 @@ class WritableStreamDefaultWriter< close(): Promise { try { - // Brand check via private access. + assertIsWritableStreamDefaultWriter(this); if (this.#stream === undefined) { throw new TypeError('This writer has been released'); } @@ -1225,7 +1248,7 @@ class WritableStreamDefaultWriter< write(chunk: W = undefined as W): Promise { try { - if (!(#stream in this)) throw new TypeError('Illegal invocation'); + assertIsWritableStreamDefaultWriter(this); return writerWriteInternal(this, chunk); } catch (e) { return PromiseReject(e) as Promise; @@ -1233,11 +1256,9 @@ class WritableStreamDefaultWriter< } releaseLock(): void { - if (!(#stream in this)) throw new TypeError('Illegal invocation'); + assertIsWritableStreamDefaultWriter(this); writerReleaseInternal(this); } - - [SymbolToStringTag] = 'WritableStreamDefaultWriter'; } // Spec WritableStreamDefaultWriterCloseWithErrorPropagation — the pipe's @@ -1262,30 +1283,58 @@ function writerCloseWithErrorPropagation( return writerCloseInternal(writer); } +const kEnumerable = { __proto__: null, enumerable: true }; + ObjectDefineProperties(WritableStream, { - length: { value: 0 }, + __proto__: null, + length: { __proto__: null, value: 0 }, }); ObjectDefineProperties(WritableStreamDefaultController, { - length: { value: 0 }, + __proto__: null, + length: { __proto__: null, value: 0 }, }); ObjectDefineProperties(WritableStream.prototype, { - locked: { enumerable: true }, - abort: { enumerable: true }, - close: { enumerable: true }, - getWriter: { enumerable: true }, + __proto__: null, + locked: kEnumerable, + abort: kEnumerable, + close: kEnumerable, + getWriter: kEnumerable, + [SymbolToStringTag]: { + __proto__: null, + value: 'WritableStream', + writable: false, + enumerable: false, + configurable: true, + }, }); ObjectDefineProperties(WritableStreamDefaultWriter.prototype, { - closed: { enumerable: true }, - ready: { enumerable: true }, - desiredSize: { enumerable: true }, - abort: { enumerable: true }, - close: { enumerable: true }, - write: { enumerable: true }, - releaseLock: { enumerable: true }, + __proto__: null, + closed: kEnumerable, + ready: kEnumerable, + desiredSize: kEnumerable, + abort: kEnumerable, + close: kEnumerable, + write: kEnumerable, + releaseLock: kEnumerable, + [SymbolToStringTag]: { + __proto__: null, + value: 'WritableStreamDefaultWriter', + writable: false, + enumerable: false, + configurable: true, + }, }); ObjectDefineProperties(WritableStreamDefaultController.prototype, { - signal: { enumerable: true }, - error: { enumerable: true }, + __proto__: null, + signal: kEnumerable, + error: kEnumerable, + [SymbolToStringTag]: { + __proto__: null, + value: 'WritableStreamDefaultController', + writable: false, + enumerable: false, + configurable: true, + }, }); module.exports = { From a0ddf961cd3ffc7f66afa986a33e05e866dd110a Mon Sep 17 00:00:00 2001 From: James M Snell Date: Wed, 8 Jul 2026 22:06:49 +0000 Subject: [PATCH 051/101] Improve normalized expected length consistency --- src/per_isolate/webstreams/native.ts | 48 ++++++++++++++++------------ 1 file changed, 27 insertions(+), 21 deletions(-) diff --git a/src/per_isolate/webstreams/native.ts b/src/per_isolate/webstreams/native.ts index 0a77b780772..47fb85bf771 100644 --- a/src/per_isolate/webstreams/native.ts +++ b/src/per_isolate/webstreams/native.ts @@ -63,7 +63,8 @@ // cancel). // - expectedLength (non-standard extension, optional): the TOTAL bytes // the source promises to produce — a non-negative bigint or integer -// number (normalized to bigint), read once at construction. +// number that fits in a uint64 (normalized to bigint), read once at +// construction. // undefined = unknown (C++ uses chunked encoding). The conduit // enforces the exact-total contract: overflow at delivery and // underflow at source-initiated close (including min-read @@ -220,33 +221,38 @@ export interface NativeStreamHooks { errorStream: (reason: unknown) => void; } +// Maximum expectedLength: uint64 max. The value is destined for the C++ +// bridge (Content-Length is carried as uint64_t through the C++/KJ HTTP +// layer), so larger totals are not representable. Mirrors the +// FixedLengthStream constructor validation in identity.ts. +const MAX_UINT64 = 0xffff_ffff_ffff_ffffn; + // Normalizes the non-standard `expectedLength` extension property: the // TOTAL number of bytes the source promises to produce over its // lifetime (undefined = unknown; C++ then uses chunked encoding). -// Accepts a non-negative bigint or a non-negative integer number -// (normalized to bigint — totals can exceed MAX_SAFE_INTEGER). -// Shape violations are TypeErrors; range violations are RangeErrors. -// Duplicated for the queued byte controller in readable.ts. +// Accepts a bigint or integer number in [0, 2^64) — totals can exceed +// MAX_SAFE_INTEGER but must fit in a uint64. Shape violations (wrong +// type) are TypeErrors; value violations (non-integer, negative, +// > uint64 max) are RangeErrors, matching FixedLengthStream +// (identity.ts). A near-duplicate (pending consolidation; no uint64 +// cap yet) lives in readable.ts for the queued byte controller. function normalizeExpectedLength(value: unknown): bigint | undefined { if (value === undefined) return undefined; - if (typeof value === 'bigint') { - if (value < 0n) { - throw new RangeError('expectedLength must be non-negative'); - } - return value; + if (typeof value !== 'bigint' && typeof value !== 'number') { + throw new TypeError( + 'expectedLength must be a non-negative bigint or integer' + ); } - if (typeof value === 'number') { - if (NumberIsNaN(value) || value % 1 !== 0) { - throw new TypeError('expectedLength must be an integer'); - } - if (value < 0) { - throw new RangeError('expectedLength must be non-negative'); - } - return BigInt(value); + // BigInt() conversion rejects NaN, ±Infinity, and fractions + // (RangeError) naturally. The typeof gate above is what prevents + // BigInt() string/object coercion (e.g. '5' → 5n) from sneaking in. + const normalized = typeof value === 'bigint' ? value : BigInt(value); + if (normalized < 0n || normalized > MAX_UINT64) { + throw new RangeError( + 'expectedLength must be a non-negative integer that fits in a uint64' + ); } - throw new TypeError( - 'expectedLength must be a non-negative bigint or integer' - ); + return normalized; } // --------------------------------------------------------------------------- From a5a60a23e953a6075c380a7ada002b010e47a99e Mon Sep 17 00:00:00 2001 From: James M Snell Date: Wed, 8 Jul 2026 22:52:09 +0000 Subject: [PATCH 052/101] Remove respondWithNewView from native facade Won't be used --- src/per_isolate/webstreams/native.ts | 86 +++++----------------------- 1 file changed, 13 insertions(+), 73 deletions(-) diff --git a/src/per_isolate/webstreams/native.ts b/src/per_isolate/webstreams/native.ts index 47fb85bf771..04ff59ee3ad 100644 --- a/src/per_isolate/webstreams/native.ts +++ b/src/per_isolate/webstreams/native.ts @@ -20,8 +20,12 @@ // is never exposed to user code. // - pull(controller) discriminates the read mode via the controller: // controller.byobRequest !== null ⇒ BYOB read — fill the request's -// view and call respond(bytesWritten)/respondWithNewView(view), -// honoring `atLeast` (the read's minimum, in bytes). +// view and call respond(bytesWritten), honoring `atLeast` (the +// read's minimum, in bytes). respondWithNewView() is deliberately +// omitted: the native source writes into consumer-provided memory +// (KJ tryRead semantics); buffer-swapping has no C++ analogue, and +// the default-read enqueue() path already covers source-allocated +// buffers. // byobRequest === null ⇒ default read — the source allocates its OWN // buffer and calls controller.enqueue(view). // - The source calls enqueue-or-respond AT MOST ONCE per pull @@ -112,7 +116,6 @@ const { AbortControllerAbort, AbortControllerSignalGet, AbortSignalAbortedGet, - ArrayBufferPrototypeByteLengthGet, ArrayPrototypePush, ArrayPrototypeShift, ArrayPrototypeSplice, @@ -289,12 +292,13 @@ let getControllerConduit: ( // The BYOB request façade // // Wraps the head pull-into descriptor for the native source's consumption -// during pull. Mirrors the shape of the global ReadableStreamBYOBRequest -// (view/atLeast/respond/respondWithNewView) but is a distinct, module- -// private class: it is only ever handed to the native source, never to -// user code, so it needs no global registration or brand hardening beyond -// its private fields. Invalidated automatically once its descriptor is no -// longer the head request (view/atLeast report null; responds throw). +// during pull. Mirrors a subset of the global ReadableStreamBYOBRequest +// (view/atLeast/respond — respondWithNewView is deliberately omitted; see +// the contract header) but is a distinct, module-private class: it is only +// ever handed to the native source, never to user code, so it needs no +// global registration or brand hardening beyond its private fields. +// Invalidated automatically once its descriptor is no longer the head +// request (view/atLeast report null; responds throw). class NativeReadableStreamBYOBRequest { #conduit: NativePullConduit; @@ -337,11 +341,6 @@ class NativeReadableStreamBYOBRequest { if (!(#desc in this)) throw new TypeError('Illegal invocation'); this.#conduit.respondToDesc(this.#desc, bytesWritten); } - - respondWithNewView(view: ArrayBufferView): void { - if (!(#desc in this)) throw new TypeError('Illegal invocation'); - this.#conduit.respondToDescWithNewView(this.#desc, view); - } } // --------------------------------------------------------------------------- @@ -1034,65 +1033,6 @@ class NativePullConduit implements ByteStreamConsumerType { } } - respondToDescWithNewView( - desc: PullIntoDescriptor, - view: ArrayBufferView - ): void { - // Source-initiated stragglers: silent discard. - if (this.#status !== 'active') return; - if (!this.isHeadDesc(desc)) { - if (this.#requests[0] === undefined) { - // REFUSAL BACKSTOP (same as enqueue/respond). - throw new TypeError( - 'delivery refused: the pull was aborted (the source must check ' + - 'signal.aborted before enqueue/respond and stash data for ' + - 'redelivery)' - ); - } - throw new TypeError('This BYOB request has been invalidated'); - } - if (!isArrayBufferView(view)) { - throw new TypeError('respondWithNewView requires an ArrayBufferView'); - } - const isDataView = - TypedArrayPrototypeGetSymbolToStringTag(view) === undefined; - const buffer = ( - isDataView - ? DataViewPrototypeGetBuffer(view) - : TypedArrayPrototypeGetBuffer(view) - ) as ArrayBuffer; - const byteOffset = ( - isDataView - ? DataViewPrototypeGetByteOffset(view) - : TypedArrayPrototypeGetByteOffset(view) - ) as number; - const byteLength = ( - isDataView - ? DataViewPrototypeGetByteLength(view) - : TypedArrayPrototypeGetByteLength(view) - ) as number; - if ( - ArrayBufferPrototypeByteLengthGet(buffer) !== - ArrayBufferPrototypeByteLengthGet(desc.buffer) - ) { - throw new RangeError( - "The view's buffer must have the same byteLength as the request" - ); - } - if (byteOffset !== desc.byteOffset + desc.bytesFilled) { - throw new RangeError( - 'The new view must start where the previous fill ended' - ); - } - if (desc.bytesFilled + byteLength > desc.byteLength) { - throw new RangeError('The new view exceeds the remaining view size'); - } - // Adopt the new backing buffer (the native source may have swapped - // buffers), then account the bytes like a plain respond. - desc.buffer = buffer; - this.respondToDesc(desc, byteLength); - } - closeFromSource(): void { if (this.#status !== 'active') return; // late settle tolerated // EXPECTED-LENGTH CONTRACT: source-initiated close before reaching From 3e38796ce51eb4c9e76355460dfa52cc4cf613aa Mon Sep 17 00:00:00 2001 From: James M Snell Date: Thu, 9 Jul 2026 07:24:59 -0700 Subject: [PATCH 053/101] Additional minor native.ts cleanups --- src/per_isolate/per_isolate-env.d.ts | 1 + src/per_isolate/webstreams/native.ts | 40 +++++++++++++++++------- src/workerd/io/per-isolate-bootstrap.c++ | 1 + 3 files changed, 31 insertions(+), 11 deletions(-) diff --git a/src/per_isolate/per_isolate-env.d.ts b/src/per_isolate/per_isolate-env.d.ts index f269aae57aa..8abb9300995 100644 --- a/src/per_isolate/per_isolate-env.d.ts +++ b/src/per_isolate/per_isolate-env.d.ts @@ -73,6 +73,7 @@ declare const primordials: { declare const utils: { isArrayBuffer(value: unknown): value is ArrayBuffer; isArrayBufferView(value: unknown): value is ArrayBufferView; + isDataView(value: unknown): value is DataView; isPromise(value: unknown): value is Promise; isSharedArrayBuffer(value: unknown): value is SharedArrayBuffer; isUint8Array(value: unknown): value is Uint8Array; diff --git a/src/per_isolate/webstreams/native.ts b/src/per_isolate/webstreams/native.ts index 04ff59ee3ad..d146e1154ca 100644 --- a/src/per_isolate/webstreams/native.ts +++ b/src/per_isolate/webstreams/native.ts @@ -134,7 +134,6 @@ const { TypedArrayPrototypeGetBuffer, TypedArrayPrototypeGetByteLength, TypedArrayPrototypeGetByteOffset, - TypedArrayPrototypeGetSymbolToStringTag, Uint8Array, uncurryThis, } = primordials; @@ -300,12 +299,22 @@ let getControllerConduit: ( // Invalidated automatically once its descriptor is no longer the head // request (view/atLeast report null; responds throw). +let assertIsNativeReadableStreamBYOBRequest: ( + self: NativeReadableStreamBYOBRequest +) => void; + class NativeReadableStreamBYOBRequest { #conduit: NativePullConduit; #desc: PullIntoDescriptor; static { getRequestDesc = (request) => request.#desc; + + assertIsNativeReadableStreamBYOBRequest = function ( + self: NativeReadableStreamBYOBRequest + ): void { + if (!(#desc in self)) throw new TypeError('Illegal invocation'); + }; } constructor(conduit: NativePullConduit, desc: PullIntoDescriptor) { @@ -314,7 +323,7 @@ class NativeReadableStreamBYOBRequest { } get view(): Uint8Array | null { - if (!(#desc in this)) throw new TypeError('Illegal invocation'); + assertIsNativeReadableStreamBYOBRequest(this); if (!this.#conduit.isHeadDesc(this.#desc)) return null; const desc = this.#desc; return new Uint8Array( @@ -330,7 +339,7 @@ class NativeReadableStreamBYOBRequest { // header). The subtraction is defensive: a live descriptor always has // bytesFilled === 0 under the once-per-read contract. get atLeast(): number | null { - if (!(#desc in this)) throw new TypeError('Illegal invocation'); + assertIsNativeReadableStreamBYOBRequest(this); if (!this.#conduit.isHeadDesc(this.#desc)) return null; const desc = this.#desc; const remaining = desc.minimumFill - desc.bytesFilled; @@ -338,7 +347,7 @@ class NativeReadableStreamBYOBRequest { } respond(bytesWritten: number): void { - if (!(#desc in this)) throw new TypeError('Illegal invocation'); + assertIsNativeReadableStreamBYOBRequest(this); this.#conduit.respondToDesc(this.#desc, bytesWritten); } } @@ -351,6 +360,10 @@ class NativeReadableStreamBYOBRequest { // ONLY by the native source — never handed to user code — but the methods // brand-check anyway, matching the file-wide convention. +let assertIsNativeReadableStreamController: ( + self: NativeReadableStreamController +) => void; + class NativeReadableStreamController { #conduit: NativePullConduit; @@ -362,6 +375,12 @@ class NativeReadableStreamController { }; getControllerConduit = (controller) => controller.#conduit; + + assertIsNativeReadableStreamController = function ( + self: NativeReadableStreamController + ): void { + if (!(#conduit in self)) throw new TypeError('Illegal invocation'); + }; } constructor(conduit: NativePullConduit) { @@ -371,7 +390,7 @@ class NativeReadableStreamController { // Mode discrimination for the native source: non-null ⇔ the request at // the head of the FIFO is a BYOB read. get byobRequest(): NativeReadableStreamBYOBRequest | null { - if (!(#conduit in this)) throw new TypeError('Illegal invocation'); + assertIsNativeReadableStreamController(this); return this.#conduit.getByobRequest(); } @@ -379,22 +398,22 @@ class NativeReadableStreamController { // source contractually never consults it: pacing is purely demand- // driven (effective high-water mark of zero; strategy is ignored). get desiredSize(): number | null { - if (!(#conduit in this)) throw new TypeError('Illegal invocation'); + assertIsNativeReadableStreamController(this); return this.#conduit.desiredSize; } enqueue(chunk: ArrayBufferView): void { - if (!(#conduit in this)) throw new TypeError('Illegal invocation'); + assertIsNativeReadableStreamController(this); this.#conduit.enqueue(chunk); } close(): void { - if (!(#conduit in this)) throw new TypeError('Illegal invocation'); + assertIsNativeReadableStreamController(this); this.#conduit.closeFromSource(); } error(reason?: unknown): void { - if (!(#conduit in this)) throw new TypeError('Illegal invocation'); + assertIsNativeReadableStreamController(this); this.#conduit.errorFromSource(reason); } } @@ -865,8 +884,7 @@ class NativePullConduit implements ByteStreamConsumerType { if (isUint8Array(chunk)) { view = chunk as Uint8Array; } else { - const isDataView = - TypedArrayPrototypeGetSymbolToStringTag(chunk) === undefined; + const isDataView = utils.isDataView(chunk); const buffer = ( isDataView ? DataViewPrototypeGetBuffer(chunk) diff --git a/src/workerd/io/per-isolate-bootstrap.c++ b/src/workerd/io/per-isolate-bootstrap.c++ index 5168747803f..1a840b6a28b 100644 --- a/src/workerd/io/per-isolate-bootstrap.c++ +++ b/src/workerd/io/per-isolate-bootstrap.c++ @@ -93,6 +93,7 @@ struct BootstrapState { #define VALUE_METHOD_MAP(V) \ V(ArrayBuffer) \ V(ArrayBufferView) \ + V(DataView) \ V(Promise) \ V(SharedArrayBuffer) \ V(Uint8Array) From baf07aaf52a6ff5f2caba3e1e2f3c53866ee1239 Mon Sep 17 00:00:00 2001 From: Logan Gatlin Date: Thu, 9 Jul 2026 11:19:48 -0500 Subject: [PATCH 054/101] Add Rust ada-url based URL implementation Introduces a Rust URL parser backed by the ada-url crate, exposed via a new src/rust/api/url.rs module and wired into node.h, with an autogate to control rollout. --- deps/rust/Cargo.lock | 4 +- deps/rust/Cargo.toml | 6 +- src/rust/api/BUILD.bazel | 6 + src/rust/api/lib.rs | 22 +++ src/rust/api/url.rs | 318 +++++++++++++++++++++++++++++++ src/workerd/api/node/BUILD.bazel | 1 + src/workerd/api/node/node.h | 28 ++- src/workerd/util/autogate.c++ | 2 + src/workerd/util/autogate.h | 4 + 9 files changed, 387 insertions(+), 4 deletions(-) create mode 100644 src/rust/api/url.rs diff --git a/deps/rust/Cargo.lock b/deps/rust/Cargo.lock index 1be4329dbda..f228e45ccb4 100644 --- a/deps/rust/Cargo.lock +++ b/deps/rust/Cargo.lock @@ -4,9 +4,9 @@ version = 4 [[package]] name = "ada-url" -version = "3.4.5" +version = "3.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30fb6a856f7a3120598fc3a4560e29e818f683a7def0ab3ebb851b6c78bdafbe" +checksum = "e6850197d6ff30bed32bf9a7a5cdf23376bd56151fd716c95803093e1c826ee2" dependencies = [ "cc", "link_args", diff --git a/deps/rust/Cargo.toml b/deps/rust/Cargo.toml index 91ff8b6657b..8382e9666f7 100644 --- a/deps/rust/Cargo.toml +++ b/deps/rust/Cargo.toml @@ -25,7 +25,11 @@ syn = { version = "2", features = ["full"] } # We prefer single-digit dependencies to stay up to date as much as possible # When adding packages here, please only enable features as needed to keep compile times and # binary sizes bounded. -ada-url = "3" +# The `bundled` feature is disabled (it is part of the default feature set): with +# it off we skip compiling the crate's vendored copy of ada and instead link +# workerd's existing C++ `@ada-url`, avoiding a duplicate copy of ada in the +# binary. +ada-url = { version = "3", default-features = false, features = ["std"] } anyhow = "1" async-trait = { version = "0", default-features = false } capnp = "0" diff --git a/src/rust/api/BUILD.bazel b/src/rust/api/BUILD.bazel index 8878c5f7f5f..7a4826ce23a 100644 --- a/src/rust/api/BUILD.bazel +++ b/src/rust/api/BUILD.bazel @@ -10,7 +10,13 @@ wd_rust_crate( test_deps = ["//src/rust/jsg-test"], visibility = ["//visibility:public"], deps = [ + # ada-url is built with its `bundled` feature disabled (see + # //deps/rust:Cargo.toml), so it does not compile its own copy of the + # C++ ada library. Provide the symbols via workerd's existing C++ + # @ada-url instead, avoiding a duplicate copy of ada in the binary. + "@ada-url", "//src/rust/jsg", + "@crates_vendor//:ada-url", "@crates_vendor//:thiserror", ], ) diff --git a/src/rust/api/lib.rs b/src/rust/api/lib.rs index 5da042c5529..b3c3f3643f0 100644 --- a/src/rust/api/lib.rs +++ b/src/rust/api/lib.rs @@ -7,8 +7,10 @@ use std::pin::Pin; use jsg::ToJS; use crate::dns::DnsUtil; +use crate::url::UrlUtil; pub mod dns; +pub mod url; #[cxx::bridge(namespace = "workerd::rust::api")] mod ffi { @@ -20,6 +22,12 @@ mod ffi { } extern "Rust" { pub fn register_nodejs_modules(registry: Pin<&mut ModuleRegistry>); + + // The Rust implementation of `node-internal:url`. Registered separately + // because it is gated by the C++ `NODEJS_URL_RUST` autogate; when the + // gate is off the C++ `UrlUtil` registers that module instead (see + // node.h). Kept as its own entry point to avoid a boolean parameter. + pub fn register_nodejs_url_module(registry: Pin<&mut ModuleRegistry>); } } @@ -37,6 +45,20 @@ pub fn register_nodejs_modules(registry: Pin<&mut ffi::ModuleRegistry>) { ); } +pub fn register_nodejs_url_module(registry: Pin<&mut ffi::ModuleRegistry>) { + jsg::modules::add_builtin( + registry, + "node-internal:url", + // SAFETY: isolate is valid and locked — called from C++ module registration. + |isolate| unsafe { + let mut lock = jsg::Lock::from_isolate_ptr(isolate); + let url_util = UrlUtil::new(); + url_util.to_js(&mut lock).into_ffi() + }, + jsg::modules::ModuleType::Internal, + ); +} + #[cfg(test)] mod tests { use jsg_test::Harness; diff --git a/src/rust/api/url.rs b/src/rust/api/url.rs new file mode 100644 index 00000000000..d5e47166e06 --- /dev/null +++ b/src/rust/api/url.rs @@ -0,0 +1,318 @@ +// Copyright (c) 2026 Cloudflare, Inc. +// Licensed under the Apache 2.0 license found in the LICENSE file or at: +// https://opensource.org/licenses/Apache-2.0 + +use std::net::IpAddr; +use std::str::FromStr; + +use ada_url::Idna; +use ada_url::Url; +use jsg_macros::jsg_method; +use jsg_macros::jsg_resource; +use thiserror::Error; + +#[derive(Debug, Error)] +pub enum UrlError { + #[error("{0}")] + ParseFailed(&'static str), +} + +impl From for jsg::Error { + fn from(val: UrlError) -> Self { + match val { + UrlError::ParseFailed(msg) => Self::new_error(msg), + } + } +} + +/// Implementation shared by `domainToASCII` and `domainToUnicode`. +/// +/// Returns the parsed (ASCII / punycode) hostname for the given domain, or +/// `None` if the domain is empty or fails to parse as a hostname. +/// +/// # Errors +/// `UrlError::ParseFailed` if the placeholder URL fails to parse (should never +/// happen in practice — it mirrors the `JSG_REQUIRE` in the C++ implementation). +fn get_hostname(domain: &str) -> Result, UrlError> { + if domain.is_empty() { + return Ok(None); + } + + // It is important to have an initial value that contains a special scheme. + // Since it will change the implementation of `set_hostname` according to the + // URL spec. + let mut out = + Url::parse("ws://x", None).map_err(|_| UrlError::ParseFailed("URL parsing failed"))?; + match out.set_hostname(Some(domain)) { + Ok(()) => Ok(Some(out.hostname().to_owned())), + Err(()) => Ok(None), + } +} + +#[jsg_resource] +pub struct UrlUtil; + +#[jsg_resource] +impl UrlUtil { + #[must_use] + pub fn new() -> jsg::Rc { + jsg::Rc::new(Self {}) + } + + /// # Errors + /// `UrlError::ParseFailed` (see `get_hostname`). + #[jsg_method(name = "domainToASCII")] + pub fn domain_to_ascii(&self, domain: String) -> Result { + Ok(get_hostname(&domain)?.unwrap_or_default()) + } + + /// # Errors + /// `UrlError::ParseFailed` (see `get_hostname`). + #[jsg_method(name = "domainToUnicode")] + pub fn domain_to_unicode(&self, domain: String) -> Result { + Ok(match get_hostname(&domain)? { + Some(hostname) => Idna::unicode(&hostname), + None => String::new(), + }) + } + + #[jsg_method(name = "toASCII")] + pub fn to_ascii(&self, url: String) -> String { + Idna::ascii(&url) + } + + /// # Errors + /// `UrlError::ParseFailed` if the input cannot be parsed as a URL. + #[jsg_method(name = "format")] + pub fn format( + &self, + input: String, + hash: bool, + unicode: bool, + search: bool, + auth: bool, + ) -> Result { + let mut out = Url::parse(input.as_str(), None) + .map_err(|_| UrlError::ParseFailed("Failed to parse URL"))?; + + if !hash { + out.set_hash(None); + } + + if !search { + out.set_search(None); + } + + if !auth { + // Clearing both the username and the password removes the userinfo + // section entirely, matching the C++ implementation which assigns + // empty strings to both fields. The setters can refuse on URLs that + // cannot have credentials, in which case there is nothing to clear. + let _ = out.set_username(Some("")); + let _ = out.set_password(Some("")); + } + + if unicode && out.has_hostname() { + // The C++ implementation assigns the IDNA Unicode form directly to + // the host field, bypassing the host parser which would otherwise + // re-encode it back to ASCII/punycode for special schemes. The Rust + // binding only exposes parser-backed setters, so we instead splice + // the Unicode hostname into the serialized href. + // + // `host_end` reliably marks the end of the (ASCII/punycode) host in + // the href, but `host_start` from `components()` points at the `@` + // separator when credentials are present, so we derive the start from + // the known host length instead. `hostname()` matches the WHATWG + // `URL.hostname` serialization, so IPv6 literals keep their brackets + // (`[::1]`) — the same brackets present in the href — and the length + // math needs no special-casing. The host is always ASCII in the href, + // so these offsets fall on valid UTF-8 boundaries; we nonetheless use + // checked arithmetic and `get()` to avoid any chance of a panic. + let hostname = out.hostname(); + let unicode_host = Idna::unicode(hostname); + let host_end = out.components().host_end as usize; + let href = out.href(); + + let host_start = host_end + .checked_sub(hostname.len()) + .ok_or(UrlError::ParseFailed("URL host offset underflow"))?; + let prefix = href + .get(..host_start) + .ok_or(UrlError::ParseFailed("URL host offset out of bounds"))?; + let suffix = href + .get(host_end..) + .ok_or(UrlError::ParseFailed("URL host offset out of bounds"))?; + + return Ok(format!("{prefix}{unicode_host}{suffix}")); + } + + Ok(out.href().to_owned()) + } + + // We return an empty string if the input is not a valid IP address. This + // mirrors `workerd::rust::net::canonicalize_ip`, reimplemented here to avoid + // a cross-crate FFI dependency now that the caller is itself Rust. + #[jsg_method(name = "canonicalizeIp")] + pub fn canonicalize_ip(&self, input: String) -> String { + IpAddr::from_str(&input).map_or(String::new(), |ip| ip.to_string()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_domain_to_ascii() { + let util = UrlUtil {}; + assert_eq!(util.domain_to_ascii(String::new()).unwrap(), ""); + assert_eq!( + util.domain_to_ascii("español.com".to_owned()).unwrap(), + "xn--espaol-zwa.com" + ); + assert_eq!( + util.domain_to_ascii("理容ナカムラ.com".to_owned()).unwrap(), + "xn--lck1c3crb1723bpq4a.com" + ); + } + + #[test] + fn test_domain_to_unicode() { + let util = UrlUtil {}; + assert_eq!(util.domain_to_unicode(String::new()).unwrap(), ""); + assert_eq!( + util.domain_to_unicode("xn--espaol-zwa.com".to_owned()) + .unwrap(), + "español.com" + ); + assert_eq!( + util.domain_to_unicode("xn--lck1c3crb1723bpq4a.com".to_owned()) + .unwrap(), + "理容ナカムラ.com" + ); + } + + #[test] + fn test_to_ascii() { + let util = UrlUtil {}; + assert_eq!( + util.to_ascii("meßagefactory.ca".to_owned()), + "xn--meagefactory-m9a.ca" + ); + } + + #[test] + fn test_format_strips_components() { + let util = UrlUtil {}; + let href = "http://user:pass@example.com/a?b=c#d".to_owned(); + + // Everything kept. + assert_eq!( + util.format(href.clone(), true, false, true, true).unwrap(), + "http://user:pass@example.com/a?b=c#d" + ); + // Drop hash. + assert_eq!( + util.format(href.clone(), false, false, true, true).unwrap(), + "http://user:pass@example.com/a?b=c" + ); + // Drop search. + assert_eq!( + util.format(href.clone(), true, false, false, true).unwrap(), + "http://user:pass@example.com/a#d" + ); + // Drop auth. + assert_eq!( + util.format(href, true, false, true, false).unwrap(), + "http://example.com/a?b=c#d" + ); + } + + #[test] + fn test_format_unicode() { + let util = UrlUtil {}; + // Unicode hostname must survive serialization for a special (http) scheme. + assert_eq!( + util.format( + "http://user:pass@xn--lck1c3crb1723bpq4a.com/a?a=b#c".to_owned(), + true, + true, + true, + true + ) + .unwrap(), + "http://user:pass@理容ナカムラ.com/a?a=b#c" + ); + // Port is preserved when splicing the unicode hostname. + assert_eq!( + util.format( + "http://user:pass@xn--0zwm56d.com:8080/path".to_owned(), + true, + true, + true, + true + ) + .unwrap(), + "http://user:pass@测试.com:8080/path" + ); + } + + #[test] + fn test_format_unicode_ipv6() { + let util = UrlUtil {}; + // IPv6 literals are bracketed in the href but `hostname()` strips the + // brackets. The unicode splice must preserve the brackets (and the + // trailing port) rather than corrupting the address. + assert_eq!( + util.format( + "http://[2001:db8::1]:8080/path".to_owned(), + true, + true, + true, + true + ) + .unwrap(), + "http://[2001:db8::1]:8080/path" + ); + // IPv6 with credentials (so `host_start` from components points at `@`). + assert_eq!( + util.format( + "http://user:pass@[::1]/path".to_owned(), + true, + true, + true, + true + ) + .unwrap(), + "http://user:pass@[::1]/path" + ); + } + + #[test] + fn test_format_invalid() { + let util = UrlUtil {}; + assert!( + util.format("not a url".to_owned(), true, false, true, true) + .is_err() + ); + } + + #[test] + fn test_canonicalize_ip() { + let util = UrlUtil {}; + // Already-canonical IPv4 is returned unchanged. + assert_eq!( + util.canonicalize_ip("192.168.1.1".to_owned()), + "192.168.1.1" + ); + // IPv6 is canonicalized (zero-compressed / lower-cased). + assert_eq!( + util.canonicalize_ip("2001:0DB8:0000:0000:0000:0000:0000:0001".to_owned()), + "2001:db8::1" + ); + // Invalid input (including leading-zero IPv4 octets, which Rust's + // IpAddr parser rejects) yields an empty string. + assert_eq!(util.canonicalize_ip("192.168.000.001".to_owned()), ""); + assert_eq!(util.canonicalize_ip("not-an-ip".to_owned()), ""); + } +} diff --git a/src/workerd/api/node/BUILD.bazel b/src/workerd/api/node/BUILD.bazel index 4612b640c97..d75824f2cc2 100644 --- a/src/workerd/api/node/BUILD.bazel +++ b/src/workerd/api/node/BUILD.bazel @@ -51,6 +51,7 @@ wd_cc_library( "//src/rust/api", "//src/workerd/api:streams-compression", "//src/workerd/io", + "//src/workerd/util:autogate", "//src/workerd/util:mimetype", "@capnp-cpp//src/kj/compat:kj-brotli", "@ncrypto", diff --git a/src/workerd/api/node/node.h b/src/workerd/api/node/node.h index 88351468469..93199db806e 100644 --- a/src/workerd/api/node/node.h +++ b/src/workerd/api/node/node.h @@ -19,6 +19,7 @@ #include #include #include +#include #include @@ -35,7 +36,6 @@ namespace workerd::api::node { V(UtilModule, "node-internal:util") \ V(DiagnosticsChannelModule, "node-internal:diagnostics_channel") \ V(ZlibUtil, "node-internal:zlib") \ - V(UrlUtil, "node-internal:url") \ V(TimersUtil, "node-internal:timers") \ V(SqliteUtil, "node-internal:sqlite") \ V(InspectorModule, "node-internal:inspector") @@ -95,6 +95,17 @@ void registerNodeJsCompatModules(Registry& registry, auto featureFlags) { #undef V + // The native `node-internal:url` module has both a C++ (`UrlUtil`) and a Rust + // implementation. The `NODEJS_URL_RUST` autogate selects the Rust one; when + // the gate is off we register the C++ implementation here instead. Exactly one + // of the two registers the `node-internal:url` specifier (the Rust side is + // told whether to register it via `register_nodejs_url_module` below). + bool useRustUrl = util::Autogate::isEnabled(util::AutogateKey::NODEJS_URL_RUST); + if (!useRustUrl) { + registry.template addBuiltinModule( + "node-internal:url", workerd::jsg::ModuleRegistry::Type::INTERNAL); + } + bool nodeJsCompatEnabled = isNodeJsCompatEnabled(featureFlags); registry.addBuiltinBundleFiltered(NODE_BUNDLE, [&](jsg::Module::Reader module) { @@ -219,6 +230,9 @@ void registerNodeJsCompatModules(Registry& registry, auto featureFlags) { ::workerd::rust::jsg::RustModuleRegistry r(registry); ::workerd::rust::api::register_nodejs_modules(r); + if (useRustUrl) { + ::workerd::rust::api::register_nodejs_url_module(r); + } } template @@ -233,6 +247,15 @@ kj::Own getInternalNodeJsCompatModuleBundle(auto fea NODEJS_MODULES_EXPERIMENTAL(V) } #undef V + + // See registerNodeJsCompatModules(): the NODEJS_URL_RUST autogate selects + // between the C++ and Rust implementations of `node-internal:url`. + bool useRustUrl = util::Autogate::isEnabled(util::AutogateKey::NODEJS_URL_RUST); + if (!useRustUrl) { + static const auto kUrlUtilSpecifier = "node-internal:url"_url; + builder.addObject(kUrlUtilSpecifier); + } + jsg::modules::ModuleBundle::getBuiltInBundleFromCapnp(builder, NODE_BUNDLE); // Register Rust-implemented Node.js modules using the reusable adapter @@ -240,6 +263,9 @@ kj::Own getInternalNodeJsCompatModuleBundle(auto fea { ::workerd::rust::jsg::RustBuiltinModuleAdapter adapter(builder); ::workerd::rust::api::register_nodejs_modules(adapter); + if (useRustUrl) { + ::workerd::rust::api::register_nodejs_url_module(adapter); + } } return builder.finish(); diff --git a/src/workerd/util/autogate.c++ b/src/workerd/util/autogate.c++ index 309a03deec7..a8bf1dc4649 100644 --- a/src/workerd/util/autogate.c++ +++ b/src/workerd/util/autogate.c++ @@ -37,6 +37,8 @@ kj::StringPtr KJ_STRINGIFY(AutogateKey key) { return "per-isolate-javascript-bootstrap"_kj; case AutogateKey::DURABLE_OBJECT_RETRIES_FETCH: return "durable-object-retries-fetch"_kj; + case AutogateKey::NODEJS_URL_RUST: + return "nodejs-url-rust"_kj; case AutogateKey::NumOfKeys: KJ_FAIL_ASSERT("NumOfKeys should not be used in getName"); } diff --git a/src/workerd/util/autogate.h b/src/workerd/util/autogate.h index e32369f0495..550630e5fb8 100644 --- a/src/workerd/util/autogate.h +++ b/src/workerd/util/autogate.h @@ -44,6 +44,10 @@ enum class AutogateKey { // Gate for the Durable Object fetch-retries feature, scoped to DO `fetch()`. Enables the // retry-token claim machinery. DURABLE_OBJECT_RETRIES_FETCH, + // When enabled, the native `node-internal:url` module is provided by the Rust + // implementation (api::node UrlUtil ported to src/rust/api) instead of the + // C++ implementation. The C++ implementation is retained for rollback. + NODEJS_URL_RUST, NumOfKeys // Reserved for iteration. }; From 7c9d584687fce03cf2236b1854424b0f7cb6aec2 Mon Sep 17 00:00:00 2001 From: James M Snell Date: Tue, 7 Jul 2026 10:17:45 -0700 Subject: [PATCH 055/101] Improve TypeScript IdentityTransformStream/FixedLengthStream impl ... and add tests --- src/per_isolate/main.ts | 16 + src/per_isolate/webstreams/identity.ts | 238 ++++- src/workerd/api/tests/BUILD.bazel | 6 + .../api/tests/ts-identity-streams-test.js | 868 ++++++++++++++++++ .../tests/ts-identity-streams-test.wd-test | 22 + src/workerd/io/compatibility-date.capnp | 9 + 6 files changed, 1145 insertions(+), 14 deletions(-) create mode 100644 src/workerd/api/tests/ts-identity-streams-test.js create mode 100644 src/workerd/api/tests/ts-identity-streams-test.wd-test diff --git a/src/per_isolate/main.ts b/src/per_isolate/main.ts index 77c8f1ab98a..d1329acbcfc 100644 --- a/src/per_isolate/main.ts +++ b/src/per_isolate/main.ts @@ -52,6 +52,7 @@ if (compatFlags['typescript_implemented_streams']) { FixedLengthStream, TextEncoderStream, TextDecoderStream, + ReadableStreamDrainingReader, } = require('webstreams/streams'); ObjectDefineProperties(globalThis, { @@ -175,4 +176,19 @@ if (compatFlags['typescript_implemented_streams']) { value: TextDecoderStream, }, }); + + // Internal-only: expose the DrainingReader for testing expectedLength + // pass-through (Content-Length integration). Gated by a separate + // experimental flag that may never lose its experimental annotation. + if (compatFlags['expose_draining_reader']) { + ObjectDefineProperties(globalThis, { + ReadableStreamDrainingReader: { + __proto__: null, + configurable: true, + enumerable: false, + writable: true, + value: ReadableStreamDrainingReader, + }, + }); + } } diff --git a/src/per_isolate/webstreams/identity.ts b/src/per_isolate/webstreams/identity.ts index 36a7a271d81..a84cdd90840 100644 --- a/src/per_isolate/webstreams/identity.ts +++ b/src/per_isolate/webstreams/identity.ts @@ -17,6 +17,27 @@ // immediately without touching the readable queue (no zero-length // chunk enqueued, no backpressure interaction, no pull). // +// RENDEZVOUS BACKPRESSURE MODEL +// +// This implementation uses a rendezvous pattern matching the original +// C++ IdentityTransformStream behavior: backpressure starts ENABLED +// (#backpressure = true) and the readable side uses highWaterMark: 0. +// A write will block in sinkWrite's `while (#backpressure)` loop +// until the readable side's pull callback fires (triggered by a +// reader.read() call), which clears backpressure. This means +// writer.write() will NOT resolve until a corresponding reader.read() +// is issued — callers must not `await writer.write()` before starting +// a read, or the result is a deadlock. +// +// Correct usage: +// const readPromise = reader.read(); // triggers pull → clears bp +// await writer.write(chunk); // now proceeds +// const { value } = await readPromise; +// +// Deadlock: +// await writer.write(chunk); // blocks forever — no read pending +// reader.read(); // never reached +// // FixedLengthStream extends IdentityTransformStream with an // `expectedLength` that flows through to the readable byte controller, // giving `new Response(fixedLengthStream.readable)` its Content-Length @@ -35,13 +56,17 @@ import type { const { ArrayBuffer, + ArrayBufferPrototypeByteLengthGet, + BigInt, DataViewPrototypeGetBuffer, DataViewPrototypeGetByteLength, DataViewPrototypeGetByteOffset, ObjectDefineProperties, ObjectGetOwnPropertyDescriptor, PromiseWithResolvers, + RangeError, Symbol, + SymbolToStringTag, TextEncoder, TextEncoderEncode, TypeError, @@ -75,6 +100,10 @@ const writableControllerError = uncurryThis( WritableStreamDefaultController.prototype.error ) as (controller: object, reason: unknown) => void; +function isActualObject(value: unknown) { + return value != null && typeof value === 'object'; +} + // --- Bootstrap captures (byte controller methods + TextEncoder) ---------- const byteControllerEnqueue = uncurryThis( @@ -157,6 +186,53 @@ function validateAndCopyChunk(chunk: unknown): Uint8Array | undefined { ); } +// Compute the byte size of a chunk for WritableStream queue tracking. +// Used ONLY as the `size` strategy callback when highWaterMark is +// specified, feeding queueTotalSize which drives desiredSize and +// writer.ready — purely advisory backpressure signaling. It does NOT +// affect data correctness, the FLS byte budget (#remaining uses actual +// byte lengths from the copied chunk), or Content-Length. +// +// Our WritableStreamDefaultController dequeues AFTER the write algorithm +// completes (writable.ts #processWrite), so in-flight bytes stay counted +// in queueTotalSize — matching the C++ model where all pipeline bytes +// (in-flight + queued) are tracked until fully consumed. +// +// For strings, uses str.length * 3 as a conservative upper-bound +// estimate (max UTF-8 bytes per UTF-16 code unit) to avoid a redundant +// TextEncoder.encode — the actual encode happens once in +// validateAndCopyChunk. The overcount is relatively harmless: since this +// only affects backpressure, overestimating just means the writable side +// signals backpressure slightly earlier than strictly necessary. +function byteSize(chunk: unknown): number { + if (typeof chunk === 'string') { + return (chunk as string).length * 3; + } + if (isArrayBuffer(chunk)) { + return ArrayBufferPrototypeByteLengthGet(chunk as ArrayBuffer); + } + if (isSharedArrayBuffer(chunk)) { + // SharedArrayBuffer.prototype.byteLength getter is separate from + // ArrayBuffer's; use Uint8Array wrapper for the uncommon SAB case. + return TypedArrayPrototypeGetByteLength( + new Uint8Array(chunk as unknown as ArrayBuffer) + ) as number; + } + if (isArrayBufferView(chunk)) { + const isDataView = + TypedArrayPrototypeGetSymbolToStringTag(chunk) === undefined; + return ( + isDataView + ? DataViewPrototypeGetByteLength(chunk) + : TypedArrayPrototypeGetByteLength(chunk) + ) as number; + } + // Invalid chunk type — validateAndCopyChunk will throw TypeError. + return 1; +} + +let assertIsIdentityTransformStream: (self: IdentityTransformStream) => void; + // --------------------------------------------------------------------------- const kPrivateSymbol: symbol = Symbol('private'); @@ -166,8 +242,23 @@ class IdentityTransformStream { #writable: WritableStreamType; #readableController: object | undefined; #writableController: object | undefined; + // Backpressure starts ENABLED — part of the rendezvous pattern. + // Writes block until a reader pull clears this flag. #backpressure: boolean = true; #backpressureChange: PromiseWithResolversType; + // Byte budget for FixedLengthStream enforcement. undefined for plain + // IdentityTransformStream; set to expectedLength for FixedLengthStream. + // Decremented on each write; overwrite/underwrite errors match C++ + // (identity-transform-stream.c++ tryReadInternal). Stored as bigint + // to preserve precision for the full uint64_t range. + #remaining: bigint | undefined; + + static { + assertIsIdentityTransformStream = function (self) { + if (!isActualObject(self) || !(#readable in self)) + throw new TypeError('Illegal invocation'); + }; + } #setBackpressure(backpressure: boolean): void { this.#backpressureChange.resolve(); @@ -189,19 +280,25 @@ class IdentityTransformStream { } constructor(writableStrategy?: QueuingStrategy); - // Internal: called by FixedLengthStream with (kPrivateSymbol, length). - constructor(internal: symbol, expectedLength: bigint | number); + // Internal: called by FixedLengthStream. + constructor( + internal: symbol, + expectedLength: bigint | number, + writableStrategy?: QueuingStrategy + ); constructor( writableStrategyOrInternal?: QueuingStrategy | symbol, - internalExpectedLength?: bigint | number + internalExpectedLength?: bigint | number, + internalWritableStrategy?: QueuingStrategy ) { // External: new IdentityTransformStream() or // new IdentityTransformStream(writableStrategy). - // Internal (from FixedLengthStream): new ITS(kPrivateSymbol, length). + // Internal (from FixedLengthStream): new ITS(kPrivateSymbol, len, strategy?). let writableStrategy: QueuingStrategy | undefined; let expectedLength: bigint | number | undefined; if (writableStrategyOrInternal === kPrivateSymbol) { expectedLength = internalExpectedLength; + writableStrategy = internalWritableStrategy; } else { writableStrategy = writableStrategyOrInternal as | QueuingStrategy @@ -209,6 +306,27 @@ class IdentityTransformStream { } writableStrategy ??= {} as QueuingStrategy; + // Initialize byte budget for FixedLengthStream enforcement. + // Stored as bigint to cover the full uint64_t range without + // precision loss. + if (expectedLength !== undefined) { + this.#remaining = + typeof expectedLength === 'bigint' + ? expectedLength + : BigInt(expectedLength); + } + + // When highWaterMark is explicitly provided, switch to byte-length + // sizing so that desiredSize tracks bytes rather than chunk count, + // matching the C++ WritableStreamInternalController which uses + // adjustWriteBufferSize with actual byte lengths. + if (writableStrategy.highWaterMark !== undefined) { + writableStrategy = { + highWaterMark: writableStrategy.highWaterMark, + size: byteSize, + }; + } + const initialBackpressureChange = PromiseWithResolvers() as PromiseWithResolversType; markPromiseHandled(initialBackpressureChange.promise); @@ -218,6 +336,24 @@ class IdentityTransformStream { const sinkWrite = async (chunk: unknown): Promise => { const copied = validateAndCopyChunk(chunk); if (copied === undefined) return; // zero-length no-op + + // FixedLengthStream overwrite enforcement (matches C++ + // identity-transform-stream.c++ tryReadInternal overwrite check). + if (this.#remaining !== undefined) { + const len = BigInt(TypedArrayPrototypeGetByteLength(copied) as number); + if (len > this.#remaining) { + const err = new RangeError( + 'Attempt to write too many bytes through a FixedLengthStream.' + ); + const rc = this.#readableController; + if (rc !== undefined) byteControllerError(rc, err); + throw err; + } + this.#remaining -= len; + } + + // RENDEZVOUS: block here until a reader.read() triggers pull, + // which sets #backpressure = false. See file-level comment. while (this.#backpressure) { await this.#backpressureChange.promise; const state = writableInternals.getState(this.#writable); @@ -233,7 +369,19 @@ class IdentityTransformStream { this.#setBackpressure(backpressure); } }; + // FixedLengthStream underwrite enforcement (matches C++ + // identity-transform-stream.c++ tryReadInternal underwrite check). + // Not called on abort — sinkAbort fires instead, naturally + // skipping the underwrite check (matching C++ behavior). const sinkClose = (): void => { + if (this.#remaining !== undefined && this.#remaining > 0n) { + const err = new RangeError( + 'FixedLengthStream did not see all expected bytes before close().' + ); + const rc = this.#readableController; + if (rc !== undefined) byteControllerError(rc, err); + throw err; + } const rc = this.#readableController; if (rc !== undefined) byteControllerClose(rc); }; @@ -255,6 +403,8 @@ class IdentityTransformStream { ); // --- Readable side (byte stream, BYOB capable) --- + // RENDEZVOUS: pull is called when a reader.read() needs data. + // Clearing backpressure here unblocks the pending sinkWrite. const sourcePull = (): Promise => { this.#setBackpressure(false); return this.#backpressureChange.promise; @@ -275,38 +425,98 @@ class IdentityTransformStream { byteSource.expectedLength = expectedLength; } + // highWaterMark: 0 ensures pull is not called eagerly — it fires + // only when a reader.read() is pending, enforcing the rendezvous. this.#readable = new ReadableStream(byteSource, { highWaterMark: 0, }); } get readable(): ReadableStreamType { - if (!(#readable in this)) throw new TypeError('Illegal invocation'); + assertIsIdentityTransformStream(this); return this.#readable; } get writable(): WritableStreamType { - if (!(#writable in this)) throw new TypeError('Illegal invocation'); + assertIsIdentityTransformStream(this); return this.#writable; } } +// Maximum expectedLength: uint64_t max. Content-Length is carried as +// uint64_t through the C++/KJ HTTP layer, so values beyond this are +// not representable. +const MAX_UINT64 = 0xffff_ffff_ffff_ffffn; + class FixedLengthStream extends IdentityTransformStream { constructor( expectedLength: bigint | number, - _writableStrategy?: QueuingStrategy + writableStrategy?: QueuingStrategy ) { - // Validation of expectedLength is handled by the - // ReadableByteStreamController (normalizeExpectedLength) at the - // readable-side construction. writableStrategy is accepted for - // signature parity but currently unused (ITS defaults apply). - super(kPrivateSymbol, expectedLength); + if ( + typeof expectedLength !== 'number' && + typeof expectedLength !== 'bigint' + ) { + throw new TypeError( + 'FixedLengthStream expected length must be a number or bigint.' + ); + } + // BigInt() conversion rejects NaN and fractions (RangeError) naturally. + const bigLen = + typeof expectedLength === 'bigint' + ? expectedLength + : BigInt(expectedLength); + if (bigLen < 0n || bigLen > MAX_UINT64) { + throw new RangeError( + 'FixedLengthStream requires a non-negative expected length ' + + 'that fits in a uint64.' + ); + } + // + // Cap highWaterMark at expectedLength, matching C++ behavior + // (identity-transform-stream.c++ FixedLengthStream::constructor): buffering more than the + // total expected output is pointless. + if ( + writableStrategy !== undefined && + writableStrategy.highWaterMark !== undefined + ) { + const numExpected = + typeof expectedLength === 'bigint' + ? Number(expectedLength) + : expectedLength; + const hwm = writableStrategy.highWaterMark; + writableStrategy = { + highWaterMark: hwm < numExpected ? hwm : numExpected, + }; + } + super(kPrivateSymbol, expectedLength, writableStrategy); } } +const kEnumerable = { __proto__: null, enumerable: true }; + ObjectDefineProperties(IdentityTransformStream.prototype, { - readable: { enumerable: true }, - writable: { enumerable: true }, + __proto__: null, + readable: kEnumerable, + writable: kEnumerable, + [SymbolToStringTag]: { + __proto__: null, + value: 'IdentityTransformStream', + writable: false, + enumerable: false, + configurable: true, + }, +}); + +ObjectDefineProperties(FixedLengthStream.prototype, { + __proto__: null, + [SymbolToStringTag]: { + __proto__: null, + value: 'FixedLengthStream', + writable: false, + enumerable: false, + configurable: true, + }, }); module.exports = { diff --git a/src/workerd/api/tests/BUILD.bazel b/src/workerd/api/tests/BUILD.bazel index 2d74a60909c..b187a1c9a6a 100644 --- a/src/workerd/api/tests/BUILD.bazel +++ b/src/workerd/api/tests/BUILD.bazel @@ -1148,6 +1148,12 @@ wd_test( data = ["ts-webstreams-test.js"], ) +wd_test( + src = "ts-identity-streams-test.wd-test", + args = ["--experimental"], + data = ["ts-identity-streams-test.js"], +) + sh_test( name = "abortIsolate", size = "medium", diff --git a/src/workerd/api/tests/ts-identity-streams-test.js b/src/workerd/api/tests/ts-identity-streams-test.js new file mode 100644 index 00000000000..5c43860016e --- /dev/null +++ b/src/workerd/api/tests/ts-identity-streams-test.js @@ -0,0 +1,868 @@ +// Copyright (c) 2026 Cloudflare, Inc. +// Licensed under the Apache 2.0 license found in the LICENSE file or at: +// https://opensource.org/licenses/Apache-2.0 + +// Manually add globals expected by eslint only for testing +/* global ReadableStreamDrainingReader */ + +import { + strictEqual, + notStrictEqual, + ok, + deepStrictEqual, + rejects, + throws, + doesNotMatch, +} from 'node:assert'; + +const enc = new TextEncoder(); + +// Verify the implementations are the TypeScript versions, not native C++. +export const isTypeScriptImpl = { + test() { + doesNotMatch(IdentityTransformStream.toString(), /\[native code\]/); + doesNotMatch(FixedLengthStream.toString(), /\[native code\]/); + }, +}; + +// SymbolToStringTag should be set correctly. +export const toStringTag = { + test() { + const its = new IdentityTransformStream(); + strictEqual( + Object.prototype.toString.call(its), + '[object IdentityTransformStream]' + ); + const fls = new FixedLengthStream(0); + strictEqual( + Object.prototype.toString.call(fls), + '[object FixedLengthStream]' + ); + }, +}; + +// readable and writable should be enumerable on the prototype. +export const propertyEnumerability = { + test() { + const its = new IdentityTransformStream(); + const descriptors = Object.getOwnPropertyDescriptors( + Object.getPrototypeOf(its) + ); + strictEqual(descriptors.readable.enumerable, true); + strictEqual(descriptors.writable.enumerable, true); + }, +}; + +// FixedLengthStream is a subclass of IdentityTransformStream. +export const fixedLengthIsSubclass = { + test() { + const fls = new FixedLengthStream(10); + ok(fls instanceof IdentityTransformStream); + ok(fls instanceof FixedLengthStream); + }, +}; + +// --- Chunk validation tests --- + +// Accepts Uint8Array. +export const acceptsUint8Array = { + async test() { + const { readable, writable } = new IdentityTransformStream(); + const writer = writable.getWriter(); + const reader = readable.getReader(); + // Start read before awaiting write to avoid backpressure deadlock. + const readPromise = reader.read(); + await writer.write(new Uint8Array([1, 2, 3])); + const { value, done } = await readPromise; + strictEqual(done, false); + deepStrictEqual([...value], [1, 2, 3]); + await writer.close(); + }, +}; + +// Accepts strings (converted to UTF-8). +export const acceptsString = { + async test() { + const { readable, writable } = new IdentityTransformStream(); + const writer = writable.getWriter(); + const reader = readable.getReader(); + const readPromise = reader.read(); + await writer.write('hello'); + const { value, done } = await readPromise; + strictEqual(done, false); + strictEqual(new TextDecoder().decode(value), 'hello'); + await writer.close(); + }, +}; + +// Accepts ArrayBuffer. +export const acceptsArrayBuffer = { + async test() { + const { readable, writable } = new IdentityTransformStream(); + const writer = writable.getWriter(); + const reader = readable.getReader(); + const buf = new Uint8Array([10, 20, 30]).buffer; + const readPromise = reader.read(); + await writer.write(buf); + const { value, done } = await readPromise; + strictEqual(done, false); + deepStrictEqual([...value], [10, 20, 30]); + await writer.close(); + }, +}; + +// Accepts DataView. +export const acceptsDataView = { + async test() { + const { readable, writable } = new IdentityTransformStream(); + const writer = writable.getWriter(); + const reader = readable.getReader(); + const buf = new Uint8Array([5, 6, 7, 8]).buffer; + const dv = new DataView(buf, 1, 2); // bytes [6, 7] + const readPromise = reader.read(); + await writer.write(dv); + const { value, done } = await readPromise; + strictEqual(done, false); + deepStrictEqual([...value], [6, 7]); + await writer.close(); + }, +}; + +// Rejects non-byte-source chunks with TypeError. +export const rejectsInvalidChunks = { + async test() { + const { writable } = new IdentityTransformStream(); + const writer = writable.getWriter(); + await rejects(writer.write(42), TypeError); + }, +}; + +// Rejects objects with TypeError. +export const rejectsObjectChunks = { + async test() { + const { writable } = new IdentityTransformStream(); + const writer = writable.getWriter(); + await rejects(writer.write({ data: 'nope' }), TypeError); + }, +}; + +// --- Zero-length write is a no-op --- + +export const zeroLengthUint8ArrayIsNoop = { + async test() { + const { readable, writable } = new IdentityTransformStream(); + const writer = writable.getWriter(); + const reader = readable.getReader(); + // Write zero-length then real data — only the real data appears. + await writer.write(new Uint8Array(0)); + writer.write(new Uint8Array([42])); + const { value, done } = await reader.read(); + strictEqual(done, false); + deepStrictEqual([...value], [42]); + await writer.close(); + }, +}; + +export const zeroLengthStringIsNoop = { + async test() { + const { readable, writable } = new IdentityTransformStream(); + const writer = writable.getWriter(); + const reader = readable.getReader(); + await writer.write(''); + writer.write('x'); + const { value, done } = await reader.read(); + strictEqual(done, false); + strictEqual(new TextDecoder().decode(value), 'x'); + await writer.close(); + }, +}; + +export const zeroLengthArrayBufferIsNoop = { + async test() { + const { readable, writable } = new IdentityTransformStream(); + const writer = writable.getWriter(); + const reader = readable.getReader(); + await writer.write(new ArrayBuffer(0)); + writer.write(new Uint8Array([99])); + const { value, done } = await reader.read(); + strictEqual(done, false); + deepStrictEqual([...value], [99]); + await writer.close(); + }, +}; + +// --- Copy semantics: writes always copy, never transfer --- + +export const writeCopiesData = { + async test() { + const { readable, writable } = new IdentityTransformStream(); + const writer = writable.getWriter(); + const reader = readable.getReader(); + const original = new Uint8Array([1, 2, 3]); + writer.write(original); + const { value } = await reader.read(); + // Mutating the original after write should not affect the read value. + original[0] = 99; + strictEqual(value[0], 1); + // The buffers must be different objects. + ok(value.buffer !== original.buffer); + await writer.close(); + }, +}; + +// --- Write subarray: offset+length are respected --- + +export const writeSubarray = { + async test() { + const { readable, writable } = new IdentityTransformStream(); + const writer = writable.getWriter(); + const reader = readable.getReader(); + const u8 = new Uint8Array([1, 2, 3, 4]); + writer.write(u8.subarray(1, 3)); + writer.close(); + const { value } = await reader.read(); + strictEqual(value.length, 2); + strictEqual(value[0], u8[1]); + strictEqual(value[1], u8[2]); + }, +}; + +// --- Readable side is a byte stream (BYOB capable) --- + +export const readableIsByteStream = { + async test() { + const { readable, writable } = new IdentityTransformStream(); + const writer = writable.getWriter(); + const reader = readable.getReader({ mode: 'byob' }); + writer.write(new Uint8Array([10, 20, 30])); + const { value, done } = await reader.read(new Uint8Array(10)); + strictEqual(done, false); + ok(value instanceof Uint8Array); + deepStrictEqual([...value.slice(0, 3)], [10, 20, 30]); + await writer.close(); + }, +}; + +// --- Write before read (backpressure) --- + +export const writeBeforeRead = { + async test() { + const MAX_RW = 10; + const { readable, writable } = new IdentityTransformStream(); + const writer = writable.getWriter(); + const reader = readable.getReader(); + const writePromises = []; + for (let i = 0; i < MAX_RW; i++) { + writePromises.push(writer.write(new Uint8Array([i]))); + } + const chunks = []; + for (let i = 0; i < MAX_RW; i++) { + chunks.push(await reader.read()); + } + await Promise.all(writePromises); + for (let i = 0; i < chunks.length; i++) { + deepStrictEqual([...chunks[i].value], [i]); + strictEqual(chunks[i].done, false); + } + const writeClosePromise = writer.close(); + const chunk = await reader.read(); + await writeClosePromise; + strictEqual(chunk.done, true); + await writer.closed; + await reader.closed; + }, +}; + +// --- Read before write --- + +export const readBeforeWrite = { + async test() { + const MAX_RW = 10; + const { readable, writable } = new IdentityTransformStream(); + const writer = writable.getWriter(); + const reader = readable.getReader(); + const chunks = []; + for (let i = 0; i < MAX_RW; i++) { + const readPromise = reader.read(); + await writer.write(new Uint8Array([i])); + chunks.push(await readPromise); + } + for (let i = 0; i < chunks.length; i++) { + deepStrictEqual([...chunks[i].value], [i]); + strictEqual(chunks[i].done, false); + } + const readClosePromise = reader.read(); + await writer.close(); + const chunk = await readClosePromise; + strictEqual(chunk.done, true); + await writer.closed; + await reader.closed; + }, +}; + +// --- Close propagation --- + +export const closeSignalsProperly = { + async test() { + const { readable, writable } = new IdentityTransformStream(); + const writer = writable.getWriter(); + await writer.close(); + const reader = readable.getReader({ mode: 'byob' }); + const result = await reader.read(new Uint8Array(10)); + strictEqual(result.done, true); + ok(result.value instanceof Uint8Array); + strictEqual(result.value.byteLength, 0); + strictEqual(result.value.buffer.byteLength, 10); + }, +}; + +// --- Abort propagation --- + +export const abortPropagation = { + async test() { + const { readable, writable } = new IdentityTransformStream(); + const writer = writable.getWriter(); + const reader = readable.getReader(); + const readPromise = reader.read(); + await writer.abort(new Error('test abort')); + await rejects(readPromise, Error); + await rejects(reader.closed, Error); + await rejects(writer.closed, Error); + }, +}; + +// --- Cancel propagation --- + +export const cancelPropagation = { + async test() { + const { readable, writable } = new IdentityTransformStream(); + const writer = writable.getWriter(); + const reader = readable.getReader(); + const writePromise = writer.write(enc.encode('test')); + const closePromise = writer.close(); + await reader.cancel(new Error('cancel reason')); + await rejects(writePromise); + await rejects(closePromise); + }, +}; + +// --- Multi-chunk read-all tests --- + +export const readAllBytes = { + async test() { + const { readable, writable } = new IdentityTransformStream(); + const writer = writable.getWriter(); + const reader = readable.getReader(); + const N = 4; + const M = 1000; + const writePromise = (async () => { + for (let i = 0; i < N; i++) { + const chunk = new Uint8Array(M); + chunk.fill(i + 1); + await writer.write(chunk); + } + await writer.close(); + })(); + // Collect all chunks via the reader. + const chunks = []; + while (true) { + const { value, done } = await reader.read(); + if (done) break; + chunks.push(value); + } + const totalLength = chunks.reduce((s, c) => s + c.byteLength, 0); + strictEqual(totalLength, N * M); + const body = new Uint8Array(totalLength); + let offset = 0; + for (const chunk of chunks) { + body.set(chunk, offset); + offset += chunk.byteLength; + } + for (let i = 0; i < body.length; i++) { + strictEqual(body[i], Math.floor(i / M) + 1); + } + await writePromise; + }, +}; + +export const readAllText = { + async test() { + const { readable, writable } = new IdentityTransformStream(); + const writer = writable.getWriter(); + const reader = readable.getReader(); + const dec = new TextDecoder(); + const writePromise = (async () => { + await writer.write('hello '); + await writer.write('world'); + await writer.close(); + })(); + // Collect all chunks via the reader. + let text = ''; + while (true) { + const { value, done } = await reader.read(); + if (done) break; + text += dec.decode(value, { stream: true }); + } + text += dec.decode(); + strictEqual(text, 'hello world'); + await writePromise; + }, +}; + +// --- FixedLengthStream --- + +export const fixedLengthStreamBasic = { + async test() { + const fls = new FixedLengthStream(5); + const writer = fls.writable.getWriter(); + const reader = fls.readable.getReader(); + const readPromise = reader.read(); + await writer.write(enc.encode('hello')); + await writer.close(); + const { value, done } = await readPromise; + strictEqual(done, false); + strictEqual(new TextDecoder().decode(value), 'hello'); + const result2 = await reader.read(); + strictEqual(result2.done, true); + }, +}; + +export const fixedLengthStreamPreconditions = { + test() { + // Can construct with zero. + new FixedLengthStream(0); + // Can construct with negative zero (coerced to 0n internally). + new FixedLengthStream(-0.0); + // Can construct with MAX_SAFE_INTEGER. + new FixedLengthStream(Number.MAX_SAFE_INTEGER); + // Can construct with bigint. + new FixedLengthStream(100n); + // Can construct with max uint64_t. + new FixedLengthStream(0xffff_ffff_ffff_ffffn); + // Cannot construct with non-integer (fraction) — BigInt(0.5) throws RangeError. + throws(() => new FixedLengthStream(0.5), RangeError); + // Cannot construct with NaN — BigInt(NaN) throws RangeError. + throws(() => new FixedLengthStream(NaN), RangeError); + // Cannot construct with negative integer. + throws(() => new FixedLengthStream(-1), RangeError); + // Cannot construct with negative bigint. + throws(() => new FixedLengthStream(-1n), RangeError); + // Cannot construct with bigint exceeding uint64_t max. + throws(() => new FixedLengthStream(0x1_0000_0000_0000_0000n), RangeError); + // Cannot construct with non-number/non-bigint. + throws(() => new FixedLengthStream('10'), TypeError); + }, +}; + +export const teeFixedLengthStreamNoHang = { + async test() { + const ts = new FixedLengthStream(11); + const writer = ts.writable.getWriter(); + writer.write(enc.encode('foo bar baz')); + writer.close(); + const [left, _right] = ts.readable.tee(); + // Read from the teed branch directly via a reader. + const reader = left.getReader(); + const dec = new TextDecoder(); + let text = ''; + while (true) { + const { value, done } = await reader.read(); + if (done) break; + text += dec.decode(value, { stream: true }); + } + text += dec.decode(); + strictEqual(text, 'foo bar baz'); + }, +}; + +// --- Lock release behavior --- + +export const closedPromiseUnderLockRelease = { + async test() { + const { readable, writable } = new IdentityTransformStream(); + const writer = writable.getWriter(); + const reader = readable.getReader(); + const writerClosed = writer.closed; + const readerClosed = reader.closed; + writer.releaseLock(); + await rejects(writerClosed, TypeError); + reader.releaseLock(); + await rejects(readerClosed, TypeError); + }, +}; + +// --- writableStrategy / highWaterMark forwarding --- + +// ITS with explicit HWM: initial desiredSize equals HWM. +export const itsWithHWMSetsDesiredSize = { + test() { + const { writable } = new IdentityTransformStream({ highWaterMark: 10 }); + const writer = writable.getWriter(); + strictEqual(writer.desiredSize, 10); + }, +}; + +// FLS: HWM larger than expectedLength is capped. +export const flsHWMCappedAtExpectedLength = { + test() { + const fls = new FixedLengthStream(10, { highWaterMark: 100 }); + const writer = fls.writable.getWriter(); + strictEqual(writer.desiredSize, 10); + }, +}; + +// FLS: HWM smaller than expectedLength is kept as-is. +export const flsHWMNotCappedWhenSmaller = { + test() { + const fls = new FixedLengthStream(100, { highWaterMark: 5 }); + const writer = fls.writable.getWriter(); + strictEqual(writer.desiredSize, 5); + }, +}; + +// FLS with no writableStrategy: default HWM (1). +export const flsNoHWMDefaultBehavior = { + test() { + const fls = new FixedLengthStream(10); + const writer = fls.writable.getWriter(); + strictEqual(writer.desiredSize, 1); + }, +}; + +// ITS with no writableStrategy: default HWM (1). +export const itsNoHWMDefaultBehavior = { + test() { + const { writable } = new IdentityTransformStream(); + const writer = writable.getWriter(); + strictEqual(writer.desiredSize, 1); + }, +}; + +// FLS with bigint expectedLength: HWM capping works. +export const flsBigintExpectedLengthCapsHWM = { + test() { + const fls = new FixedLengthStream(10n, { highWaterMark: 100 }); + const writer = fls.writable.getWriter(); + strictEqual(writer.desiredSize, 10); + }, +}; + +// ITS with HWM: desiredSize tracks byte count (not chunk count), +// including bytes currently being processed (in-flight). This matches +// the C++ WritableStreamInternalController.adjustWriteBufferSize model +// where all pipeline bytes are counted until fully consumed. +export const itsHWMByteLevelQueueTracking = { + async test() { + const { readable, writable } = new IdentityTransformStream({ + highWaterMark: 20, + }); + const writer = writable.getWriter(); + const reader = readable.getReader(); + + strictEqual(writer.desiredSize, 20); + + // Writes stay counted in desiredSize until fully consumed. + writer.write(new Uint8Array(5)); + strictEqual(writer.desiredSize, 15); // 20 - 5 + + writer.write(new Uint8Array(5)); + strictEqual(writer.desiredSize, 10); // 20 - 10 + + writer.write(new Uint8Array(7)); + strictEqual(writer.desiredSize, 3); // 20 - 17 + + // Read all chunks to drain. After each read resolves a write, + // the bytes are subtracted and desiredSize recovers. + await reader.read(); + await reader.read(); + await reader.read(); + + // All writes consumed — desiredSize back to HWM. + strictEqual(writer.desiredSize, 20); + + await writer.close(); + }, +}; + +// ITS with HWM + string writes: byteSize uses str.length * 3 as a +// conservative upper-bound estimate (max UTF-8 bytes per UTF-16 code +// unit). This is intentional — byteSize only feeds backpressure +// signaling, so overcounting is harmless (just signals backpressure +// slightly earlier). +export const itsHWMStringOverestimate = { + async test() { + const { readable, writable } = new IdentityTransformStream({ + highWaterMark: 100, + }); + const writer = writable.getWriter(); + const reader = readable.getReader(); + + strictEqual(writer.desiredSize, 100); + + // "hello" is 5 chars, all ASCII → 5 actual UTF-8 bytes. + // byteSize estimates 5 * 3 = 15. + writer.write('hello'); + strictEqual(writer.desiredSize, 85); // 100 - 15 + + // After reading, the estimate is subtracted back. + await reader.read(); + strictEqual(writer.desiredSize, 100); + + await writer.close(); + }, +}; + +// Mirrors the C++ identitytransformstream-backpressure-test: desiredSize +// tracking with interleaved reads after the controller has started. +export const itsHWMBackpressureMatchesCpp = { + async test() { + const ts = new IdentityTransformStream({ highWaterMark: 10 }); + const writer = ts.writable.getWriter(); + const reader = ts.readable.getReader(); + + strictEqual(writer.desiredSize, 10); + + // Wait for writer.ready so the controller's start has resolved. + const firstReady = writer.ready; + await writer.ready; + + writer.write(new Uint8Array(1)); + strictEqual(writer.desiredSize, 9); + + // A second write that fills the buffer completely. + writer.write(new Uint8Array(9)); + strictEqual(writer.desiredSize, 0); + + // The ready promise should have been replaced (backpressure on). + notStrictEqual(firstReady, writer.ready); + + async function waitForReady() { + strictEqual(writer.desiredSize, 0); + await writer.ready; + // After reading 1 byte, backpressure relieved by that amount. + strictEqual(writer.desiredSize, 1); + } + + await Promise.all([waitForReady(), reader.read()]); + + // Read the remaining 9 bytes. + await reader.read(); + strictEqual(writer.desiredSize, 10); + }, +}; + +// FLS: HWM forwarding + capping work together with data flow. +export const flsHWMForwardingWithDataFlow = { + async test() { + const data = 'hello world, padding'; // exactly 20 bytes + const fls = new FixedLengthStream(data.length, { highWaterMark: 50 }); + const writer = fls.writable.getWriter(); + const reader = fls.readable.getReader(); + // Capped: min(50, 20) = 20. + strictEqual(writer.desiredSize, data.length); + const readPromise = reader.read(); + await writer.write(data); + const { value, done } = await readPromise; + strictEqual(done, false); + strictEqual(new TextDecoder().decode(value), data); + await writer.close(); + }, +}; + +// --- Illegal invocation on wrong receiver --- + +// --- FixedLengthStream overwrite / underwrite enforcement --- + +export const flsOverwriteThrows = { + async test() { + const fls = new FixedLengthStream(5); + const writer = fls.writable.getWriter(); + const reader = fls.readable.getReader(); + // Write exactly 5 bytes — reader unblocks the rendezvous. + const writeP = writer.write(new Uint8Array([1, 2, 3, 4, 5])); + await reader.read(); // drain to unblock the write + await writeP; + // Now remaining is 0; next write fails before the rendezvous. + await rejects( + () => writer.write(new Uint8Array([6])), + (err) => { + ok(err instanceof RangeError); + ok( + err.message.includes( + 'Attempt to write too many bytes through a FixedLengthStream' + ) + ); + return true; + } + ); + }, +}; + +export const flsOverwriteSingleChunkThrows = { + async test() { + // Single write exceeding the limit. + const fls = new FixedLengthStream(3); + const writer = fls.writable.getWriter(); + await rejects( + () => writer.write(new Uint8Array([1, 2, 3, 4])), + (err) => { + ok(err instanceof RangeError); + ok(err.message.includes('too many bytes')); + return true; + } + ); + }, +}; + +export const flsUnderwriteThrows = { + async test() { + const fls = new FixedLengthStream(10); + const writer = fls.writable.getWriter(); + const reader = fls.readable.getReader(); + // Write 3 of 10, drain to unblock, then close. + const writeP = writer.write(new Uint8Array([1, 2, 3])); + await reader.read(); + await writeP; + await rejects( + () => writer.close(), + (err) => { + ok(err instanceof RangeError); + ok( + err.message.includes('did not see all expected bytes before close()') + ); + return true; + } + ); + }, +}; + +export const flsUnderwriteZeroBytesThrows = { + async test() { + // Close immediately without writing anything. + const fls = new FixedLengthStream(5); + const writer = fls.writable.getWriter(); + await rejects( + () => writer.close(), + (err) => { + ok(err instanceof RangeError); + ok(err.message.includes('did not see all expected bytes')); + return true; + } + ); + }, +}; + +export const flsExactWriteSucceeds = { + async test() { + const fls = new FixedLengthStream(6); + const writer = fls.writable.getWriter(); + const reader = fls.readable.getReader(); + // Write in two chunks totaling exactly 6 bytes. + // Interleave reads to unblock the rendezvous. + const w1 = writer.write(new Uint8Array([1, 2, 3])); + const r1 = await reader.read(); + await w1; + strictEqual(r1.value.byteLength, 3); + const w2 = writer.write(new Uint8Array([4, 5, 6])); + const r2 = await reader.read(); + await w2; + strictEqual(r2.value.byteLength, 3); + await writer.close(); + const r3 = await reader.read(); + strictEqual(r3.done, true); + }, +}; + +export const flsAbortSkipsUnderwriteCheck = { + async test() { + // Abort should NOT trigger the underwrite error — only 1 of 100 + // bytes written, but abort is not close, so no underwrite check. + const fls = new FixedLengthStream(100); + const writer = fls.writable.getWriter(); + const reader = fls.readable.getReader(); + const writeP = writer.write(new Uint8Array([1])); + await reader.read(); // drain to unblock + await writeP; + // Abort should resolve without throwing underwrite RangeError. + await writer.abort(new Error('cancelled')); + // Writable is errored (aborted), not just closed normally. + strictEqual(fls.writable.locked, true); + writer.releaseLock(); + strictEqual(fls.writable.locked, false); + }, +}; + +export const flsOverwriteErrorsReadable = { + async test() { + // After overwrite, the readable side should also be errored. + const fls = new FixedLengthStream(2); + const writer = fls.writable.getWriter(); + const reader = fls.readable.getReader(); + await rejects( + () => writer.write(new Uint8Array([1, 2, 3])), + (err) => err instanceof RangeError + ); + // Readable should be errored too. + await rejects( + () => reader.read(), + (err) => err instanceof RangeError + ); + }, +}; + +// --- expectedLength / Content-Length integration via DrainingReader --- + +export const flsExpectedLengthViaReader = { + test() { + // FixedLengthStream(42) should expose expectedLength=42n through a + // DrainingReader on its readable side (the C++ Content-Length path). + const fls = new FixedLengthStream(42); + const reader = new ReadableStreamDrainingReader(fls.readable); + strictEqual(reader.expectedLength, 42n); + reader.cancel(); + }, +}; + +export const flsBigintExpectedLengthViaReader = { + test() { + // bigint expectedLength should pass through as-is. + const fls = new FixedLengthStream(9007199254740993n); + const reader = new ReadableStreamDrainingReader(fls.readable); + strictEqual(reader.expectedLength, 9007199254740993n); + reader.cancel(); + }, +}; + +export const itsNoExpectedLengthViaReader = { + test() { + // Plain IdentityTransformStream has no expectedLength (undefined → + // chunked encoding). + const its = new IdentityTransformStream(); + const reader = new ReadableStreamDrainingReader(its.readable); + strictEqual(reader.expectedLength, undefined); + reader.cancel(); + }, +}; + +export const flsExpectedLengthZeroViaReader = { + test() { + // FixedLengthStream(0) — valid, means "the source will close without + // delivering any bytes". expectedLength should be 0n. + const fls = new FixedLengthStream(0); + const reader = new ReadableStreamDrainingReader(fls.readable); + strictEqual(reader.expectedLength, 0n); + reader.cancel(); + }, +}; + +export const illegalInvocation = { + test() { + const its = new IdentityTransformStream(); + const desc = Object.getOwnPropertyDescriptor( + Object.getPrototypeOf(its), + 'readable' + ); + throws(() => desc.get.call({}), TypeError); + throws(() => desc.get.call(null), TypeError); + }, +}; diff --git a/src/workerd/api/tests/ts-identity-streams-test.wd-test b/src/workerd/api/tests/ts-identity-streams-test.wd-test new file mode 100644 index 00000000000..91cfe2cae9b --- /dev/null +++ b/src/workerd/api/tests/ts-identity-streams-test.wd-test @@ -0,0 +1,22 @@ +using Workerd = import "/workerd/workerd.capnp"; + +const unitTests :Workerd.Config = ( + services = [ + ( name = "ts-identity-streams-test", + worker = ( + modules = [ + (name = "worker", esModule = embed "ts-identity-streams-test.js") + ], + compatibilityFlags = [ + "nodejs_compat", + "typescript_implemented_streams", + "expose_draining_reader", + "experimental", + ], + ) + ), + ], + autogates = [ + "workerd-autogate-per-isolate-javascript-bootstrap", + ], +); diff --git a/src/workerd/io/compatibility-date.capnp b/src/workerd/io/compatibility-date.capnp index caf148fecd9..de851318b80 100644 --- a/src/workerd/io/compatibility-date.capnp +++ b/src/workerd/io/compatibility-date.capnp @@ -1616,4 +1616,13 @@ struct CompatibilityFlags @0x8f8c1b68151b6cef { $experimental; # When enabled, the workers runtime uses the new typescript Web Streams # implementation. + + exposeDrainingReader @184 :Bool + $compatEnableFlag("expose_draining_reader") + $experimental; + # Exposes the internal ReadableStreamDrainingReader class on globalThis. + # The DrainingReader provides the expectedLength pass-through for the + # C++ bridge (Content-Length integration for FixedLengthStream). This + # flag is intended for internal testing only and may never have its + # experimental annotation removed. } From 4d3a15aae6c0847ba222a92506e014af66dcd9c2 Mon Sep 17 00:00:00 2001 From: Dan Lapid Date: Fri, 10 Jul 2026 01:04:05 +0000 Subject: [PATCH 056/101] Add span for access getIdentity --- src/workerd/api/global-scope.c++ | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/workerd/api/global-scope.c++ b/src/workerd/api/global-scope.c++ index 593549b7680..fca7885d930 100644 --- a/src/workerd/api/global-scope.c++ +++ b/src/workerd/api/global-scope.c++ @@ -114,7 +114,10 @@ jsg::Promise AccessContext::getIdentity(jsg::Lock& js, // embedder-supplied subrequest channel. If no identity service channel is available for this // request (e.g. service-token auth), there is no identity to fetch, so resolve to `undefined`. // Otherwise the binding worker's result (or error) propagates to the caller unchanged. + auto& context = IoContext::current(); + auto span = context.makeTraceSpan("access_get_identity"_kjc); KJ_IF_SOME(channel, info->getIdentityServiceChannel()) { + span.setTag("access.has_identity_service"_kjc, true); auto fetcher = js.alloc(channel, Fetcher::RequiresHostAndProtocol::NO, true /* isInHouse */); auto rpcProp = JSG_REQUIRE_NONNULL(fetcher->getRpcMethodInternal(js, kj::str("getIdentity")), @@ -128,8 +131,9 @@ jsg::Promise AccessContext::getIdentity(jsg::Lock& js, // via a resolver so we're independent of the `unwrapCustomThenables` compat flag. auto paf = js.newPromiseAndResolver(); paf.resolver.resolve(js, getIdentityFn(js)); - return kj::mv(paf.promise); + return context.attachSpans(js, kj::mv(paf.promise), kj::mv(span)); } + span.setTag("access.has_identity_service"_kjc, false); return js.resolvedPromise(jsg::Value(js.v8Isolate, v8::Undefined(js.v8Isolate))); } From 216f5e963dee14d4cf8b038a16219319ec2af007 Mon Sep 17 00:00:00 2001 From: Logan Gatlin Date: Fri, 10 Jul 2026 21:41:04 +0000 Subject: [PATCH 057/101] fix: incorrect handling of `node:process` imports in new module registry * extra tests * Allow shadowing of node:process * init See merge request cloudflare/ew/workerd!453 --- .../api/tests/new-module-registry-test.js | 31 ++++ src/workerd/jsg/modules-new-test.c++ | 128 +++++++++++++++++ src/workerd/jsg/modules-new.c++ | 135 +++++++++++------- 3 files changed, 240 insertions(+), 54 deletions(-) diff --git a/src/workerd/api/tests/new-module-registry-test.js b/src/workerd/api/tests/new-module-registry-test.js index cb7a49a8cfb..9d2191f1cd6 100644 --- a/src/workerd/api/tests/new-module-registry-test.js +++ b/src/workerd/api/tests/new-module-registry-test.js @@ -15,6 +15,7 @@ import { default as fs } from 'node:fs'; import { Buffer } from 'buffer'; // Intentionally omit the 'node:' prefix import { foo as foo2, default as def2 } from 'bar'; import { createRequire } from 'module'; // Intentionally omit the 'node:' prefix +import { default as processStatic } from 'node:process'; import * as workers from 'cloudflare:workers'; strictEqual(typeof workers, 'object'); @@ -460,6 +461,36 @@ export const createRequireFromEval = { }, }; +// Regression test: `node:process` (and bare `process`) is redirected to an +// internal module (node-internal:public_process / legacy_process) that only +// resolves in the builtin bucket. The static-import and require() paths force a +// BUILTIN_ONLY resolve context for the redirect, but the dynamic-import path +// previously derived its resolve context type from the *referrer* (a bundle +// module), so `await import('node:process')` failed with +// "Module not found: node-internal:public_process". This pins all three routes +// to the same instance so the dynamic path can't silently regress again. +// See maybeRedirectNodeProcess + dynamicResolve in jsg/modules-new.c++. +export const processRedirectAcrossResolutionRoutes = { + async test() { + const myRequire = createRequire(import.meta.url); + + // Static import (the branch that already worked). + ok(processStatic, 'node:process default export should exist'); + strictEqual(typeof processStatic.nextTick, 'function'); + + // Dynamic import, both prefixed and bare. This is the branch that used to + // throw "Module not found: node-internal:public_process". + const viaNode = await import('node:process'); + const viaBare = await import('process'); + strictEqual(typeof viaNode.default.nextTick, 'function'); + + // Every route redirects to the same single internal process instance. + strictEqual(viaNode.default, viaBare.default); + strictEqual(viaNode.default, processStatic); + strictEqual(myRequire('node:process'), processStatic); + }, +}; + // TODO(now): Tests // * [x] Include tests for all known module types // * [x] ESM diff --git a/src/workerd/jsg/modules-new-test.c++ b/src/workerd/jsg/modules-new-test.c++ index 519ceb1c9c6..22398a19fdf 100644 --- a/src/workerd/jsg/modules-new-test.c++ +++ b/src/workerd/jsg/modules-new-test.c++ @@ -728,6 +728,134 @@ KJ_TEST("Bundle shadows built-in") { // ====================================================================================== +KJ_TEST("A worker bundle module can shadow node:process") { + // Regression test: unlike every other built-in (e.g. node:buffer, which *can* be + // shadowed by a same-named bundle module -- see "Bundle shadows built-in" above), + // "node:process" used to be unconditionally redirected to an internal + // node-internal:public_process/legacy_process module *before* the worker bundle + // ever had a chance to resolve it. This meant a worker bundle module registered + // under "node:process" was silently unreachable. Verify that a worker bundle + // module named "node:process" now takes priority over the internal redirect, + // exactly as it would for any other built-in. + PREAMBLE([&](Lock& js) { + ResolveObserver resolveObserver; + CompilationObserver compilationObserver; + ModuleRegistry::Builder registryBuilder(resolveObserver, BASE); + + // Fake stand-ins for the real node-internal:public_process / legacy_process + // modules. If the redirect were incorrectly taken instead of the shadowing + // bundle module below, we'd observe one of these values instead. + ModuleBundle::BuiltinBuilder internalBuilder(ModuleBundle::BuiltinBuilder::Type::BUILTIN_ONLY); + auto publicSource = "export default 'internal-public-process';"_kjc; + auto legacySource = "export default 'internal-legacy-process';"_kjc; + internalBuilder.addEsm("node-internal:public_process"_url, publicSource.asArray()); + internalBuilder.addEsm("node-internal:legacy_process"_url, legacySource.asArray()); + registryBuilder.add(internalBuilder.finish()); + + ModuleBundle::BundleBuilder bundleBuilder(BASE); + auto shadowSource = kj::str("export default 'shadowed-process';"); + bundleBuilder.addEsmModule("node:process", shadowSource); + registryBuilder.add(bundleBuilder.finish()); + + auto registry = registryBuilder.finish(); + auto attached = registry->attachToIsolate(js, compilationObserver); + + js.tryCatch([&] { + auto val = ModuleRegistry::resolve(js, "node:process"); + KJ_ASSERT(val.isString()); + KJ_ASSERT(kj::str(val) == "shadowed-process"_kjc); + }, [&](Value exception) { js.throwException(kj::mv(exception)); }); + }); +} + +// ====================================================================================== + +KJ_TEST("A worker bundle module can shadow node:process via dynamic import") { + // Companion to "A worker bundle module can shadow node:process", which only + // exercises the static resolve path (ModuleRegistry::resolve -> resolveCallback). + // The dynamic import path (dynamicImportModuleCallback -> + // IsolateModuleRegistry::dynamicResolve) previously derived its resolve context + // type from the referrer and applied the node:process -> internal redirect + // *before* consulting the worker bundle, so `await import('node:process')` + // bypassed a same-named bundle module. Verify the bundle module also wins on the + // dynamic-import path, so the shadow-priority fix in dynamicResolve() cannot + // silently regress. + PREAMBLE([&](Lock& js) { + ResolveObserver resolveObserver; + CompilationObserver compilationObserver; + ModuleRegistry::Builder registryBuilder(resolveObserver, BASE); + + // Fake stand-ins for the real node-internal:public_process / legacy_process + // modules. If the redirect were incorrectly taken instead of the shadowing + // bundle module below, we'd observe one of these values instead. + ModuleBundle::BuiltinBuilder internalBuilder(ModuleBundle::BuiltinBuilder::Type::BUILTIN_ONLY); + auto publicSource = "export default 'internal-public-process';"_kjc; + auto legacySource = "export default 'internal-legacy-process';"_kjc; + internalBuilder.addEsm("node-internal:public_process"_url, publicSource.asArray()); + internalBuilder.addEsm("node-internal:legacy_process"_url, legacySource.asArray()); + registryBuilder.add(internalBuilder.finish()); + + ModuleBundle::BundleBuilder bundleBuilder(BASE); + auto shadowSource = kj::str("export default 'shadowed-process';"); + bundleBuilder.addEsmModule("node:process", shadowSource); + // Entrypoint module that reaches node:process exclusively through dynamic + // import(), forcing resolution through dynamicResolve() rather than the + // static resolveCallback path. + auto mainSource = kj::str("export default (await import('node:process')).default;"); + bundleBuilder.addEsmModule("main", mainSource); + registryBuilder.add(bundleBuilder.finish()); + + auto registry = registryBuilder.finish(); + auto attached = registry->attachToIsolate(js, compilationObserver); + + js.tryCatch([&] { + auto val = ModuleRegistry::resolve(js, "file:///main", "default"_kjc); + KJ_ASSERT(val.isString()); + KJ_ASSERT(kj::str(val) == "shadowed-process"_kjc); + }, [&](Value exception) { js.throwException(kj::mv(exception)); }); + }); +} + +// ====================================================================================== + +KJ_TEST("node:process falls back to the internal module when not shadowed") { + // Companion to the test above: when no worker bundle module shadows + // "node:process", resolution must still fall back to the internal + // node-internal:public_process / legacy_process module selected by the + // enable_nodejs_process_v2 compat flag. + PREAMBLE([&](Lock& js) { + ResolveObserver resolveObserver; + CompilationObserver compilationObserver; + ModuleRegistry::Builder registryBuilder(resolveObserver, BASE); + + ModuleBundle::BuiltinBuilder internalBuilder(ModuleBundle::BuiltinBuilder::Type::BUILTIN_ONLY); + auto publicSource = "export default 'internal-public-process';"_kjc; + auto legacySource = "export default 'internal-legacy-process';"_kjc; + internalBuilder.addEsm("node-internal:public_process"_url, publicSource.asArray()); + internalBuilder.addEsm("node-internal:legacy_process"_url, legacySource.asArray()); + registryBuilder.add(internalBuilder.finish()); + + auto registry = registryBuilder.finish(); + auto attached = registry->attachToIsolate(js, compilationObserver); + + js.tryCatch([&] { + auto val = ModuleRegistry::resolve(js, "node:process"); + KJ_ASSERT(val.isString()); + KJ_ASSERT(kj::str(val) == "internal-legacy-process"_kjc); + }, [&](Value exception) { js.throwException(kj::mv(exception)); }); + + js.setNodeJsProcessV2Enabled(); + + js.tryCatch([&] { + auto val = ModuleRegistry::resolve(js, "node:process"); + KJ_ASSERT(val.isString()); + KJ_ASSERT(kj::str(val) == "internal-public-process"_kjc); + }, [&](Value exception) { js.throwException(kj::mv(exception)); }); + }); +} + +// ====================================================================================== + KJ_TEST("Attaching a module registry works") { PREAMBLE(([&](Lock& js) { ResolveObserver resolveObserver; diff --git a/src/workerd/jsg/modules-new.c++ b/src/workerd/jsg/modules-new.c++ index 34f97b3a4ea..d3426ec9764 100644 --- a/src/workerd/jsg/modules-new.c++ +++ b/src/workerd/jsg/modules-new.c++ @@ -448,6 +448,31 @@ class IsolateModuleRegistry final { return found.key.getHandle(js); } + // Nothing resolved it through the ordinary bundle/builtin search — e.g. no + // worker bundle module was registered under this exact specifier to shadow a + // built-in. node:process is special: unlike other built-ins, it has no direct + // registration of its own. It must be redirected to one of two internal + // implementations selected by the enable_nodejs_process_v2 compat flag. We only + // apply that redirect here, as a last resort, so that a worker bundle module + // that intentionally shadows "node:process" (exactly as it could for any other + // built-in) gets first crack at resolving it above. + KJ_IF_SOME(processUrl, maybeRedirectNodeProcess(js, context.normalizedSpecifier.getHref())) { + auto processSpec = kj::str(processUrl.getHref()); + ResolveContext processContext = { + .type = ResolveContext::Type::BUILTIN_ONLY, + .source = context.source, + .normalizedSpecifier = processUrl, + .referrerNormalizedSpecifier = context.referrerNormalizedSpecifier, + .rawSpecifier = processSpec.asPtr(), + }; + KJ_IF_SOME(found, lookupCache.find>(processContext)) { + return found.key.getHandle(js); + } + KJ_IF_SOME(found, resolveWithCaching(js, processContext)) { + return found.key.getHandle(js); + } + } + // Nothing found? Aw... fail! JSG_FAIL_REQUIRE(Error, kj::str("Module not found: ", context.normalizedSpecifier.getHref())); } @@ -482,8 +507,8 @@ class IsolateModuleRegistry final { TypeError, kj::str("Referring module not found in the registry: ", referrer.getHref())); // Now that we know the referrer module, we can set the context for the - // next resolve. In particular, the "type" of the context is determine - // by the type of the referring module. + // next resolve. The "type" of the context is determined by the type of + // the referring module. ResolveContext context = { .type = moduleTypeToResolveContextType(referring.module.type()), .source = ResolveContext::Source::DYNAMIC_IMPORT, @@ -553,6 +578,31 @@ class IsolateModuleRegistry final { return handleFoundModule(found); } + // Nothing resolved it through the ordinary bundle/builtin search — e.g. no + // worker bundle module was registered under this exact specifier to shadow a + // built-in. node:process is special: unlike other built-ins, it has no direct + // registration of its own. It must be redirected to one of two internal + // implementations selected by the enable_nodejs_process_v2 compat flag. We only + // apply that redirect here, as a last resort, so that a worker bundle module + // that intentionally shadows "node:process" gets first crack at resolving it + // above. + KJ_IF_SOME(processUrl, maybeRedirectNodeProcess(js, normalizedSpecifier.getHref())) { + auto processSpec = kj::str(processUrl.getHref()); + ResolveContext processContext = { + .type = ResolveContext::Type::BUILTIN_ONLY, + .source = ResolveContext::Source::DYNAMIC_IMPORT, + .normalizedSpecifier = processUrl, + .referrerNormalizedSpecifier = referrer, + .rawSpecifier = processSpec.asPtr(), + }; + KJ_IF_SOME(found, lookupCache.find>(processContext)) { + return handleFoundModule(found); + } + KJ_IF_SOME(found, resolveWithCaching(js, processContext)) { + return handleFoundModule(found); + } + } + // Nothing found? Aw... fail! // A module that cannot be resolved is a lookup failure, not a type error, // so this is an Error (matching the static-import and require paths, and @@ -717,17 +767,6 @@ class IsolateModuleRegistry final { }; return js.tryCatch([&]() -> v8::MaybeLocal { - KJ_IF_SOME(processUrl, maybeRedirectNodeProcess(js, context.normalizedSpecifier.getHref())) { - ResolveContext newContext{ - .type = ResolveContext::Type::BUILTIN_ONLY, - .source = context.source, - .normalizedSpecifier = processUrl, - .referrerNormalizedSpecifier = context.referrerNormalizedSpecifier, - .rawSpecifier = context.rawSpecifier, - }; - return require(js, newContext, option); - } - // Do we already have a cached module for this context? KJ_IF_SOME(found, lookupCache.find>(context)) { // Extract module handle and Module& before calling evaluate, since @@ -745,6 +784,25 @@ class IsolateModuleRegistry final { inner.getEvaluator(), option); } + // Nothing resolved it through the ordinary bundle/builtin search — e.g. no + // worker bundle module was registered under this exact specifier to shadow a + // built-in. node:process is special: unlike other built-ins, it has no direct + // registration of its own. It must be redirected to one of two internal + // implementations selected by the enable_nodejs_process_v2 compat flag. We only + // apply that redirect here, as a last resort, so that a worker bundle module + // that intentionally shadows "node:process" gets first crack at resolving it + // above. + KJ_IF_SOME(processUrl, maybeRedirectNodeProcess(js, context.normalizedSpecifier.getHref())) { + ResolveContext newContext{ + .type = ResolveContext::Type::BUILTIN_ONLY, + .source = context.source, + .normalizedSpecifier = processUrl, + .referrerNormalizedSpecifier = context.referrerNormalizedSpecifier, + .rawSpecifier = context.rawSpecifier, + }; + return require(js, newContext, option); + } + if ((option & RequireOption::RETURN_EMPTY) == RequireOption::RETURN_EMPTY) { return {}; } @@ -991,12 +1049,11 @@ v8::MaybeLocal dynamicImportModuleCallback(v8::Local c } } - // Handle process module redirection based on enable_nodejs_process_v2 flag - KJ_IF_SOME(processUrl, maybeRedirectNodeProcess(js, spec.asPtr())) { - auto processSpec = kj::str(processUrl.getHref()); - return registry.dynamicResolve( - js, processUrl.clone(), kj::mv(referrer), processSpec, isSourcePhase, importType); - } + // Note: node:process has no direct registration of its own in the builtin + // bundle; registry.dynamicResolve() below falls back to redirecting it to + // one of two internal implementations (selected by the + // enable_nodejs_process_v2 compat flag) only if nothing else — e.g. a worker + // bundle module intentionally shadowing "node:process" — resolves it first. KJ_IF_SOME(url, referrer.tryResolve(spec.asPtr())) { return registry.dynamicResolve(js, url.clone(Url::EquivalenceOption::NORMALIZE_PATH), @@ -1098,41 +1155,11 @@ v8::MaybeLocal> resolv } } - // Handle process module redirection based on enable_nodejs_process_v2 flag - if constexpr (!IsSourcePhase) { - KJ_IF_SOME(processUrl, maybeRedirectNodeProcess(js, spec.asPtr())) { - auto processSpec = kj::str(processUrl.getHref()); - ResolveContext resolveContext = { - .type = ResolveContext::Type::BUILTIN_ONLY, - .source = ResolveContext::Source::STATIC_IMPORT, - .normalizedSpecifier = processUrl, - .referrerNormalizedSpecifier = referrerUrl, - .rawSpecifier = processSpec.asPtr(), - }; - auto maybeResolved = registry.resolve(js, resolveContext); - v8::Local resolved; - if (!maybeResolved.ToLocal(&resolved)) { - return {}; - } - if (resolved->GetStatus() == v8::Module::kErrored) { - js.throwException(JsValue(resolved->GetException())); - return {}; - } - if (resolved->GetStatus() == v8::Module::kEvaluating) { - // A circular dependency is a module-graph/loading error, not a type - // error, so this is an Error (matching the require path and Node's - // ERR_REQUIRE_CYCLE_MODULE which extends Error). - js.throwException( - js.error(kj::str("Circular dependency when resolving module: ", spec))); - return {}; - } - // Validate import type attribute against the resolved module's content type. - KJ_IF_SOME(entry, registry.lookup(js, resolved)) { - validateImportType(js, importType, entry.module, spec); - } - return resolved; - } - } + // Note: node:process has no direct registration of its own in the builtin + // bundle; registry.resolve() below falls back to redirecting it to one of two + // internal implementations (selected by the enable_nodejs_process_v2 compat + // flag) only if nothing else — e.g. a worker bundle module intentionally + // shadowing "node:process" — resolves it first. KJ_IF_SOME(url, referrerUrl.tryResolve(spec)) { // Make sure that percent-encoding in the path is normalized so we can match correctly. From 5bdf81bf5233219f88212fd1fb9f2d648ef644e0 Mon Sep 17 00:00:00 2001 From: Dan Carney Date: Sat, 11 Jul 2026 11:05:40 +0000 Subject: [PATCH 058/101] Revert "Use non-sandbox buffer to read in internal.c++" This reverts commit 8ab143b2e82ab7b6a16b8c88b9dd49c8e21cfb5f. --- src/workerd/api/streams/internal.c++ | 61 ++++++++++++++++++---------- 1 file changed, 40 insertions(+), 21 deletions(-) diff --git a/src/workerd/api/streams/internal.c++ b/src/workerd/api/streams/internal.c++ index a8c479263fa..7d50a8ab55a 100644 --- a/src/workerd/api/streams/internal.c++ +++ b/src/workerd/api/streams/internal.c++ @@ -528,14 +528,27 @@ kj::Maybe> ReadableStreamInternalController::read( js.typeError("Unable to allocate memory for read"_kj)); } - // All reads go through a temporary kj-heap buffer outside the V8 sandbox - // in case the sandbox is MPK protected with a key not held by the kj - // sink called from the event loop. - // - // We will memcpy back into the user's BackingStore in the .then() - // continuation, after re-validating the BackingStore is still attached - // and large enough. - if (theStore->IsResizableByUserJavaScript()) { + // In the case the ArrayBuffer is detached/transfered while the read is pending, we + // need to make sure that the ptr remains stable, so we grab a shared ptr to the + // backing store and use that to get the pointer to the data. If the buffer is detached + // while the read is pending, this does mean that the read data will end up being lost, + // but there's not really a better option. The best we can do here is warn the user + // that this is happening so they can avoid doing it in the future. + // Also, the user really shouldn't do this because the read will end up completing into + // the detached backing store still which could cause issues with whatever code now actually + // owns the transfered buffer. Below we'll warn the user about this if it happens so they + // can avoid doing it in the future. + auto backing = theStore->GetBackingStore(); + + // For resizable ArrayBuffers, the buffer may be resized while the read is + // pending, decommitting memory pages and making the pointer invalid (SIGSEGV). + // We read into a temporary buffer and copy the data back in the .then() + // callback, where we can validate the buffer is still large enough. + bool isResizable = theStore->IsResizableByUserJavaScript(); + + kj::Array tempBuffer; + kj::byte* readPtr; + if (isResizable) { auto currentByteLength = theStore->ByteLength(); if (byteOffset >= currentByteLength) { readPending = false; @@ -551,15 +564,19 @@ kj::Maybe> ReadableStreamInternalController::read( atLeast = byteLength > 0 ? byteLength : 1; } } + tempBuffer = kj::heapArray(byteLength); + readPtr = tempBuffer.begin(); + } else { + auto ptr = static_cast(backing->Data()); + readPtr = ptr + byteOffset; } - - auto tempBuffer = kj::heapArray(byteLength); - auto bytes = tempBuffer.asPtr(); + auto bytes = kj::arrayPtr(readPtr, byteLength); KJ_ASSERT(atLeast <= bytes.size(), "minBytes must not exceed maxBytes in tryRead"); - auto promise = - kj::evalNow([&] { return readable->tryRead(bytes.begin(), atLeast, bytes.size()); }); + auto promise = kj::evalNow([&] { + return readable->tryRead(bytes.begin(), atLeast, bytes.size()).attach(kj::mv(backing)); + }); KJ_IF_SOME(readerLock, readState.tryGetUnsafe()) { promise = KJ_ASSERT_NONNULL(readerLock.getCanceler())->wrap(kj::mv(promise)); } @@ -575,10 +592,10 @@ kj::Maybe> ReadableStreamInternalController::read( auto& ioContext = IoContext::current(); return ioContext.awaitIoLegacy(js, kj::mv(promise)) .then(js, - ioContext.addFunctor( - [ref = addRef(), store = js.v8Ref(store), byteOffset, byteLength, - isByob = maybeByobOptions != kj::none, tempBuffer = kj::mv(tempBuffer)]( - jsg::Lock& js, size_t amount) mutable -> jsg::Promise { + ioContext.addFunctor([ref = addRef(), store = js.v8Ref(store), byteOffset, byteLength, + isByob = maybeByobOptions != kj::none, isResizable, readPtr, + tempBuffer = kj::mv(tempBuffer)](jsg::Lock& js, + size_t amount) mutable -> jsg::Promise { auto& controller = static_cast(ref->getController()); controller.readPending = false; KJ_ASSERT(amount <= byteLength); @@ -646,10 +663,12 @@ kj::Maybe> ReadableStreamInternalController::read( amount = handle->ByteLength() - byteOffset; } - // Copy the data from the kj-heap temporary buffer into the user's - // BackingStore. - auto destPtr = static_cast(handle->GetBackingStore()->Data()); - memcpy(destPtr + byteOffset, tempBuffer.begin(), amount); + if (isResizable && byteOffset + amount <= handle->ByteLength()) { + // For resizable buffers, the data was read into a temporary buffer. + // Copy it back into the user's (still valid) buffer region. + auto destPtr = static_cast(handle->GetBackingStore()->Data()); + memcpy(destPtr + byteOffset, readPtr, amount); + } auto u8 = v8::Uint8Array::New(store.getHandle(js), byteOffset, amount); return js.resolvedPromise(ReadResult{ From 935c6bf92fdd84d90b9cab2591015474c6a26305 Mon Sep 17 00:00:00 2001 From: Felix Hanau Date: Fri, 10 Jul 2026 20:37:46 -0400 Subject: [PATCH 059/101] [o11y] Emit warning when reporting trace with exception outcome without actual exception Include the event type in the log, which will make it easier to track down the different ways this happens. --- src/workerd/io/trace.c++ | 32 ++++++++++++++++++++++++++++++++ src/workerd/io/trace.h | 8 ++++++++ src/workerd/io/tracer.c++ | 7 +++++++ 3 files changed, 47 insertions(+) diff --git a/src/workerd/io/trace.c++ b/src/workerd/io/trace.c++ index 61e331a4483..5fdb7e7496c 100644 --- a/src/workerd/io/trace.c++ +++ b/src/workerd/io/trace.c++ @@ -399,6 +399,10 @@ ConnectEventInfo ConnectEventInfo::clone() const { return ConnectEventInfo(); } +kj::String ConnectEventInfo::toString() const { + return kj::str("ConnectEventInfo"); +} + FetchEventInfo::FetchEventInfo( kj::HttpMethod method, kj::String url, kj::String cfJson, kj::Array
headers) : method(method), @@ -492,6 +496,10 @@ ScheduledEventInfo ScheduledEventInfo::clone() const { return ScheduledEventInfo(scheduledTime, kj::str(cron)); } +kj::String ScheduledEventInfo::toString() const { + return kj::str("ScheduledEventInfo"); +} + AlarmEventInfo::AlarmEventInfo(kj::Date scheduledTime): scheduledTime(scheduledTime) {} AlarmEventInfo::AlarmEventInfo(rpc::Trace::AlarmEventInfo::Reader reader) @@ -505,6 +513,10 @@ AlarmEventInfo AlarmEventInfo::clone() const { return AlarmEventInfo(scheduledTime); } +kj::String AlarmEventInfo::toString() const { + return kj::str("AlarmEventInfo"); +} + QueueEventInfo::QueueEventInfo(kj::String queueName, uint32_t batchSize) : queueName(kj::mv(queueName)), batchSize(batchSize) {} @@ -522,6 +534,10 @@ QueueEventInfo QueueEventInfo::clone() const { return QueueEventInfo(kj::str(queueName), batchSize); } +kj::String QueueEventInfo::toString() const { + return kj::str("QueueEventInfo"); +} + EmailEventInfo::EmailEventInfo(kj::String mailFrom, kj::String rcptTo, uint32_t rawSize) : mailFrom(kj::mv(mailFrom)), rcptTo(kj::mv(rcptTo)), @@ -542,6 +558,10 @@ EmailEventInfo EmailEventInfo::clone() const { return EmailEventInfo(kj::str(mailFrom), kj::str(rcptTo), rawSize); } +kj::String EmailEventInfo::toString() const { + return kj::str("EmailEventInfo"); +} + namespace { kj::Vector getTraceItemsFromTraces( kj::ArrayPtr> traces) { @@ -571,6 +591,10 @@ TraceEventInfo TraceEventInfo::clone() const { return TraceEventInfo(KJ_MAP(item, traces) { return item.clone(); }); } +kj::String TraceEventInfo::toString() const { + return kj::str("TraceEventInfo"); +} + TracePreview::TracePreview(kj::String id, kj::String slug, kj::String name) : id(kj::mv(id)), slug(kj::mv(slug)), @@ -720,6 +744,14 @@ HibernatableWebSocketEventInfo::Type HibernatableWebSocketEventInfo::readFrom( KJ_UNREACHABLE; } +kj::String HibernatableWebSocketEventInfo::toString() const { + return kj::str("HibernatableWebSocketEventInfo"); +} + +kj::String CustomEventInfo::toString() const { + return kj::str("CustomEventInfo"); +} + FetchResponseInfo::FetchResponseInfo(uint16_t statusCode): statusCode(statusCode) {} FetchResponseInfo::FetchResponseInfo(rpc::Trace::FetchResponseInfo::Reader reader) diff --git a/src/workerd/io/trace.h b/src/workerd/io/trace.h index 0201cf91a2f..52c6c0eb8a8 100644 --- a/src/workerd/io/trace.h +++ b/src/workerd/io/trace.h @@ -425,6 +425,7 @@ class ConnectEventInfo { void copyTo(rpc::Trace::ConnectEventInfo::Builder builder) const; ConnectEventInfo clone() const; + kj::String toString() const; }; // Describes a scheduled request @@ -440,6 +441,7 @@ struct ScheduledEventInfo final { void copyTo(rpc::Trace::ScheduledEventInfo::Builder builder) const; ScheduledEventInfo clone() const; + kj::String toString() const; }; // Describes a Durable Object alarm request @@ -454,6 +456,7 @@ struct AlarmEventInfo final { void copyTo(rpc::Trace::AlarmEventInfo::Builder builder) const; AlarmEventInfo clone() const; + kj::String toString() const; }; // Describes a queue worker request @@ -469,6 +472,7 @@ struct QueueEventInfo final { void copyTo(rpc::Trace::QueueEventInfo::Builder builder) const; QueueEventInfo clone() const; + kj::String toString() const; }; // Describes an email request @@ -485,6 +489,7 @@ struct EmailEventInfo final { void copyTo(rpc::Trace::EmailEventInfo::Builder builder) const; EmailEventInfo clone() const; + kj::String toString() const; }; // Describes a buffered tail worker request @@ -530,6 +535,7 @@ struct TraceEventInfo final { void copyTo(rpc::Trace::TraceEventInfo::Builder builder) const; TraceEventInfo clone() const; + kj::String toString() const; }; // Describes a hibernatable web socket event @@ -554,12 +560,14 @@ struct HibernatableWebSocketEventInfo final { void copyTo(rpc::Trace::HibernatableWebSocketEventInfo::Builder builder) const; HibernatableWebSocketEventInfo clone() const; static Type readFrom(rpc::Trace::HibernatableWebSocketEventInfo::Reader reader); + kj::String toString() const; }; // Describes a custom event struct CustomEventInfo final { explicit CustomEventInfo() {}; CustomEventInfo(rpc::Trace::CustomEventInfo::Reader reader) {}; + kj::String toString() const; }; // Describes a fetch response diff --git a/src/workerd/io/tracer.c++ b/src/workerd/io/tracer.c++ index e2b4fbca228..3a5c87a0573 100644 --- a/src/workerd/io/tracer.c++ +++ b/src/workerd/io/tracer.c++ @@ -400,6 +400,13 @@ void WorkerTracer::setOutcome(EventOutcome outcome, kj::Duration cpuTime, kj::Du trace->cpuTime = cpuTime; trace->wallTime = wallTime; + if (outcome == EventOutcome::EXCEPTION && pipelineLogLevel != PipelineLogLevel::NONE && + !trace->exceededExceptionLimit && trace->exceptions.empty()) { + LOG_PERIODICALLY(WARNING, + "NOSENTRY reporting trace with exception outcome, but no actual exceptions", + trace->eventInfo); + } + // Defer reporting the actual outcome event to the WorkerTracer destructor: The outcome is // reported when the metrics request is deallocated, but with ctx.waitUntil() there might be spans // continuing to exist beyond that point. By the time the WorkerTracer is deallocated, the From 2c86b876a9f214949efd8957b8103c5c826ab996 Mon Sep 17 00:00:00 2001 From: Nuno Pereira Date: Fri, 10 Jul 2026 19:58:23 +0100 Subject: [PATCH 060/101] RAG-1400: add conversion output options to Markdown Conversion types --- src/cloudflare/internal/to-markdown-api.ts | 9 ++++++++- types/defines/to-markdown.d.ts | 9 ++++++++- types/generated-snapshot/experimental/index.d.ts | 8 ++++++-- types/generated-snapshot/experimental/index.ts | 8 ++++++-- types/generated-snapshot/index.d.ts | 8 ++++++-- types/generated-snapshot/index.ts | 8 ++++++-- 6 files changed, 40 insertions(+), 10 deletions(-) diff --git a/src/cloudflare/internal/to-markdown-api.ts b/src/cloudflare/internal/to-markdown-api.ts index 676ab5cb50a..83cee15b9db 100644 --- a/src/cloudflare/internal/to-markdown-api.ts +++ b/src/cloudflare/internal/to-markdown-api.ts @@ -15,12 +15,14 @@ export type MarkdownDocument = { blob: Blob; }; +export type OutputFormat = 'markdown' | 'text'; + export type ConversionResponse = | { id: string; name: string; mimeType: string; - format: 'markdown'; + format: OutputFormat; tokens: number; data: string; } @@ -41,7 +43,12 @@ export type EmbeddedImageConversionOptions = ImageConversionOptions & { maxConvertedImages?: number; }; +export type ConversionOutputOptions = { + format?: OutputFormat; +}; + export type ConversionOptions = { + output?: ConversionOutputOptions; html?: { images?: EmbeddedImageConversionOptions & { convertOGImage?: boolean }; hostname?: string; diff --git a/types/defines/to-markdown.d.ts b/types/defines/to-markdown.d.ts index fe628216ddc..b29cdc4ce23 100644 --- a/types/defines/to-markdown.d.ts +++ b/types/defines/to-markdown.d.ts @@ -3,11 +3,13 @@ export type MarkdownDocument = { blob: Blob; } +export type OutputFormat = 'markdown' | 'text'; + export type ConversionResponse = { id: string; name: string; mimeType: string; - format: 'markdown'; + format: OutputFormat; tokens: number; data: string; } | { @@ -27,7 +29,12 @@ export type EmbeddedImageConversionOptions = ImageConversionOptions & { maxConvertedImages?: number; }; +export type ConversionOutputOptions = { + format?: OutputFormat; +} + export type ConversionOptions = { + output?: ConversionOutputOptions, html?: { images?: EmbeddedImageConversionOptions & { convertOGImage?: boolean }; hostname?: string; diff --git a/types/generated-snapshot/experimental/index.d.ts b/types/generated-snapshot/experimental/index.d.ts index 04247055096..2179ceee815 100755 --- a/types/generated-snapshot/experimental/index.d.ts +++ b/types/generated-snapshot/experimental/index.d.ts @@ -16312,12 +16312,13 @@ type MarkdownDocument = { name: string; blob: Blob; }; +type OutputFormat = "markdown" | "text"; type ConversionResponse = | { id: string; name: string; mimeType: string; - format: "markdown"; + format: OutputFormat; tokens: number; data: string; } @@ -16335,7 +16336,11 @@ type EmbeddedImageConversionOptions = ImageConversionOptions & { convert?: boolean; maxConvertedImages?: number; }; +type ConversionOutputOptions = { + format?: OutputFormat; +}; type ConversionOptions = { + output?: ConversionOutputOptions; html?: { images?: EmbeddedImageConversionOptions & { convertOGImage?: boolean; @@ -17115,7 +17120,6 @@ interface WorkflowInstanceTerminateOptions { */ rollback?: boolean; } - interface WorkflowInstanceRestartOptions { /** * Restart from a specific step. If omitted, the instance restarts from the beginning. diff --git a/types/generated-snapshot/experimental/index.ts b/types/generated-snapshot/experimental/index.ts index 53d1dd39ae1..d5867f3e04f 100755 --- a/types/generated-snapshot/experimental/index.ts +++ b/types/generated-snapshot/experimental/index.ts @@ -16280,12 +16280,13 @@ export type MarkdownDocument = { name: string; blob: Blob; }; +export type OutputFormat = "markdown" | "text"; export type ConversionResponse = | { id: string; name: string; mimeType: string; - format: "markdown"; + format: OutputFormat; tokens: number; data: string; } @@ -16303,7 +16304,11 @@ export type EmbeddedImageConversionOptions = ImageConversionOptions & { convert?: boolean; maxConvertedImages?: number; }; +export type ConversionOutputOptions = { + format?: OutputFormat; +}; export type ConversionOptions = { + output?: ConversionOutputOptions; html?: { images?: EmbeddedImageConversionOptions & { convertOGImage?: boolean; @@ -17074,7 +17079,6 @@ export interface WorkflowInstanceTerminateOptions { */ rollback?: boolean; } - export interface WorkflowInstanceRestartOptions { /** * Restart from a specific step. If omitted, the instance restarts from the beginning. diff --git a/types/generated-snapshot/index.d.ts b/types/generated-snapshot/index.d.ts index 2f12217385e..80bb26d6682 100755 --- a/types/generated-snapshot/index.d.ts +++ b/types/generated-snapshot/index.d.ts @@ -15664,12 +15664,13 @@ type MarkdownDocument = { name: string; blob: Blob; }; +type OutputFormat = "markdown" | "text"; type ConversionResponse = | { id: string; name: string; mimeType: string; - format: "markdown"; + format: OutputFormat; tokens: number; data: string; } @@ -15687,7 +15688,11 @@ type EmbeddedImageConversionOptions = ImageConversionOptions & { convert?: boolean; maxConvertedImages?: number; }; +type ConversionOutputOptions = { + format?: OutputFormat; +}; type ConversionOptions = { + output?: ConversionOutputOptions; html?: { images?: EmbeddedImageConversionOptions & { convertOGImage?: boolean; @@ -16467,7 +16472,6 @@ interface WorkflowInstanceTerminateOptions { */ rollback?: boolean; } - interface WorkflowInstanceRestartOptions { /** * Restart from a specific step. If omitted, the instance restarts from the beginning. diff --git a/types/generated-snapshot/index.ts b/types/generated-snapshot/index.ts index 929e893acfd..d665dec061e 100755 --- a/types/generated-snapshot/index.ts +++ b/types/generated-snapshot/index.ts @@ -15632,12 +15632,13 @@ export type MarkdownDocument = { name: string; blob: Blob; }; +export type OutputFormat = "markdown" | "text"; export type ConversionResponse = | { id: string; name: string; mimeType: string; - format: "markdown"; + format: OutputFormat; tokens: number; data: string; } @@ -15655,7 +15656,11 @@ export type EmbeddedImageConversionOptions = ImageConversionOptions & { convert?: boolean; maxConvertedImages?: number; }; +export type ConversionOutputOptions = { + format?: OutputFormat; +}; export type ConversionOptions = { + output?: ConversionOutputOptions; html?: { images?: EmbeddedImageConversionOptions & { convertOGImage?: boolean; @@ -16426,7 +16431,6 @@ export interface WorkflowInstanceTerminateOptions { */ rollback?: boolean; } - export interface WorkflowInstanceRestartOptions { /** * Restart from a specific step. If omitted, the instance restarts from the beginning. From dff2d25c47aac974b7fda226b903998c82b63323 Mon Sep 17 00:00:00 2001 From: Dan Lapid Date: Mon, 13 Jul 2026 16:38:37 +0100 Subject: [PATCH 061/101] Support windows arm This has been confirmed to work in https://github.com/cloudflare/workerd/issues/2735 it runs the binary under emulation --- npm/lib/node-platform.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/npm/lib/node-platform.ts b/npm/lib/node-platform.ts index ae9264b4bd1..2901c08c3a3 100644 --- a/npm/lib/node-platform.ts +++ b/npm/lib/node-platform.ts @@ -18,6 +18,7 @@ export const knownPackages: Record = { "linux arm64 LE": "@cloudflare/workerd-linux-arm64", "linux x64 LE": "@cloudflare/workerd-linux-64", "win32 x64 LE": "@cloudflare/workerd-windows-64", + "win32 arm64 LE": "@cloudflare/workerd-windows-64", }; const maybeExeExtension = process.platform === "win32" ? ".exe" : ""; From 23133f67f5335f847672042700290b92e40ece18 Mon Sep 17 00:00:00 2001 From: Nuno Pereira Date: Mon, 13 Jul 2026 18:28:02 +0100 Subject: [PATCH 062/101] WOR-000: fix types for forward steps with delay functions --- types/defines/rpc.d.ts | 51 +++- .../experimental/index.d.ts | 57 +++- .../generated-snapshot/experimental/index.ts | 57 +++- types/generated-snapshot/index.d.ts | 57 +++- types/generated-snapshot/index.ts | 57 +++- types/test/types/rpc.ts | 262 ++++++++++++++++++ 6 files changed, 498 insertions(+), 43 deletions(-) diff --git a/types/defines/rpc.d.ts b/types/defines/rpc.d.ts index 1e22efb5282..f9f12e13d36 100644 --- a/types/defines/rpc.d.ts +++ b/types/defines/rpc.d.ts @@ -317,6 +317,28 @@ declare namespace CloudflareWorkersModule { sensitive?: WorkflowStepSensitivity; }; + // Internal discriminators used only for `WorkflowStep.do` overload + // resolution. They mirror `WorkflowStepConfig` but pin `retries.delay` to a + // single kind so the callback context can be narrowed based on the shape of + // the config argument (rather than on an inferred type parameter, which is + // lost when the caller supplies an explicit return-type argument). Not + // exported: they must not widen the public type surface. + type WorkflowStepConfigWithStaticDelay = Omit & { + retries?: { + limit: number; + delay: WorkflowDelayDuration | number; + backoff?: WorkflowBackoff; + }; + }; + + type WorkflowStepConfigWithDelayFunction = Omit & { + retries: { + limit: number; + delay: WorkflowDelayFunction; + backoff?: WorkflowBackoff; + }; + }; + export type WorkflowStepRollbackConfig = Pick< WorkflowStepConfig, 'retries' | 'timeout' @@ -387,18 +409,33 @@ declare namespace CloudflareWorkersModule { callback: (ctx: WorkflowStepContext) => Promise, rollbackOptions?: WorkflowStepRollbackOptions ): Promise; - do, const C extends WorkflowStepConfig>( + // The config overloads discriminate on the shape of `config.retries.delay` + // so the callback context reflects whether the resolved delay is present + // (static delay) or omitted (dynamic delay function). Each has a single + // type parameter, so an explicit return-type argument (`do(...)`) still + // resolves here. ORDERING IS LOAD-BEARING: the broad `WorkflowStepConfig` + // fallback MUST remain last, otherwise it shadows the discriminating + // overloads and narrowing is silently lost. + do>( + name: string, + config: WorkflowStepConfigWithDelayFunction, + callback: (ctx: WorkflowStepContext) => Promise, + rollbackOptions?: WorkflowStepRollbackOptions + ): Promise; + do>( name: string, - config: C, + config: WorkflowStepConfigWithStaticDelay, callback: ( - ctx: WorkflowStepContext< - C['retries'] extends { delay: infer D } - ? D - : WorkflowDelayDuration | number - > + ctx: WorkflowStepContext ) => Promise, rollbackOptions?: WorkflowStepRollbackOptions ): Promise; + do>( + name: string, + config: WorkflowStepConfig, + callback: (ctx: WorkflowStepContext) => Promise, + rollbackOptions?: WorkflowStepRollbackOptions + ): Promise; sleep: (name: string, duration: WorkflowSleepDuration) => Promise; sleepUntil: (name: string, timestamp: Date | number) => Promise; waitForEvent>( diff --git a/types/generated-snapshot/experimental/index.d.ts b/types/generated-snapshot/experimental/index.d.ts index 2179ceee815..6af00edf70c 100755 --- a/types/generated-snapshot/experimental/index.d.ts +++ b/types/generated-snapshot/experimental/index.d.ts @@ -15420,6 +15420,32 @@ declare namespace CloudflareWorkersModule { timeout?: WorkflowTimeoutDuration | number; sensitive?: WorkflowStepSensitivity; }; + // Internal discriminators used only for `WorkflowStep.do` overload + // resolution. They mirror `WorkflowStepConfig` but pin `retries.delay` to a + // single kind so the callback context can be narrowed based on the shape of + // the config argument (rather than on an inferred type parameter, which is + // lost when the caller supplies an explicit return-type argument). Not + // exported: they must not widen the public type surface. + type WorkflowStepConfigWithStaticDelay = Omit< + WorkflowStepConfig, + "retries" + > & { + retries?: { + limit: number; + delay: WorkflowDelayDuration | number; + backoff?: WorkflowBackoff; + }; + }; + type WorkflowStepConfigWithDelayFunction = Omit< + WorkflowStepConfig, + "retries" + > & { + retries: { + limit: number; + delay: WorkflowDelayFunction; + backoff?: WorkflowBackoff; + }; + }; export type WorkflowStepRollbackConfig = Pick< WorkflowStepConfig, "retries" | "timeout" @@ -15482,20 +15508,33 @@ declare namespace CloudflareWorkersModule { callback: (ctx: WorkflowStepContext) => Promise, rollbackOptions?: WorkflowStepRollbackOptions, ): Promise; - do, const C extends WorkflowStepConfig>( + // The config overloads discriminate on the shape of `config.retries.delay` + // so the callback context reflects whether the resolved delay is present + // (static delay) or omitted (dynamic delay function). Each has a single + // type parameter, so an explicit return-type argument (`do(...)`) still + // resolves here. ORDERING IS LOAD-BEARING: the broad `WorkflowStepConfig` + // fallback MUST remain last, otherwise it shadows the discriminating + // overloads and narrowing is silently lost. + do>( + name: string, + config: WorkflowStepConfigWithDelayFunction, + callback: (ctx: WorkflowStepContext) => Promise, + rollbackOptions?: WorkflowStepRollbackOptions, + ): Promise; + do>( name: string, - config: C, + config: WorkflowStepConfigWithStaticDelay, callback: ( - ctx: WorkflowStepContext< - C["retries"] extends { - delay: infer D; - } - ? D - : WorkflowDelayDuration | number - >, + ctx: WorkflowStepContext, ) => Promise, rollbackOptions?: WorkflowStepRollbackOptions, ): Promise; + do>( + name: string, + config: WorkflowStepConfig, + callback: (ctx: WorkflowStepContext) => Promise, + rollbackOptions?: WorkflowStepRollbackOptions, + ): Promise; sleep: (name: string, duration: WorkflowSleepDuration) => Promise; sleepUntil: (name: string, timestamp: Date | number) => Promise; waitForEvent>( diff --git a/types/generated-snapshot/experimental/index.ts b/types/generated-snapshot/experimental/index.ts index d5867f3e04f..9d15cda8da2 100755 --- a/types/generated-snapshot/experimental/index.ts +++ b/types/generated-snapshot/experimental/index.ts @@ -15398,6 +15398,32 @@ export declare namespace CloudflareWorkersModule { timeout?: WorkflowTimeoutDuration | number; sensitive?: WorkflowStepSensitivity; }; + // Internal discriminators used only for `WorkflowStep.do` overload + // resolution. They mirror `WorkflowStepConfig` but pin `retries.delay` to a + // single kind so the callback context can be narrowed based on the shape of + // the config argument (rather than on an inferred type parameter, which is + // lost when the caller supplies an explicit return-type argument). Not + // exported: they must not widen the public type surface. + type WorkflowStepConfigWithStaticDelay = Omit< + WorkflowStepConfig, + "retries" + > & { + retries?: { + limit: number; + delay: WorkflowDelayDuration | number; + backoff?: WorkflowBackoff; + }; + }; + type WorkflowStepConfigWithDelayFunction = Omit< + WorkflowStepConfig, + "retries" + > & { + retries: { + limit: number; + delay: WorkflowDelayFunction; + backoff?: WorkflowBackoff; + }; + }; export type WorkflowStepRollbackConfig = Pick< WorkflowStepConfig, "retries" | "timeout" @@ -15460,20 +15486,33 @@ export declare namespace CloudflareWorkersModule { callback: (ctx: WorkflowStepContext) => Promise, rollbackOptions?: WorkflowStepRollbackOptions, ): Promise; - do, const C extends WorkflowStepConfig>( + // The config overloads discriminate on the shape of `config.retries.delay` + // so the callback context reflects whether the resolved delay is present + // (static delay) or omitted (dynamic delay function). Each has a single + // type parameter, so an explicit return-type argument (`do(...)`) still + // resolves here. ORDERING IS LOAD-BEARING: the broad `WorkflowStepConfig` + // fallback MUST remain last, otherwise it shadows the discriminating + // overloads and narrowing is silently lost. + do>( + name: string, + config: WorkflowStepConfigWithDelayFunction, + callback: (ctx: WorkflowStepContext) => Promise, + rollbackOptions?: WorkflowStepRollbackOptions, + ): Promise; + do>( name: string, - config: C, + config: WorkflowStepConfigWithStaticDelay, callback: ( - ctx: WorkflowStepContext< - C["retries"] extends { - delay: infer D; - } - ? D - : WorkflowDelayDuration | number - >, + ctx: WorkflowStepContext, ) => Promise, rollbackOptions?: WorkflowStepRollbackOptions, ): Promise; + do>( + name: string, + config: WorkflowStepConfig, + callback: (ctx: WorkflowStepContext) => Promise, + rollbackOptions?: WorkflowStepRollbackOptions, + ): Promise; sleep: (name: string, duration: WorkflowSleepDuration) => Promise; sleepUntil: (name: string, timestamp: Date | number) => Promise; waitForEvent>( diff --git a/types/generated-snapshot/index.d.ts b/types/generated-snapshot/index.d.ts index 80bb26d6682..4698edbdd60 100755 --- a/types/generated-snapshot/index.d.ts +++ b/types/generated-snapshot/index.d.ts @@ -14772,6 +14772,32 @@ declare namespace CloudflareWorkersModule { timeout?: WorkflowTimeoutDuration | number; sensitive?: WorkflowStepSensitivity; }; + // Internal discriminators used only for `WorkflowStep.do` overload + // resolution. They mirror `WorkflowStepConfig` but pin `retries.delay` to a + // single kind so the callback context can be narrowed based on the shape of + // the config argument (rather than on an inferred type parameter, which is + // lost when the caller supplies an explicit return-type argument). Not + // exported: they must not widen the public type surface. + type WorkflowStepConfigWithStaticDelay = Omit< + WorkflowStepConfig, + "retries" + > & { + retries?: { + limit: number; + delay: WorkflowDelayDuration | number; + backoff?: WorkflowBackoff; + }; + }; + type WorkflowStepConfigWithDelayFunction = Omit< + WorkflowStepConfig, + "retries" + > & { + retries: { + limit: number; + delay: WorkflowDelayFunction; + backoff?: WorkflowBackoff; + }; + }; export type WorkflowStepRollbackConfig = Pick< WorkflowStepConfig, "retries" | "timeout" @@ -14834,20 +14860,33 @@ declare namespace CloudflareWorkersModule { callback: (ctx: WorkflowStepContext) => Promise, rollbackOptions?: WorkflowStepRollbackOptions, ): Promise; - do, const C extends WorkflowStepConfig>( + // The config overloads discriminate on the shape of `config.retries.delay` + // so the callback context reflects whether the resolved delay is present + // (static delay) or omitted (dynamic delay function). Each has a single + // type parameter, so an explicit return-type argument (`do(...)`) still + // resolves here. ORDERING IS LOAD-BEARING: the broad `WorkflowStepConfig` + // fallback MUST remain last, otherwise it shadows the discriminating + // overloads and narrowing is silently lost. + do>( + name: string, + config: WorkflowStepConfigWithDelayFunction, + callback: (ctx: WorkflowStepContext) => Promise, + rollbackOptions?: WorkflowStepRollbackOptions, + ): Promise; + do>( name: string, - config: C, + config: WorkflowStepConfigWithStaticDelay, callback: ( - ctx: WorkflowStepContext< - C["retries"] extends { - delay: infer D; - } - ? D - : WorkflowDelayDuration | number - >, + ctx: WorkflowStepContext, ) => Promise, rollbackOptions?: WorkflowStepRollbackOptions, ): Promise; + do>( + name: string, + config: WorkflowStepConfig, + callback: (ctx: WorkflowStepContext) => Promise, + rollbackOptions?: WorkflowStepRollbackOptions, + ): Promise; sleep: (name: string, duration: WorkflowSleepDuration) => Promise; sleepUntil: (name: string, timestamp: Date | number) => Promise; waitForEvent>( diff --git a/types/generated-snapshot/index.ts b/types/generated-snapshot/index.ts index d665dec061e..8e9d132614d 100755 --- a/types/generated-snapshot/index.ts +++ b/types/generated-snapshot/index.ts @@ -14750,6 +14750,32 @@ export declare namespace CloudflareWorkersModule { timeout?: WorkflowTimeoutDuration | number; sensitive?: WorkflowStepSensitivity; }; + // Internal discriminators used only for `WorkflowStep.do` overload + // resolution. They mirror `WorkflowStepConfig` but pin `retries.delay` to a + // single kind so the callback context can be narrowed based on the shape of + // the config argument (rather than on an inferred type parameter, which is + // lost when the caller supplies an explicit return-type argument). Not + // exported: they must not widen the public type surface. + type WorkflowStepConfigWithStaticDelay = Omit< + WorkflowStepConfig, + "retries" + > & { + retries?: { + limit: number; + delay: WorkflowDelayDuration | number; + backoff?: WorkflowBackoff; + }; + }; + type WorkflowStepConfigWithDelayFunction = Omit< + WorkflowStepConfig, + "retries" + > & { + retries: { + limit: number; + delay: WorkflowDelayFunction; + backoff?: WorkflowBackoff; + }; + }; export type WorkflowStepRollbackConfig = Pick< WorkflowStepConfig, "retries" | "timeout" @@ -14812,20 +14838,33 @@ export declare namespace CloudflareWorkersModule { callback: (ctx: WorkflowStepContext) => Promise, rollbackOptions?: WorkflowStepRollbackOptions, ): Promise; - do, const C extends WorkflowStepConfig>( + // The config overloads discriminate on the shape of `config.retries.delay` + // so the callback context reflects whether the resolved delay is present + // (static delay) or omitted (dynamic delay function). Each has a single + // type parameter, so an explicit return-type argument (`do(...)`) still + // resolves here. ORDERING IS LOAD-BEARING: the broad `WorkflowStepConfig` + // fallback MUST remain last, otherwise it shadows the discriminating + // overloads and narrowing is silently lost. + do>( + name: string, + config: WorkflowStepConfigWithDelayFunction, + callback: (ctx: WorkflowStepContext) => Promise, + rollbackOptions?: WorkflowStepRollbackOptions, + ): Promise; + do>( name: string, - config: C, + config: WorkflowStepConfigWithStaticDelay, callback: ( - ctx: WorkflowStepContext< - C["retries"] extends { - delay: infer D; - } - ? D - : WorkflowDelayDuration | number - >, + ctx: WorkflowStepContext, ) => Promise, rollbackOptions?: WorkflowStepRollbackOptions, ): Promise; + do>( + name: string, + config: WorkflowStepConfig, + callback: (ctx: WorkflowStepContext) => Promise, + rollbackOptions?: WorkflowStepRollbackOptions, + ): Promise; sleep: (name: string, duration: WorkflowSleepDuration) => Promise; sleepUntil: (name: string, timestamp: Date | number) => Promise; waitForEvent>( diff --git a/types/test/types/rpc.ts b/types/test/types/rpc.ts index 761a579b782..e911fedd9bb 100644 --- a/types/test/types/rpc.ts +++ b/types/test/types/rpc.ts @@ -15,7 +15,9 @@ import { type WorkflowDynamicDelayContext, type WorkflowEvent, type WorkflowStep, + type WorkflowStepConfig, type WorkflowStepContext, + type WorkflowStepSensitivity, } from 'cloudflare:workers'; import { expectTypeOf } from 'expect-type'; @@ -941,6 +943,266 @@ const dynamicDelay: WorkflowDelayFunction = ({ctx, error}) => { }; void dynamicDelay; +// --------------------------------------------------------------------------- +// Regression coverage (WOR-1364) and `step.do` overload matrix. +// +// The config overloads discriminate on the shape of `config.retries.delay` +// (static duration vs. delay function) via distinct single-type-parameter +// overloads, plus a broad `WorkflowStepConfig` fallback. This fixes the +// regression where a single explicit type argument (the return type) combined +// with a config object failed to resolve to a config overload, and it also +// preserves callback-context narrowing even when an explicit type argument is +// supplied. +// --------------------------------------------------------------------------- + +// G1: return type propagates across every arg form (inferred + explicit). +expectTypeOf( + workflowStep.do('g1: 2-arg explicit', async () => 'ok') +).toEqualTypeOf>(); +// The `T extends Rpc.Serializable` constraint preserves the inferred literal +// return type rather than widening it to `number`. +expectTypeOf( + workflowStep.do('g1: 2-arg inferred literal', async () => 42) +).toEqualTypeOf>(); +expectTypeOf( + workflowStep.do( + 'g1: 3-arg static explicit', + {retries: {limit: 1, delay: 0}}, + async () => 'ok' + ) +).toEqualTypeOf>(); +expectTypeOf( + workflowStep.do( + 'g1: 3-arg dynamic inferred', + {retries: {limit: 1, delay: (): WorkflowDelayDuration => 5}}, + async (): Promise => 'ok' + ) +).toEqualTypeOf>(); +expectTypeOf( + workflowStep.do( + 'g1: 3-arg dynamic explicit', + {retries: {limit: 1, delay: (): WorkflowDelayDuration => 5}}, + async () => 'ok' + ) +).toEqualTypeOf>(); +expectTypeOf( + workflowStep.do( + 'g1: 4-arg config + rollback explicit', + {retries: {limit: 1, delay: 0}}, + async () => 'ok', + {rollback: async () => {}} + ) +).toEqualTypeOf>(); +expectTypeOf( + workflowStep.do('g1: object return', async () => ({foo: 'x'})) +).toEqualTypeOf>(); +expectTypeOf( + workflowStep.do('g1: union return', async (): Promise => 1) +).toEqualTypeOf>(); +expectTypeOf( + workflowStep.do('g1: void return', async (): Promise => {}) +).toEqualTypeOf>(); +expectTypeOf( + workflowStep.do('g1: array return', async (): Promise => []) +).toEqualTypeOf>(); + +// G2: a static delay is surfaced (present) in the callback context, and this +// holds even when the caller supplies an explicit type argument (the case that +// previously regressed to a spurious type error). +workflowStep.do( + 'g2: explicit static number delay present', + {retries: {limit: 1, delay: 0}}, + async (ctx) => { + expectTypeOf(ctx.config.retries?.delay).toEqualTypeOf< + WorkflowDelayDuration | number | undefined + >(); + return 'ok'; + } +); +workflowStep.do( + 'g2: explicit static duration delay present', + {retries: {limit: 1, delay: '2 hours'}}, + async (ctx) => { + expectTypeOf(ctx.config.retries?.delay).toEqualTypeOf< + WorkflowDelayDuration | number | undefined + >(); + return 'ok'; + } +); +workflowStep.do( + 'g2: inferred static delay with backoff', + {retries: {limit: 1, delay: 0, backoff: 'exponential'}}, + async (ctx): Promise => { + expectTypeOf(ctx.config.retries?.backoff).toEqualTypeOf< + WorkflowBackoff | undefined + >(); + expectTypeOf(ctx.config.retries?.delay).toEqualTypeOf< + WorkflowDelayDuration | number | undefined + >(); + return 'ok'; + } +); + +// G3: a delay function omits the resolved delay from the callback context, +// including under an explicit type argument and for async delay functions. +workflowStep.do( + 'g3: explicit dynamic delay absent', + {retries: {limit: 1, delay: (): WorkflowDelayDuration => '1 minute'}}, + async (ctx) => { + expectTypeOf(ctx.config.retries).toEqualTypeOf< + {limit: number; backoff?: WorkflowBackoff} | undefined + >(); + // @ts-expect-error delay is absent under an explicit type argument too + ctx.config.retries?.delay; + return 'ok'; + } +); +workflowStep.do( + 'g3: explicit async dynamic delay absent', + {retries: {limit: 1, delay: async (): Promise => 5}}, + async (ctx) => { + expectTypeOf(ctx.config.retries).toEqualTypeOf< + {limit: number; backoff?: WorkflowBackoff} | undefined + >(); + return 'ok'; + } +); +workflowStep.do( + 'g3: inferred async dynamic delay absent', + {retries: {limit: 1, delay: async (): Promise => 5}}, + async (ctx): Promise => { + expectTypeOf(ctx.config.retries).toEqualTypeOf< + {limit: number; backoff?: WorkflowBackoff} | undefined + >(); + // @ts-expect-error delay is absent for an async delay function + ctx.config.retries?.delay; + return 'ok'; + } +); +workflowStep.do( + 'g3: dynamic delay with backoff, delay absent', + {retries: {limit: 1, delay: (): WorkflowDelayDuration => 5, backoff: 'linear'}}, + async (ctx): Promise => { + expectTypeOf(ctx.config.retries?.backoff).toEqualTypeOf< + WorkflowBackoff | undefined + >(); + // @ts-expect-error delay is absent even when backoff is present + ctx.config.retries?.delay; + return 'ok'; + } +); + +// G4: a config typed broadly (e.g. built separately) resolves to the fallback +// overload; it still compiles and exposes the non-narrowed (static) context. +declare const broadConfig: WorkflowStepConfig; +expectTypeOf( + workflowStep.do('g4: prebuilt broad config', broadConfig, async (ctx) => { + expectTypeOf(ctx.config.retries?.delay).toEqualTypeOf< + WorkflowDelayDuration | number | undefined + >(); + return 'ok'; + }) +).toMatchTypeOf>(); +expectTypeOf( + workflowStep.do('g4: prebuilt broad config explicit', broadConfig, async () => 'ok') +).toEqualTypeOf>(); +declare const unionDelay: + | WorkflowDelayDuration + | number + | WorkflowDelayFunction; +expectTypeOf( + workflowStep.do( + 'g4: union-typed delay falls back', + {retries: {limit: 1, delay: unionDelay}}, + async (): Promise => 'ok' + ) +).toMatchTypeOf>(); + +// G5: minimal / retries-less configs. +workflowStep.do( + 'g5: timeout only', + {timeout: '10 seconds'}, + async (ctx): Promise => { + expectTypeOf(ctx.config.timeout).toMatchTypeOf(); + return 'ok'; + } +); +workflowStep.do('g5: numeric timeout', {timeout: 5000}, async (): Promise => 'ok'); +workflowStep.do('g5: empty config', {}, async (): Promise => 'ok'); +workflowStep.do( + 'g5: sensitive only', + {sensitive: 'output'}, + async (ctx): Promise => { + expectTypeOf(ctx.config.sensitive).toEqualTypeOf< + WorkflowStepSensitivity | undefined + >(); + return 'ok'; + } +); +workflowStep.do( + 'g5: full config', + {retries: {limit: 1, delay: 0}, timeout: '1 minute', sensitive: 'output'}, + async (ctx): Promise => { + expectTypeOf(ctx.config.retries?.delay).toEqualTypeOf< + WorkflowDelayDuration | number | undefined + >(); + return 'ok'; + } +); + +// G6: rollback combinations. +workflowStep.do( + 'g6: config + rollbackConfig delay function', + {retries: {limit: 1, delay: 0}}, + async (): Promise => 'ok', + {rollback: async () => {}, rollbackConfig: {retries: {limit: 0, delay: () => 0}}} +); +workflowStep.do('g6: rollback output non-string', async (): Promise<{foo: string}> => ({foo: 'x'}), { + rollback: async (rollbackCtx) => { + expectTypeOf(rollbackCtx.output).toEqualTypeOf<{foo: string} | undefined>(); + }, +}); +workflowStep.do( + 'g6: explicit + config + rollback options', + {retries: {limit: 1, delay: 0}}, + async () => 'ok', + {rollback: async () => {}, rollbackConfig: {retries: {limit: 0, delay: 0}}} +); + +// G7: negatives. +// @ts-expect-error delay must not be a boolean +workflowStep.do('g7: delay boolean', {retries: {limit: 1, delay: true}}, async () => 'ok'); +// @ts-expect-error delay must not be an object +workflowStep.do('g7: delay object', {retries: {limit: 1, delay: {}}}, async () => 'ok'); +// @ts-expect-error retries requires a limit +workflowStep.do('g7: retries no limit', {retries: {delay: 0}}, async () => 'ok'); +// @ts-expect-error retries requires a delay +workflowStep.do('g7: retries no delay', {retries: {limit: 1}}, async () => 'ok'); +// @ts-expect-error timeout must not be a boolean +workflowStep.do('g7: timeout boolean', {timeout: true}, async () => 'ok'); +// @ts-expect-error sensitive must be the 'output' literal +workflowStep.do('g7: sensitive wrong', {sensitive: 'nope'}, async () => 'ok'); +// @ts-expect-error a sync delay function must return a delay duration +workflowStep.do('g7: delay fn wrong return', {retries: {limit: 1, delay: (): boolean => true}}, async () => 'ok'); +// @ts-expect-error an async delay function must resolve to a delay duration +workflowStep.do('g7: async delay fn wrong return', {retries: {limit: 1, delay: async (): Promise => true}}, async () => 'ok'); +// @ts-expect-error each overload declares a single type parameter +workflowStep.do('g7: too many type args', {retries: {limit: 1, delay: 0}}, async () => 'ok'); +// @ts-expect-error the callback must return a Promise +workflowStep.do('g7: callback not async', {retries: {limit: 1, delay: 0}}, () => 'ok'); +// @ts-expect-error the step name must be a string +workflowStep.do(123, async (): Promise => 'ok'); +// @ts-expect-error unknown config keys are rejected +workflowStep.do('g7: excess config key', {retries: {limit: 1, delay: 0}, bogus: 1}, async () => 'ok'); + +// G8: standalone delay functions may return any accepted delay duration shape. +const delayReturningNumber: WorkflowDelayFunction = () => 5; +void delayReturningNumber; +const delayReturningDuration: WorkflowDelayFunction = () => '1 minute'; +void delayReturningDuration; +const asyncDelay: WorkflowDelayFunction = async () => 5; +void asyncDelay; + declare const cronSchedule: WorkflowCronSchedule; expectTypeOf(cronSchedule.cron).toEqualTypeOf(); expectTypeOf(cronSchedule.scheduledTime).toEqualTypeOf(); From daeb2f16da8584f0eb1d4685b40bdbc719462458 Mon Sep 17 00:00:00 2001 From: James M Snell Date: Tue, 7 Jul 2026 13:14:28 -0700 Subject: [PATCH 063/101] Introduce JsWritableStream abstraction --- src/workerd/api/BUILD.bazel | 17 +- src/workerd/api/api-rtti-test.c++ | 10 + src/workerd/api/js-writable-stream-test.c++ | 310 ++++++++++++++++++++ src/workerd/api/js-writable-stream.c++ | 203 +++++++++++++ src/workerd/api/js-writable-stream.h | 180 ++++++++++++ src/workerd/api/streams.h | 1 + 6 files changed, 718 insertions(+), 3 deletions(-) create mode 100644 src/workerd/api/js-writable-stream-test.c++ create mode 100644 src/workerd/api/js-writable-stream.c++ create mode 100644 src/workerd/api/js-writable-stream.h diff --git a/src/workerd/api/BUILD.bazel b/src/workerd/api/BUILD.bazel index 84a87af13e5..b7a60868851 100644 --- a/src/workerd/api/BUILD.bazel +++ b/src/workerd/api/BUILD.bazel @@ -26,6 +26,7 @@ filegroup( "hibernatable-web-socket.c++", "http.c++", "js-readable-stream.c++", + "js-writable-stream.c++", "messagechannel.c++", "performance.c++", "queue.c++", @@ -77,6 +78,7 @@ filegroup( "hibernatable-web-socket.h", "http.h", "js-readable-stream.h", + "js-writable-stream.h", "messagechannel.h", "performance.h", "queue.h", @@ -238,9 +240,10 @@ wd_cc_library( # for using EW_STREAMS_ISOLATE_TYPES with workerd-api etc. To use the different streams APIs, # include the headers you need directly. # -# Note: JsReadableStream lives in js-readable-stream.{h,c++}, which is folded into -# //src/workerd/io (via the :srcs/:hdrs filegroups) because http.h depends on it. This target -# re-exports it via streams.h. +# Note: JsReadableStream and JsWritableStream live in js-readable-stream.{h,c++} and +# js-writable-stream.{h,c++}, which are folded into //src/workerd/io (via the :srcs/:hdrs +# filegroups) because io-target headers (http.h, sockets.h, container.h) depend on them. This +# target re-exports them via streams.h. wd_cc_library( name = "streams", hdrs = ["streams.h"], @@ -617,6 +620,14 @@ kj_test( ], ) +kj_test( + src = "js-writable-stream-test.c++", + deps = [ + "//src/workerd/io", + "//src/workerd/tests:test-fixture", + ], +) + kj_test( src = "system-streams-test.c++", deps = [ diff --git a/src/workerd/api/api-rtti-test.c++ b/src/workerd/api/api-rtti-test.c++ index 47e49ccc635..3b9c8c18a32 100644 --- a/src/workerd/api/api-rtti-test.c++ +++ b/src/workerd/api/api-rtti-test.c++ @@ -55,5 +55,15 @@ KJ_TEST("JsReadableStream delegates its RTTI to ReadableStream") { KJ_EXPECT(builder.structure("workerd::api::ReadableStream"_kj) != kj::none); } +KJ_TEST("JsWritableStream delegates its RTTI to WritableStream") { + // JsWritableStream declares `using JsgRttiDelegate = jsg::Ref`, so RTTI (and + // therefore generated TypeScript) must describe it exactly as it describes WritableStream. + jsg::rtti::Builder builder((CompatibilityFlags::Reader())); + auto type = builder.type(); + KJ_ASSERT(type.isStructure()); + KJ_EXPECT(type.getStructure().getFullyQualifiedName() == "workerd::api::WritableStream"_kj); + KJ_EXPECT(builder.structure("workerd::api::WritableStream"_kj) != kj::none); +} + } // namespace } // namespace workerd::api diff --git a/src/workerd/api/js-writable-stream-test.c++ b/src/workerd/api/js-writable-stream-test.c++ new file mode 100644 index 00000000000..dc8dc5ccb18 --- /dev/null +++ b/src/workerd/api/js-writable-stream-test.c++ @@ -0,0 +1,310 @@ +// Copyright (c) 2017-2022 Cloudflare, Inc. +// Licensed under the Apache 2.0 license found in the LICENSE file or at: +// https://opensource.org/licenses/Apache-2.0 + +#include +#include + +#include + +namespace workerd::api { +namespace { + +// The user-visible message produced when performing a lock-checked operation on a stream that is +// locked to a writer. This must match the message produced by WritableStream exactly. +constexpr kj::StringPtr kWriterLockedError = + "This WritableStream is currently locked to a writer."_kj; + +// A WritableStreamSink recording its interactions into externally-owned state. +class RecordingSink final: public WritableStreamSink { + public: + RecordingSink(kj::Vector& data, bool& ended, bool& aborted) + : data(data), + ended(ended), + aborted(aborted) {} + + kj::Promise write(kj::ArrayPtr buffer) override { + data.addAll(buffer); + return kj::READY_NOW; + } + + kj::Promise write(kj::ArrayPtr> pieces) override { + for (auto& piece: pieces) { + data.addAll(piece); + } + return kj::READY_NOW; + } + + kj::Promise end() override { + ended = true; + return kj::READY_NOW; + } + + void abort(kj::Exception reason) override { + aborted = true; + } + + private: + kj::Vector& data; + bool& ended; + bool& aborted; +}; + +// Test state bundling the externally-owned sink observations. Declared outside runInIoContext +// (which runs synchronously to completion) so the sink's references remain valid for the whole +// test. +struct SinkState { + kj::Vector written; + bool ended = false; + bool aborted = false; + + kj::Own makeSink() { + return kj::heap(written, ended, aborted); + } +}; + +KJ_TEST("JsWritableStream null state") { + TestFixture testFixture; + testFixture.runInIoContext([](const TestFixture::Environment& env) -> kj::Promise { + auto& js = env.js; + + JsWritableStream stream; + KJ_EXPECT(stream.isNull()); + KJ_EXPECT(!stream.isLocked(js)); + KJ_EXPECT(!stream.isClosedOrClosing(js)); + KJ_EXPECT(stream.addRef(js).isNull()); + + // setPendingClosure() and forceAbort() are teardown-path operations that tolerate null. + stream.setPendingClosure(js); + return env.context.awaitJs(js, stream.forceAbort(js, kj::none)); + }); +} + +KJ_TEST("JsWritableStream create wraps a native sink; forceClose ends it") { + TestFixture testFixture; + SinkState state; + testFixture.runInIoContext([&](const TestFixture::Environment& env) -> kj::Promise { + auto& js = env.js; + + auto stream = JsWritableStream::create(js, env.context, state.makeSink(), kj::none); + KJ_EXPECT(!stream.isNull()); + KJ_EXPECT(!stream.isLocked(js)); + KJ_EXPECT(!stream.isClosedOrClosing(js)); + + auto promise = stream.forceClose(js).then(js, + JSG_VISITABLE_LAMBDA((stream = kj::mv(stream)), (stream), + (jsg::Lock& js) { KJ_EXPECT(stream.isClosedOrClosing(js)); })); + return env.context.awaitJs(js, kj::mv(promise)); + }); + KJ_EXPECT(state.ended); + KJ_EXPECT(!state.aborted); +} + +KJ_TEST("JsWritableStream forceAbort aborts the sink") { + TestFixture testFixture; + SinkState state; + testFixture.runInIoContext([&](const TestFixture::Environment& env) -> kj::Promise { + auto& js = env.js; + + auto stream = JsWritableStream::create(js, env.context, state.makeSink(), kj::none); + return env.context.awaitJs(js, stream.forceAbort(js, js.error("connection lost"))); + }); + KJ_EXPECT(state.aborted); + KJ_EXPECT(!state.ended); +} + +KJ_TEST("JsWritableStream flush resolves on an idle stream") { + TestFixture testFixture; + SinkState state; + testFixture.runInIoContext([&](const TestFixture::Environment& env) -> kj::Promise { + auto& js = env.js; + + auto stream = JsWritableStream::create(js, env.context, state.makeSink(), kj::none); + return env.context.awaitJs(js, stream.flush(js)); + }); + // flush() completes pending writes but does not end the stream. + KJ_EXPECT(!state.ended); +} + +KJ_TEST("JsWritableStream flush rejects when a writer is held; forceFlush succeeds") { + TestFixture testFixture; + SinkState state; + testFixture.runInIoContext([&](const TestFixture::Environment& env) -> kj::Promise { + auto& js = env.js; + + // Pre-lock the stream by attaching a writer to the underlying WritableStream before adopting + // it into the abstraction. + auto ws = js.alloc(env.context, state.makeSink(), kj::none); + auto writer = ws->getWriter(js); + JsWritableStream stream(kj::mv(ws)); + KJ_EXPECT(stream.isLocked(js)); + + auto promise = stream.flush(js) + .then(js, [](jsg::Lock& js) { + KJ_FAIL_REQUIRE("expected flush() of a writer-locked stream to reject"); + }, [](jsg::Lock& js, jsg::Value exception) { + auto e = js.exceptionToKj(kj::mv(exception)); + KJ_EXPECT( + e.getDescription() == kj::str("jsg.TypeError: ", kWriterLockedError), e.getDescription()); + }).then(js, JSG_VISITABLE_LAMBDA((stream = kj::mv(stream)), (stream), (jsg::Lock& js) { + // forceFlush() bypasses the writer lock. + return stream.forceFlush(js); + })); + return env.context.awaitJs(js, kj::mv(promise)).attach(kj::mv(writer)); + }); +} + +KJ_TEST("JsWritableStream forceAbort succeeds despite a held writer") { + TestFixture testFixture; + SinkState state; + testFixture.runInIoContext([&](const TestFixture::Environment& env) -> kj::Promise { + auto& js = env.js; + + auto ws = js.alloc(env.context, state.makeSink(), kj::none); + auto writer = ws->getWriter(js); + JsWritableStream stream(kj::mv(ws)); + KJ_EXPECT(stream.isLocked(js)); + + return env.context.awaitJs(js, stream.forceAbort(js, kj::none)).attach(kj::mv(writer)); + }); + KJ_EXPECT(state.aborted); +} + +KJ_TEST("JsWritableStream forceClose succeeds despite a held writer") { + TestFixture testFixture; + SinkState state; + testFixture.runInIoContext([&](const TestFixture::Environment& env) -> kj::Promise { + auto& js = env.js; + + auto ws = js.alloc(env.context, state.makeSink(), kj::none); + auto writer = ws->getWriter(js); + JsWritableStream stream(kj::mv(ws)); + KJ_EXPECT(stream.isLocked(js)); + + return env.context.awaitJs(js, stream.forceClose(js)).attach(kj::mv(writer)); + }); + KJ_EXPECT(state.ended); +} + +KJ_TEST("JsWritableStream detach neutralizes the stream without ending the sink") { + TestFixture testFixture; + SinkState state; + testFixture.runInIoContext([&](const TestFixture::Environment& env) { + auto& js = env.js; + + auto stream = JsWritableStream::create(js, env.context, state.makeSink(), kj::none); + stream.detach(js); + + // The stream is left permanently locked and closed to further writes... + KJ_EXPECT(stream.isLocked(js)); + KJ_EXPECT(stream.isClosedOrClosing(js)); + }); + // ...but detach is a takeover, not a close: the sink is dropped without end(). + KJ_EXPECT(!state.ended); + KJ_EXPECT(!state.aborted); +} + +KJ_TEST("JsWritableStream detach throws when a writer is held") { + TestFixture testFixture; + SinkState state; + testFixture.runInIoContext([&](const TestFixture::Environment& env) { + auto& js = env.js; + + auto ws = js.alloc(env.context, state.makeSink(), kj::none); + auto writer = ws->getWriter(js); + JsWritableStream stream(kj::mv(ws)); + + js.tryCatch([&] { + stream.detach(js); + KJ_FAIL_REQUIRE("expected detach() of a writer-locked stream to throw"); + }, [&](jsg::Value exception) { + auto e = js.exceptionToKj(kj::mv(exception)); + KJ_EXPECT(e.getDescription().contains(kWriterLockedError), e.getDescription()); + }); + }); +} + +KJ_TEST("JsWritableStream detach of a closed stream throws") { + TestFixture testFixture; + SinkState state; + testFixture.runInIoContext([&](const TestFixture::Environment& env) -> kj::Promise { + auto& js = env.js; + + auto stream = JsWritableStream::create(js, env.context, state.makeSink(), kj::none); + auto promise = + stream.forceClose(js).then(js, + JSG_VISITABLE_LAMBDA((stream = kj::mv(stream)), (stream), (jsg::Lock& js) { + KJ_EXPECT(stream.isClosedOrClosing(js)); + js.tryCatch([&] { + stream.detach(js); + KJ_FAIL_REQUIRE("expected detach() of a closed stream to throw"); + }, [&](jsg::Value exception) { + auto e = js.exceptionToKj(kj::mv(exception)); + KJ_EXPECT(e.getDescription().contains("This WritableStream is closed."), + e.getDescription()); + }); + })); + return env.context.awaitJs(js, kj::mv(promise)); + }); +} + +KJ_TEST("JsWritableStream addRef shares the underlying stream") { + TestFixture testFixture; + SinkState state; + testFixture.runInIoContext([&](const TestFixture::Environment& env) -> kj::Promise { + auto& js = env.js; + + auto stream = JsWritableStream::create(js, env.context, state.makeSink(), kj::none); + auto ref = stream.addRef(js); + KJ_EXPECT(!ref.isNull()); + KJ_EXPECT(!ref.isClosedOrClosing(js)); + + // Closing through the addRef closes the original: both wrap the same stream. + auto promise = ref.forceClose(js).then(js, + JSG_VISITABLE_LAMBDA((stream = kj::mv(stream)), (stream), + (jsg::Lock& js) { KJ_EXPECT(stream.isClosedOrClosing(js)); })); + return env.context.awaitJs(js, kj::mv(promise)); + }); + KJ_EXPECT(state.ended); +} + +KJ_TEST("JsWritableStream setPendingClosure is safe on live and null streams") { + TestFixture testFixture; + SinkState state; + testFixture.runInIoContext([&](const TestFixture::Environment& env) { + auto& js = env.js; + + JsWritableStream().setPendingClosure(js); // null: no-op + + auto stream = JsWritableStream::create(js, env.context, state.makeSink(), kj::none); + stream.setPendingClosure(js); + KJ_EXPECT(!stream.isNull()); + }); +} + +KJ_TEST("JsWritableStream create honors the closure waitable") { + // Note: create()'s maybeHighWaterMark is likewise a pass-through to the internal controller; + // its backpressure behavior is not observable through the abstraction's narrow API and is + // pinned by streams/internal-test.c++ and the socket tests. + TestFixture testFixture; + SinkState state; + testFixture.runInIoContext([&](const TestFixture::Environment& env) -> kj::Promise { + auto& js = env.js; + + auto prp = js.newPromiseAndResolver(); + auto stream = JsWritableStream::create( + js, env.context, state.makeSink(), kj::none, kj::none, kj::mv(prp.promise)); + + auto closePromise = stream.forceClose(js); + // The closure waitable has not resolved, so the close must not have completed (the internal + // controller's closeImpl is gated on the waitable and cannot have run yet). + KJ_EXPECT(!state.ended); + + prp.resolver.resolve(js); + return env.context.awaitJs(js, kj::mv(closePromise)); + }); + KJ_EXPECT(state.ended); +} + +} // namespace +} // namespace workerd::api diff --git a/src/workerd/api/js-writable-stream.c++ b/src/workerd/api/js-writable-stream.c++ new file mode 100644 index 00000000000..7ebace79199 --- /dev/null +++ b/src/workerd/api/js-writable-stream.c++ @@ -0,0 +1,203 @@ +// Copyright (c) 2017-2022 Cloudflare, Inc. +// Licensed under the Apache 2.0 license found in the LICENSE file or at: +// https://opensource.org/licenses/Apache-2.0 + +#include + +#include + +namespace workerd::api { + +JsWritableStream::JsWritableStream(jsg::Ref stream) + : impl(Impl{ + .stream = StreamImpl(kj::mv(stream)), + }) {} + +JsWritableStream::JsWritableStream(jsg::Lock&, jsg::JsRef) { + KJ_UNIMPLEMENTED("TypeScript-backed WritableStream is not yet supported"); +} + +JsWritableStream JsWritableStream::create(jsg::Lock& js, + IoContext& ioContext, + kj::Own sink, + kj::Maybe> observer, + kj::Maybe maybeHighWaterMark, + kj::Maybe> maybeClosureWaitable) { + // TODO(streams-ts): Dispatch on the worker's configuration to construct either the legacy + // C++ WritableStream or a TypeScript-backed stream. + return JsWritableStream(js.alloc( + ioContext, kj::mv(sink), kj::mv(observer), maybeHighWaterMark, kj::mv(maybeClosureWaitable))); +} + +JsWritableStream JsWritableStream::addRef(jsg::Lock& js) { + KJ_IF_SOME(i, impl) { + KJ_SWITCH_ONEOF(i.stream) { + KJ_CASE_ONEOF(stream, jsg::Ref) { + return JsWritableStream(Impl{ + .stream = StreamImpl(stream.addRef()), + }); + } + KJ_CASE_ONEOF(obj, jsg::JsRef) { + KJ_UNIMPLEMENTED("TypeScript-backed WritableStream is not yet supported"); + } + } + KJ_UNREACHABLE; + } + // addRef() of a null stream is a null stream. + return JsWritableStream(); +} + +bool JsWritableStream::isNull() const { + return impl == kj::none; +} + +bool JsWritableStream::isLocked(jsg::Lock& js) { + KJ_IF_SOME(i, impl) { + KJ_SWITCH_ONEOF(i.stream) { + KJ_CASE_ONEOF(stream, jsg::Ref) { + return stream->isLocked(); + } + KJ_CASE_ONEOF(obj, jsg::JsRef) { + KJ_UNIMPLEMENTED("TypeScript-backed WritableStream is not yet supported"); + } + } + KJ_UNREACHABLE; + } + return false; +} + +bool JsWritableStream::isClosedOrClosing(jsg::Lock& js) { + KJ_IF_SOME(i, impl) { + KJ_SWITCH_ONEOF(i.stream) { + KJ_CASE_ONEOF(stream, jsg::Ref) { + return stream->getController().isClosedOrClosing(); + } + KJ_CASE_ONEOF(obj, jsg::JsRef) { + KJ_UNIMPLEMENTED("TypeScript-backed WritableStream is not yet supported"); + } + } + KJ_UNREACHABLE; + } + return false; +} + +jsg::Promise JsWritableStream::flush(jsg::Lock& js) { + auto& i = KJ_ASSERT_NONNULL(impl, "flush() called on a null JsWritableStream"); + KJ_SWITCH_ONEOF(i.stream) { + KJ_CASE_ONEOF(stream, jsg::Ref) { + // WritableStream::flush() itself performs the writer-lock check, rejecting with the + // exact user-visible TypeError text when locked. + return stream->flush(js); + } + KJ_CASE_ONEOF(obj, jsg::JsRef) { + // TODO(streams-ts): Compose from the isLocked query (rejecting with the exact + // "This WritableStream is currently locked to a writer." text) and the flush hook. + KJ_UNIMPLEMENTED("TypeScript-backed WritableStream is not yet supported"); + } + } + KJ_UNREACHABLE; +} + +jsg::Promise JsWritableStream::forceFlush(jsg::Lock& js) { + auto& i = KJ_ASSERT_NONNULL(impl, "forceFlush() called on a null JsWritableStream"); + KJ_SWITCH_ONEOF(i.stream) { + KJ_CASE_ONEOF(stream, jsg::Ref) { + // Going through the controller (rather than WritableStream::flush()) deliberately + // bypasses the "is locked to a writer" check. + return stream->getController().flush(js); + } + KJ_CASE_ONEOF(obj, jsg::JsRef) { + KJ_UNIMPLEMENTED("TypeScript-backed WritableStream is not yet supported"); + } + } + KJ_UNREACHABLE; +} + +jsg::Promise JsWritableStream::forceAbort(jsg::Lock& js, jsg::Optional reason) { + KJ_IF_SOME(i, impl) { + KJ_SWITCH_ONEOF(i.stream) { + KJ_CASE_ONEOF(stream, jsg::Ref) { + // Going through the controller (rather than WritableStream::abort()) deliberately + // bypasses the "is locked to a writer" check: this aborts the stream out from under any + // writer. + return stream->getController().abort(js, kj::mv(reason)); + } + KJ_CASE_ONEOF(obj, jsg::JsRef) { + KJ_UNIMPLEMENTED("TypeScript-backed WritableStream is not yet supported"); + } + } + KJ_UNREACHABLE; + } + // Force-aborting a null stream is a no-op. + return js.resolvedPromise(); +} + +jsg::Promise JsWritableStream::forceClose(jsg::Lock& js) { + auto& i = KJ_ASSERT_NONNULL(impl, "forceClose() called on a null JsWritableStream"); + KJ_SWITCH_ONEOF(i.stream) { + KJ_CASE_ONEOF(stream, jsg::Ref) { + // Going through the controller (rather than WritableStream::close()) deliberately + // bypasses the "is locked to a writer" check. + return stream->getController().close(js); + } + KJ_CASE_ONEOF(obj, jsg::JsRef) { + KJ_UNIMPLEMENTED("TypeScript-backed WritableStream is not yet supported"); + } + } + KJ_UNREACHABLE; +} + +void JsWritableStream::setPendingClosure(jsg::Lock& js) { + KJ_IF_SOME(i, impl) { + KJ_SWITCH_ONEOF(i.stream) { + KJ_CASE_ONEOF(stream, jsg::Ref) { + stream->getController().setPendingClosure(); + } + KJ_CASE_ONEOF(obj, jsg::JsRef) { + KJ_UNIMPLEMENTED("TypeScript-backed WritableStream is not yet supported"); + } + } + } +} + +void JsWritableStream::detach(jsg::Lock& js) { + auto& i = KJ_ASSERT_NONNULL(impl, "detach() called on a null JsWritableStream"); + KJ_SWITCH_ONEOF(i.stream) { + KJ_CASE_ONEOF(stream, jsg::Ref) { + stream->detach(js); + } + KJ_CASE_ONEOF(obj, jsg::JsRef) { + KJ_UNIMPLEMENTED("TypeScript-backed WritableStream is not yet supported"); + } + } +} + +void JsWritableStream::visitForGc(jsg::GcVisitor& visitor) { + KJ_IF_SOME(i, impl) { + KJ_SWITCH_ONEOF(i.stream) { + KJ_CASE_ONEOF(stream, jsg::Ref) { + visitor.visit(stream); + } + KJ_CASE_ONEOF(obj, jsg::JsRef) { + visitor.visit(obj); + } + } + } +} + +void JsWritableStream::visitForMemoryInfo(jsg::MemoryTracker& tracker) const { + KJ_IF_SOME(i, impl) { + KJ_SWITCH_ONEOF(i.stream) { + KJ_CASE_ONEOF(stream, jsg::Ref) { + tracker.trackField("stream", stream); + } + KJ_CASE_ONEOF(obj, jsg::JsRef) { + // TODO(streams-ts): track the JS object's memory once TS-backed streams are supported. + // (jsg::JsRef does not satisfy the MemoryRetainer concept, so it can't be passed to + // trackField() directly.) + } + } + } +} + +} // namespace workerd::api diff --git a/src/workerd/api/js-writable-stream.h b/src/workerd/api/js-writable-stream.h new file mode 100644 index 00000000000..7982a051c9e --- /dev/null +++ b/src/workerd/api/js-writable-stream.h @@ -0,0 +1,180 @@ +// Copyright (c) 2017-2022 Cloudflare, Inc. +// Licensed under the Apache 2.0 license found in the LICENSE file or at: +// https://opensource.org/licenses/Apache-2.0 + +#pragma once + +#include + +#include +#include + +namespace workerd::api { + +// An abstraction of a WritableStream, backed by either a C++ implemented WritableStream +// (defined in src/workerd/api/streams/*) or a TypeScript implemented WritableStream (defined +// in src/per_isolate/webstreams). The API is limited strictly to the methods that are needed +// by the C++ side of workerd. It is not intended to be a complete implementation of the +// WritableStream API. This is the writable-side counterpart of JsReadableStream (see +// js-readable-stream.h); the two follow the same conventions. +// +// A JsWritableStream is one of: +// * null / empty (isNull()) -- the default-constructed or moved-from state; or +// * stream-backed -- wraps a WritableStream. +// +// Unlike JsReadableStream there is no buffer-backed (rewindable) state: writables have no +// retransmission concept. There is also no "disturbed" concept for writables. +// +// Backend branching: the underlying stream is stored as a kj::OneOf so that a TypeScript +// implemented WritableStream (represented as a JS object) can eventually be supported alongside +// the legacy C++ WritableStream. Every method that touches the underlying stream switches on the +// backend. Today only the C++ WritableStream backend is implemented; the TypeScript backend +// branches are KJ_UNIMPLEMENTED and the corresponding constructor cannot yet be used. When the +// TypeScript implementation lands, fill in those branches -- the stored type (and therefore the +// consumers) will not need to change. +class JsWritableStream final { + public: + // The underlying stream. Today only the legacy C++ WritableStream alternative is populated; the + // jsg::JsRef alternative is reserved for the future TypeScript implementation. + using StreamImpl = kj::OneOf, jsg::JsRef>; + + struct Impl { + StreamImpl stream; + }; + + // Create a null / empty JsWritableStream. Unlike JsReadableStream's null state (which + // represents "no body" and is a real consumer-visible state), a null JsWritableStream is only + // ever a default-constructed or moved-from artifact. State queries are null-safe; most + // operations assert non-null (see the individual method comments). + JsWritableStream() = default; + + // Adopt an existing legacy C++ WritableStream. + JsWritableStream(jsg::Ref stream); + + // Adopt a TypeScript-implemented WritableStream (a JS object). Not yet supported. + JsWritableStream(jsg::Lock& js, jsg::JsRef obj); + + JsWritableStream(JsWritableStream&&) = default; + JsWritableStream& operator=(JsWritableStream&&) = default; + JsWritableStream(const JsWritableStream&) = delete; + JsWritableStream& operator=(const JsWritableStream&) = delete; + + // Create a JsWritableStream wrapping the given native data sink. This is the canonical way for + // C++ code to mint a new WritableStream to hand to JavaScript. + // + // The observer (used for byte stream metrics) is provided by the caller (typically + // ioContext.getMetrics().tryCreateWritableByteStreamObserver()). maybeHighWaterMark configures + // the internal write buffer's backpressure threshold. If maybeClosureWaitable is provided, + // closing the stream will not complete until the given promise resolves (used by sockets to + // gate closure on connection establishment). + // + // TODO(streams-ts): This is the future compatibility-flag dispatch point. Once the + // TypeScript implementation lands, this will construct either the legacy C++ WritableStream + // or a TypeScript-backed stream depending on the worker's configuration. + static JsWritableStream create(jsg::Lock& js, + IoContext& ioContext, + kj::Own sink, + kj::Maybe> observer, + kj::Maybe maybeHighWaterMark = kj::none, + kj::Maybe> maybeClosureWaitable = kj::none); + + // Returns a new JsWritableStream sharing this one's underlying stream. Both instances observe + // the same underlying stream state (e.g. the stream closing through one is visible through the + // other), and passing either through the type wrapper yields the same JavaScript object. This + // is what identity-preserving accessors (e.g. socket.writable === socket.writable) are built + // from. addRef() of a null stream is a null stream. + JsWritableStream addRef(jsg::Lock& js); + + // True if this is a null / empty stream. Inspects only C++-side state; a jsg::Lock& is not + // required because it never dispatches to the JS backend. + bool isNull() const; + + // True if the underlying stream is currently locked to a writer. A null stream is never + // locked. Not const: for the C++ backend this dispatches to WritableStream::isLocked(), which + // is non-const. + bool isLocked(jsg::Lock& js); + + // True if the underlying stream is closed or in the process of closing. A null stream answers + // false. Not const for the same reason as isLocked(). + bool isClosedOrClosing(jsg::Lock& js); + + // Waits for all pending writes to complete. Rejects if the stream is currently locked to a + // writer, matching the (workerd-internal, non-standard) WritableStream::flush() extension. + // Precondition: !isNull(). + jsg::Promise flush(jsg::Lock& js); + + // Like flush(), but bypasses the writer-lock check by going through the controller. Used when + // pending data must be flushed regardless of what JavaScript is doing with the stream (e.g. + // when a Socket is being closed). Precondition: !isNull(). + jsg::Promise forceFlush(jsg::Lock& js); + + // Immediately interrupts pending writes and errors the stream, bypassing the writer-lock check + // by going through the controller. This is a forcible teardown used when the stream's + // underlying connection is going away regardless of what JavaScript is doing with the stream + // (e.g. a Socket's writable side when the socket is closed or errors). Force-aborting a null + // stream is a no-op (resolved promise). + jsg::Promise forceAbort(jsg::Lock& js, jsg::Optional reason); + + // Closes the stream once all pending writes complete, bypassing the writer-lock check by going + // through the controller. Used when the stream must be closed regardless of what JavaScript is + // doing with it (e.g. closing a Socket's write side when its readable side reaches EOF). + // Precondition: !isNull(). + jsg::Promise forceClose(jsg::Lock& js); + + // Mark the stream as being in the process of shutting down (e.g. the Socket it belongs to is + // closing), before the closure has actually completed. A no-op on a null stream. + void setPendingClosure(jsg::Lock& js); + + // Detach the underlying stream from its implementation, leaving it permanently locked and + // unusable for further writes. Unlike JsReadableStream::detach(), nothing is returned: the + // stream is simply neutralized in place (used when the underlying connection is taken over by + // another consumer, e.g. startTls or Socket::takeConnectionStream). Throws if the stream is + // locked to a writer or is closed/closing; if the stream is errored, throws the stored error. + // Precondition: !isNull(). + void detach(jsg::Lock& js); + + void visitForGc(jsg::GcVisitor& visitor); + void visitForMemoryInfo(jsg::MemoryTracker& tracker) const; + + // Describe this type to RTTI (and therefore to generated TypeScript) exactly as a + // WritableStream. See the delegated-RTTI support in jsg/rtti.h. + using JsgRttiDelegate = jsg::Ref; + + static v8::Local jsgWrap(auto& typeWrapper, + jsg::Lock& js, + v8::Local context, + kj::Maybe> creator, + JsWritableStream stream) { + // Wrapping a null JsWritableStream indicates a bug: APIs that can produce "no stream" + // express that as kj::Maybe / jsg::Optional so that + // absence maps to JS null/undefined rather than to a fabricated stream. + auto& impl = KJ_ASSERT_NONNULL(stream.impl, "cannot wrap a null JsWritableStream"); + KJ_SWITCH_ONEOF(impl.stream) { + KJ_CASE_ONEOF(legacy, jsg::Ref) { + return typeWrapper.wrap(js, context, creator, kj::mv(legacy)); + } + KJ_CASE_ONEOF(ts, jsg::JsRef) { + KJ_UNIMPLEMENTED("TypeScript WritableStream not yet supported"); + } + } + KJ_UNREACHABLE; + } + + static kj::Maybe jsgTryUnwrap(auto& typeWrapper, + jsg::Lock& js, + v8::Local context, + v8::Local handle, + kj::Maybe> parentObject) { + // For now, we only support unwrapping the legacy C++ WritableStream. + // Later we will also support the TypeScript implementation. + return typeWrapper.tryUnwrap( + js, context, handle, static_cast*>(nullptr), parentObject); + } + + private: + explicit JsWritableStream(Impl impl): impl(kj::mv(impl)) {} + + kj::Maybe impl; +}; + +} // namespace workerd::api diff --git a/src/workerd/api/streams.h b/src/workerd/api/streams.h index 21f284ddda0..624c82f3285 100644 --- a/src/workerd/api/streams.h +++ b/src/workerd/api/streams.h @@ -8,6 +8,7 @@ // This is the most over-engineered spec... #include +#include #include #include #include From 576b8bf9ec7d7cec73540c1149705ccbd93504cf Mon Sep 17 00:00:00 2001 From: James M Snell Date: Tue, 7 Jul 2026 14:01:22 -0700 Subject: [PATCH 064/101] Scaffold pipeTo/pipeThrough support in JsReadableStream/JsWritableStream --- src/workerd/api/api-rtti-test.c++ | 12 ++ src/workerd/api/js-readable-stream.c++ | 80 +++++++++ src/workerd/api/js-readable-stream.h | 23 +++ src/workerd/api/js-writable-stream-test.c++ | 184 ++++++++++++++++++++ src/workerd/api/js-writable-stream.c++ | 9 + src/workerd/api/js-writable-stream.h | 83 +++++++++ 6 files changed, 391 insertions(+) diff --git a/src/workerd/api/api-rtti-test.c++ b/src/workerd/api/api-rtti-test.c++ index 3b9c8c18a32..ef408af4f72 100644 --- a/src/workerd/api/api-rtti-test.c++ +++ b/src/workerd/api/api-rtti-test.c++ @@ -65,5 +65,17 @@ KJ_TEST("JsWritableStream delegates its RTTI to WritableStream") { KJ_EXPECT(builder.structure("workerd::api::WritableStream"_kj) != kj::none); } +KJ_TEST("JsReadableWritablePair delegates its RTTI to ReadableStream::Transform") { + // JsReadableWritablePair declares `using JsgRttiDelegate = ReadableStream::Transform`, so RTTI + // (and therefore generated TypeScript) must describe it exactly as it describes the + // ReadableWritablePair-shaped Transform struct. + jsg::rtti::Builder builder((CompatibilityFlags::Reader())); + auto type = builder.type(); + KJ_ASSERT(type.isStructure()); + KJ_EXPECT( + type.getStructure().getFullyQualifiedName() == "workerd::api::ReadableStream::Transform"_kj, + type.getStructure().getFullyQualifiedName()); +} + } // namespace } // namespace workerd::api diff --git a/src/workerd/api/js-readable-stream.c++ b/src/workerd/api/js-readable-stream.c++ index b9569372be8..3eac34142c0 100644 --- a/src/workerd/api/js-readable-stream.c++ +++ b/src/workerd/api/js-readable-stream.c++ @@ -4,6 +4,7 @@ #include #include +#include #include #include #include @@ -369,6 +370,85 @@ kj::Promise> JsReadableStream::pumpTo( KJ_UNREACHABLE; } +jsg::Promise JsReadableStream::pipeTo( + jsg::Lock& js, JsWritableStream& destination, PipeToOptions options) { + auto& i = KJ_ASSERT_NONNULL(impl, "pipeTo() called on a null JsReadableStream"); + auto& d = KJ_ASSERT_NONNULL( + destination.impl, "pipeTo() called with a null destination JsWritableStream"); + KJ_SWITCH_ONEOF(i.stream) { + KJ_CASE_ONEOF(stream, jsg::Ref) { + KJ_SWITCH_ONEOF(d.stream) { + KJ_CASE_ONEOF(writable, jsg::Ref) { + // Both ends are C++: delegate to ReadableStream::pipeTo, which preserves the exact + // observable behavior (locked-end rejections and their texts) and internally selects + // the most efficient pump for the endpoint types. + return stream->pipeTo(js, writable.addRef(), kj::mv(options)); + } + KJ_CASE_ONEOF(obj, jsg::JsRef) { + // TODO(streams-ts): TS/TS pipes go through the TS pipeTo hook; mixed-backend pipes + // are a wiring-session concern. + KJ_UNIMPLEMENTED("TypeScript-backed WritableStream is not yet supported"); + } + } + KJ_UNREACHABLE; + } + KJ_CASE_ONEOF(obj, jsg::JsRef) { + KJ_UNIMPLEMENTED("TypeScript-backed ReadableStream is not yet supported"); + } + } + KJ_UNREACHABLE; +} + +JsReadableStream JsReadableStream::pipeThrough( + jsg::Lock& js, JsReadableWritablePair transform, PipeToOptions options) { + auto& i = KJ_ASSERT_NONNULL(impl, "pipeThrough() called on a null JsReadableStream"); + auto& r = KJ_ASSERT_NONNULL( + transform.readable.impl, "pipeThrough() called with a null transform.readable"); + auto& w = KJ_ASSERT_NONNULL( + transform.writable.impl, "pipeThrough() called with a null transform.writable"); + + KJ_SWITCH_ONEOF(i.stream) { + KJ_CASE_ONEOF(stream, jsg::Ref) { + KJ_SWITCH_ONEOF(w.stream) { + KJ_CASE_ONEOF(writable, jsg::Ref) { + KJ_SWITCH_ONEOF(r.stream) { + KJ_CASE_ONEOF(readable, jsg::Ref) { + // All three arms are C++: delegate to ReadableStream::pipeThrough, which preserves + // the exact observable behavior (synchronous locked-end throws and their texts, + // the pipeThrough-marked handled pipe promise, and the source keepalive). + auto returned = stream->pipeThrough(js, + ReadableStream::Transform{ + .readable = readable.addRef(), + .writable = writable.addRef(), + }, + kj::mv(options)); + // ReadableStream::pipeThrough returns the transform's readable untouched. Return + // the abstraction value it arrived in (the same underlying stream), which + // preserves any additional abstraction-level state (e.g. buffer-backedness). + KJ_ASSERT(returned.get() == readable.get()); + return kj::mv(transform.readable); + } + KJ_CASE_ONEOF(obj, jsg::JsRef) { + KJ_UNIMPLEMENTED("TypeScript-backed ReadableStream is not yet supported"); + } + } + KJ_UNREACHABLE; + } + KJ_CASE_ONEOF(obj, jsg::JsRef) { + // TODO(streams-ts): all-TS pipes go through the TS pipeThrough hook; mixed-backend + // pipes are a wiring-session concern. + KJ_UNIMPLEMENTED("TypeScript-backed WritableStream is not yet supported"); + } + } + KJ_UNREACHABLE; + } + KJ_CASE_ONEOF(obj, jsg::JsRef) { + KJ_UNIMPLEMENTED("TypeScript-backed ReadableStream is not yet supported"); + } + } + KJ_UNREACHABLE; +} + JsReadableStream::Tee JsReadableStream::tee(jsg::Lock& js) { auto& i = KJ_ASSERT_NONNULL(impl, "tee() called on a null JsReadableStream"); KJ_SWITCH_ONEOF(i.stream) { diff --git a/src/workerd/api/js-readable-stream.h b/src/workerd/api/js-readable-stream.h index e8befde92a0..69b8303dd35 100644 --- a/src/workerd/api/js-readable-stream.h +++ b/src/workerd/api/js-readable-stream.h @@ -16,6 +16,8 @@ namespace workerd::api { class Blob; class FormData; +class JsWritableStream; +struct JsReadableWritablePair; class URLSearchParams; namespace url { class URLSearchParams; @@ -173,6 +175,27 @@ class JsReadableStream final { kj::Promise> pumpTo( jsg::Lock& js, kj::Own sink, EndStream end); + // Pipe this stream into the given destination, exactly like ReadableStream.prototype.pipeTo(): + // both ends are locked for the duration of the pipe, and the returned promise settles when the + // pipe completes. Rejects (does not throw) if either end is already locked. By default the + // destination is closed when this stream ends and aborted if it errors; see PipeToOptions. + // + // The destination is borrowed, not consumed: the caller's handle remains valid and observes the + // stream's state as the pipe progresses. Unlike pumpTo(), this is a spec-level, isolate-bound + // pipe with no deferred-proxy support; when both ends are backed by the C++ implementation the + // pipe still uses the controllers' internal native fast paths. Preconditions: !isNull(), + // !destination.isNull(). + jsg::Promise pipeTo( + jsg::Lock& js, JsWritableStream& destination, PipeToOptions options = {}); + + // Pipe this stream through the given transform, exactly like + // ReadableStream.prototype.pipeThrough(): pipes this into transform.writable (with the pipe + // promise marked as handled) and returns transform.readable. Throws synchronously if this + // stream or transform.writable is already locked. The transform is consumed. Preconditions: + // !isNull(), neither transform member is null. + JsReadableStream pipeThrough( + jsg::Lock& js, JsReadableWritablePair transform, PipeToOptions options = {}); + // Split a live stream into two independent branches that each read the same data, consuming // (nullifying) this. Both branches share the retransmit buffer (if any). Works for any stream. // Precondition: !isNull(). diff --git a/src/workerd/api/js-writable-stream-test.c++ b/src/workerd/api/js-writable-stream-test.c++ index dc8dc5ccb18..5914c46dbd0 100644 --- a/src/workerd/api/js-writable-stream-test.c++ +++ b/src/workerd/api/js-writable-stream-test.c++ @@ -3,6 +3,8 @@ // https://opensource.org/licenses/Apache-2.0 #include +#include +#include #include #include @@ -10,6 +12,14 @@ namespace workerd::api { namespace { +// The full wrap/unwrap template bodies are only instantiated once these types appear in +// JSG-visible signatures; until then, at least pin the SelfConvertible signatures. +static_assert(jsg::SelfConvertible); +static_assert(jsg::SelfConvertible); + +constexpr uint64_t kLimit = 1024 * 1024; +constexpr kj::StringPtr kData = "hello world"_kj; + // The user-visible message produced when performing a lock-checked operation on a stream that is // locked to a writer. This must match the message produced by WritableStream exactly. constexpr kj::StringPtr kWriterLockedError = @@ -282,6 +292,180 @@ KJ_TEST("JsWritableStream setPendingClosure is safe on live and null streams") { }); } +// Build a JsReadableWritablePair from the two ends of an IdentityTransformStream, the way the +// two-tier unwrap's brand-first path would. +JsReadableWritablePair makeIdentityPair(jsg::Lock& js) { + auto transform = IdentityTransformStream::constructor(js); + return JsReadableWritablePair{ + .readable = JsReadableStream(transform->getReadable()), + .writable = JsWritableStream(transform->getWritable()), + }; +} + +KJ_TEST("JsReadableStream pipeTo pipes into a JsWritableStream and closes it") { + TestFixture testFixture; + SinkState state; + testFixture.runInIoContext([&](const TestFixture::Environment& env) -> kj::Promise { + auto& js = env.js; + + JsReadableStream source(js, kj::str(kData)); + auto destination = JsWritableStream::create(js, env.context, state.makeSink(), kj::none); + auto promise = source.pipeTo(js, destination); + return env.context.awaitJs(js, kj::mv(promise)); + }); + KJ_EXPECT(state.written.asPtr() == kData.asBytes()); + KJ_EXPECT(state.ended); +} + +KJ_TEST("JsReadableStream pipeTo with preventClose leaves the destination open") { + TestFixture testFixture; + SinkState state; + testFixture.runInIoContext([&](const TestFixture::Environment& env) -> kj::Promise { + auto& js = env.js; + + JsReadableStream source(js, kj::str(kData)); + auto destination = JsWritableStream::create(js, env.context, state.makeSink(), kj::none); + auto promise = source.pipeTo(js, destination, PipeToOptions{.preventClose = true}); + return env.context.awaitJs(js, kj::mv(promise)); + }); + KJ_EXPECT(state.written.asPtr() == kData.asBytes()); + KJ_EXPECT(!state.ended); +} + +KJ_TEST("JsReadableStream pipeTo rejects when the source is locked") { + TestFixture testFixture; + SinkState state; + testFixture.runInIoContext([&](const TestFixture::Environment& env) -> kj::Promise { + auto& js = env.js; + + // detach() leaves the original as a locked husk, which is a convenient way to produce a + // locked stream without a reader. + JsReadableStream source(js, kj::str(kData)); + auto detached = source.detach(js); + KJ_EXPECT(source.isLocked(js)); + + auto destination = JsWritableStream::create(js, env.context, state.makeSink(), kj::none); + auto promise = source.pipeTo(js, destination).then(js, [](jsg::Lock& js) { + KJ_FAIL_REQUIRE("expected pipeTo() from a locked source to reject"); + }, [](jsg::Lock& js, jsg::Value exception) { + auto e = js.exceptionToKj(kj::mv(exception)); + KJ_EXPECT(e.getDescription().contains("currently locked to a reader"), e.getDescription()); + }); + return env.context.awaitJs(js, kj::mv(promise)); + }); + KJ_EXPECT(!state.ended); +} + +KJ_TEST("JsReadableStream pipeTo rejects when the destination is locked") { + TestFixture testFixture; + SinkState state; + testFixture.runInIoContext([&](const TestFixture::Environment& env) -> kj::Promise { + auto& js = env.js; + + JsReadableStream source(js, kj::str(kData)); + auto ws = js.alloc(env.context, state.makeSink(), kj::none); + auto writer = ws->getWriter(js); + JsWritableStream destination(kj::mv(ws)); + + auto promise = source.pipeTo(js, destination).then(js, [](jsg::Lock& js) { + KJ_FAIL_REQUIRE("expected pipeTo() into a locked destination to reject"); + }, [](jsg::Lock& js, jsg::Value exception) { + auto e = js.exceptionToKj(kj::mv(exception)); + // Note: unlike every other occurrence of this message, ReadableStream::pipeTo's + // destination-locked rejection has no trailing period. That text is load-bearing. + KJ_EXPECT(e.getDescription() == + "jsg.TypeError: This WritableStream is currently locked to a writer"_kj, + e.getDescription()); + }); + return env.context.awaitJs(js, kj::mv(promise)).attach(kj::mv(writer)); + }); +} + +KJ_TEST("JsReadableStream pipeThrough pipes through an identity transform") { + TestFixture testFixture; + testFixture.runInIoContext([](const TestFixture::Environment& env) -> kj::Promise { + auto& js = env.js; + + JsReadableStream source(js, kj::str(kData)); + auto result = source.pipeThrough(js, makeIdentityPair(js)); + KJ_EXPECT(!result.isNull()); + // The source is now locked into the pipe. + KJ_EXPECT(source.isLocked(js)); + + auto promise = result.text(js, kLimit).then(js, [](jsg::Lock& js, kj::String text) { + KJ_EXPECT(text == kData); + }); + return env.context.awaitJs(js, kj::mv(promise)); + }); +} + +KJ_TEST("JsReadableStream pipeThrough composes with pipeTo") { + // The readable returned by pipeThrough() feeds directly into further pipeline calls: + // source.pipeThrough(transform).pipeTo(destination). (Chaining two IdentityTransformStreams + // back to back is NOT covered here: the legacy C++ implementation rejects direct + // inter-IdentityTransformStream pipes -- "Inter-TransformStream ReadableStream.pipeTo() is + // not implemented", identity-transform-stream.c++ -- a pre-existing implementation + // limitation, identical in JavaScript, and not this abstraction's behavior to pin.) + TestFixture testFixture; + SinkState state; + testFixture.runInIoContext([&](const TestFixture::Environment& env) -> kj::Promise { + auto& js = env.js; + + JsReadableStream source(js, kj::str(kData)); + auto destination = JsWritableStream::create(js, env.context, state.makeSink(), kj::none); + auto promise = source.pipeThrough(js, makeIdentityPair(js)).pipeTo(js, destination); + return env.context.awaitJs(js, kj::mv(promise)); + }); + KJ_EXPECT(state.written.asPtr() == kData.asBytes()); + KJ_EXPECT(state.ended); +} + +KJ_TEST("JsReadableStream pipeThrough throws synchronously when the source is locked") { + TestFixture testFixture; + testFixture.runInIoContext([](const TestFixture::Environment& env) { + auto& js = env.js; + + JsReadableStream source(js, kj::str(kData)); + auto detached = source.detach(js); + KJ_EXPECT(source.isLocked(js)); + + js.tryCatch([&] { + source.pipeThrough(js, makeIdentityPair(js)); + KJ_FAIL_REQUIRE("expected pipeThrough() from a locked source to throw"); + }, [&](jsg::Value exception) { + auto e = js.exceptionToKj(kj::mv(exception)); + KJ_EXPECT(e.getDescription().contains("currently locked to a reader"), e.getDescription()); + }); + }); +} + +KJ_TEST("JsReadableStream pipeThrough throws synchronously when the transform writable is locked") { + TestFixture testFixture; + testFixture.runInIoContext([](const TestFixture::Environment& env) { + auto& js = env.js; + + JsReadableStream source(js, kj::str(kData)); + + auto transform = IdentityTransformStream::constructor(js); + auto writable = transform->getWritable(); + auto writer = writable->getWriter(js); + JsReadableWritablePair pair{ + .readable = JsReadableStream(transform->getReadable()), + .writable = JsWritableStream(kj::mv(writable)), + }; + + js.tryCatch([&] { + source.pipeThrough(js, kj::mv(pair)); + KJ_FAIL_REQUIRE("expected pipeThrough() into a locked transform writable to throw"); + }, [&](jsg::Value exception) { + auto e = js.exceptionToKj(kj::mv(exception)); + // pipeThrough's synchronous destination-locked message carries the trailing period, + // unlike pipeTo's rejection. + KJ_EXPECT(e.getDescription().contains(kWriterLockedError), e.getDescription()); + }); + }); +} + KJ_TEST("JsWritableStream create honors the closure waitable") { // Note: create()'s maybeHighWaterMark is likewise a pass-through to the internal controller; // its backpressure behavior is not observable through the abstraction's narrow API and is diff --git a/src/workerd/api/js-writable-stream.c++ b/src/workerd/api/js-writable-stream.c++ index 7ebace79199..1b54d4bbb4a 100644 --- a/src/workerd/api/js-writable-stream.c++ +++ b/src/workerd/api/js-writable-stream.c++ @@ -200,4 +200,13 @@ void JsWritableStream::visitForMemoryInfo(jsg::MemoryTracker& tracker) const { } } +void JsReadableWritablePair::visitForGc(jsg::GcVisitor& visitor) { + visitor.visit(readable, writable); +} + +void JsReadableWritablePair::visitForMemoryInfo(jsg::MemoryTracker& tracker) const { + readable.visitForMemoryInfo(tracker); + writable.visitForMemoryInfo(tracker); +} + } // namespace workerd::api diff --git a/src/workerd/api/js-writable-stream.h b/src/workerd/api/js-writable-stream.h index 7982a051c9e..402107246ef 100644 --- a/src/workerd/api/js-writable-stream.h +++ b/src/workerd/api/js-writable-stream.h @@ -4,6 +4,8 @@ #pragma once +#include +#include #include #include @@ -175,6 +177,87 @@ class JsWritableStream final { explicit JsWritableStream(Impl impl): impl(kj::mv(impl)) {} kj::Maybe impl; + + // JsReadableStream::pipeTo()/pipeThrough() dispatch on the backend of both pipe ends, which + // requires access to the destination's internal arm. + friend class JsReadableStream; +}; + +// A transform endpoint pair: a readable side and a writable side, typically (but not +// necessarily) the two ends of a TransformStream, without prescribing which backend implements +// them. This is the abstraction-level equivalent of the spec's ReadableWritablePair dictionary +// (and of the ReadableStream::Transform JSG_STRUCT), and is the argument type of +// JsReadableStream::pipeThrough(). +struct JsReadableWritablePair { + JsReadableStream readable; + JsWritableStream writable; + + // Describe this type to RTTI (and therefore to generated TypeScript) exactly as + // ReadableStream::Transform (whose TS override is ReadableWritablePair). See the + // delegated-RTTI support in jsg/rtti.h. + using JsgRttiDelegate = ReadableStream::Transform; + + static v8::Local jsgWrap(auto& typeWrapper, + jsg::Lock& js, + v8::Local context, + kj::Maybe> creator, + JsReadableWritablePair pair) { + // Dictionary semantics: produce a plain { readable, writable } object, wrapping each member + // through its own conversion. Null members trip the members' own null-wrap asserts. + auto obj = js.obj(); + obj.set(js, "readable"_kj, + jsg::JsValue(typeWrapper.wrap(js, context, creator, kj::mv(pair.readable)))); + obj.set(js, "writable"_kj, + jsg::JsValue(typeWrapper.wrap(js, context, creator, kj::mv(pair.writable)))); + return obj; + } + + static kj::Maybe jsgTryUnwrap(auto& typeWrapper, + jsg::Lock& js, + v8::Local context, + v8::Local handle, + kj::Maybe> parentObject) { + // Tier 1 (brand-first): a genuine C++ TransformStream (or subclass, e.g. + // IdentityTransformStream) is used directly via its C++ accessors. No JS property reads + // occur, so instance-shadowed readable/writable getters are ignored -- exactly the semantics + // a jsg::Ref parameter has today. + KJ_IF_SOME(transform, + typeWrapper.tryUnwrap( + js, context, handle, static_cast*>(nullptr), parentObject)) { + return JsReadableWritablePair{ + .readable = JsReadableStream(transform->getReadable()), + .writable = JsWritableStream(transform->getWritable()), + }; + } + + // Tier 2 (dictionary-shaped fallback): read the readable/writable properties (in + // ReadableStream::Transform field order) and unwrap each through the member abstractions' + // own conversions. This is what keeps the pair backend-agnostic: brand checks live in the + // members (including, later, the TypeScript implementation's). Either member failing fails + // the whole unwrap. + if (!handle->IsObject()) { + return kj::none; + } + auto obj = jsg::JsObject(handle.As()); + auto readable = obj.get(js, "readable"_kj); + auto writable = obj.get(js, "writable"_kj); + KJ_IF_SOME(r, + typeWrapper.tryUnwrap( + js, context, readable, static_cast(nullptr), parentObject)) { + KJ_IF_SOME(w, + typeWrapper.tryUnwrap( + js, context, writable, static_cast(nullptr), parentObject)) { + return JsReadableWritablePair{ + .readable = kj::mv(r), + .writable = kj::mv(w), + }; + } + } + return kj::none; + } + + void visitForGc(jsg::GcVisitor& visitor); + void visitForMemoryInfo(jsg::MemoryTracker& tracker) const; }; } // namespace workerd::api From dd171a4d124a8b5438247b8f61f99892d2b13e64 Mon Sep 17 00:00:00 2001 From: James M Snell Date: Tue, 7 Jul 2026 14:18:11 -0700 Subject: [PATCH 065/101] Use JsWritableStream in sockets code --- src/workerd/api/js-writable-stream.c++ | 13 ++++++++++++ src/workerd/api/js-writable-stream.h | 10 +++++++++ src/workerd/api/sockets-test.c++ | 2 +- src/workerd/api/sockets.c++ | 28 ++++++++++++-------------- src/workerd/api/sockets.h | 12 +++++------ 5 files changed, 43 insertions(+), 22 deletions(-) diff --git a/src/workerd/api/js-writable-stream.c++ b/src/workerd/api/js-writable-stream.c++ index 1b54d4bbb4a..61382dba6d6 100644 --- a/src/workerd/api/js-writable-stream.c++ +++ b/src/workerd/api/js-writable-stream.c++ @@ -172,6 +172,19 @@ void JsWritableStream::detach(jsg::Lock& js) { } } +jsg::Ref JsWritableStream::getUnderlyingForTest(jsg::Lock& js) { + auto& i = KJ_ASSERT_NONNULL(impl, "getUnderlyingForTest() called on a null JsWritableStream"); + KJ_SWITCH_ONEOF(i.stream) { + KJ_CASE_ONEOF(stream, jsg::Ref) { + return stream.addRef(); + } + KJ_CASE_ONEOF(obj, jsg::JsRef) { + KJ_UNIMPLEMENTED("TypeScript-backed WritableStream is not yet supported"); + } + } + KJ_UNREACHABLE; +} + void JsWritableStream::visitForGc(jsg::GcVisitor& visitor) { KJ_IF_SOME(i, impl) { KJ_SWITCH_ONEOF(i.stream) { diff --git a/src/workerd/api/js-writable-stream.h b/src/workerd/api/js-writable-stream.h index 402107246ef..a73aafcfb6d 100644 --- a/src/workerd/api/js-writable-stream.h +++ b/src/workerd/api/js-writable-stream.h @@ -135,6 +135,16 @@ class JsWritableStream final { // Precondition: !isNull(). void detach(jsg::Lock& js); + // Returns the underlying legacy C++ WritableStream. FOR TESTS ONLY: this exists so that tests + // of consumers (e.g. sockets-test.c++'s output-gate test) can drive operations the deliberately + // narrow production API does not expose, such as enqueueing writes through the standard write + // machinery. Production code must never call this -- it would break the moment the stream is + // backed by the TypeScript implementation. Precondition: !isNull() and legacy-backed. + // + // TODO(streams-ts): Revisit once the TypeScript arm is wired up -- tests that need to drive + // writes will need a backend-neutral mechanism (or per-backend test variants). + jsg::Ref getUnderlyingForTest(jsg::Lock& js); + void visitForGc(jsg::GcVisitor& visitor); void visitForMemoryInfo(jsg::MemoryTracker& tracker) const; diff --git a/src/workerd/api/sockets-test.c++ b/src/workerd/api/sockets-test.c++ index 61efdfb2942..7bc7b3161c4 100644 --- a/src/workerd/api/sockets-test.c++ +++ b/src/workerd/api/sockets-test.c++ @@ -120,7 +120,7 @@ KJ_TEST("socket writes are blocked by output gate") { // Prepare write data and lock gate BEFORE any co_await (Worker lock still held). auto paf = kj::newPromiseAndFulfiller(); auto blocker = actor.getOutputGate().lockWhile(kj::mv(paf.promise), nullptr); - auto writable = socket->getWritable(); + auto writable = socket->getWritable(env.js).getUnderlyingForTest(env.js); jsg::JsValue jsBuffer = jsg::JsUint8Array::create(env.js, "hi"_kjb); writable->getController().write(env.js, jsBuffer).markAsHandled(env.js); diff --git a/src/workerd/api/sockets.c++ b/src/workerd/api/sockets.c++ index 80dfda7b0f2..e39294637f8 100644 --- a/src/workerd/api/sockets.c++ +++ b/src/workerd/api/sockets.c++ @@ -239,7 +239,7 @@ jsg::Ref setupSocket(jsg::Lock& js, return js.newPromiseAndResolver(); }); openedPrPair.promise.markAsHandled(js); - auto writable = js.alloc(ioContext, kj::mv(sysStreams.writable), + auto writable = JsWritableStream::create(js, ioContext, kj::mv(sysStreams.writable), ioContext.getMetrics().tryCreateWritableByteStreamObserver(), getWritableHighWaterMark(options), openedPrPair.promise.whenResolved(js)); @@ -372,7 +372,7 @@ jsg::Promise Socket::close(jsg::Lock& js) { } isClosing = true; - writable->getController().setPendingClosure(); + writable.setPendingClosure(js); readable.setPendingClosure(js); // Wait until the socket connects (successfully or otherwise) @@ -381,9 +381,8 @@ jsg::Promise Socket::close(jsg::Lock& js) { return openedPromiseCopy.whenResolved(js) .then(js, [self = JSG_THIS](jsg::Lock& js) mutable { - auto& controller = self->writable->getController(); - if (!controller.isClosedOrClosing()) { - return controller.flush(js); + if (!self->writable.isClosedOrClosing(js)) { + return self->writable.forceFlush(js); } else { return js.resolvedPromise(); } @@ -392,7 +391,7 @@ jsg::Promise Socket::close(jsg::Lock& js) { [self = JSG_THIS](jsg::Lock& js) mutable { // Forcibly abort the readable/writable streams. auto cancelPromise = self->readable.forceCancel(js, kj::none); - auto abortPromise = self->writable->getController().abort(js, kj::none); + auto abortPromise = self->writable.forceAbort(js, kj::none); // The below is effectively `Promise.all(cancelPromise, abortPromise)` return cancelPromise.then(js, [abortPromise = kj::mv(abortPromise)](jsg::Lock& js) mutable { @@ -442,7 +441,7 @@ jsg::Ref Socket::startTls(jsg::Lock& js, jsg::Optional tlsOp auto& context = IoContext::current(); auto openedPrPair = js.newPromiseAndResolver(); auto secureStreamPromise = context.awaitJs(js, - writable->flush(js).then(js, + writable.flush(js).then(js, // The openedResolver is a jsg::Promise::Resolver. It should be gc visited here in // case the opened promise it resolves captures a circular references to itself in // JavaScript (which is most likely). This prevents a possible memory leak. @@ -459,7 +458,7 @@ jsg::Ref Socket::startTls(jsg::Lock& js, jsg::Optional tlsOp (self, openedResolver), (jsg::Lock & js) mutable { auto& context = IoContext::current(); - self->writable->detach(js); + self->writable.detach(js); self->readable.detach(js, IgnoreDisturbed::YES); // We should set this before closedResolver.resolve() in order to give the user @@ -605,7 +604,7 @@ void Socket::handleProxyError(jsg::Lock& js, kj::Exception e) { resolveFulfiller(js, e.clone()); openedResolver.reject(js, e.clone()); readable.forceCancel(js, kj::none).markAsHandled(js); - writable->getController().abort(js, js.error(e.getDescription())).markAsHandled(js); + writable.forceAbort(js, js.error(e.getDescription())).markAsHandled(js); } void Socket::handleReadableEof(jsg::Lock& js, jsg::Promise onEof) { @@ -625,9 +624,9 @@ jsg::Promise Socket::maybeCloseWriteSide(jsg::Lock& js) { // `allowHalfOpen` is false. KJ_ASSERT(!getAllowHalfOpen(options)); - // Do not call `close` on a controller that has already been closed or is in the process + // Do not call `close` on a stream that has already been closed or is in the process // of closing. - if (writable->getController().isClosedOrClosing()) { + if (writable.isClosedOrClosing(js)) { return js.resolvedPromise(); } @@ -635,8 +634,7 @@ jsg::Promise Socket::maybeCloseWriteSide(jsg::Lock& js) { // below by calling `close` on the WritableStream which ensures that any data pending on it // is flushed. Then once the `close` either completes or fails we can be sure that any data has // been flushed. - return writable->getController() - .close(js) + return writable.forceClose(js) .catch_(js, JSG_VISITABLE_LAMBDA((ref = JSG_THIS), (ref), (jsg::Lock& js, jsg::Value&& exc) { @@ -665,7 +663,7 @@ kj::Own Socket::takeConnectionStream(jsg::Lock& js) { // We do not care if the socket was disturbed, we require the user to ensure the socket is not // being used. - writable->detach(js); + writable.detach(js); readable.detach(js, IgnoreDisturbed::YES); // Move the stream out of the socket, to ensure the stream is properly destroyed when the @@ -773,7 +771,7 @@ jsg::Promise> SocketsModule::internalNewHttpClient( // Flush the writable stream before taking the connection stream to ensure all data is written // before the stream is detatched - return socket->getWritable()->flush(js).then( + return socket->getWritable(js).flush(js).then( js, JSG_VISITABLE_LAMBDA((socket = kj::mv(socket)), (socket), (jsg::Lock & js) mutable { auto& ioctx = IoContext::current(); diff --git a/src/workerd/api/sockets.h b/src/workerd/api/sockets.h index a4166e39511..f85d9c68d94 100644 --- a/src/workerd/api/sockets.h +++ b/src/workerd/api/sockets.h @@ -5,7 +5,7 @@ #pragma once #include -#include +#include #include #include #include @@ -66,7 +66,7 @@ class Socket: public jsg::Object { kj::Maybe remoteAddress, kj::Maybe localAddress, JsReadableStream readableParam, - jsg::Ref writable, + JsWritableStream writable, jsg::PromiseResolverPair closedPrPair, kj::Promise watchForDisconnectTask, jsg::Optional options, @@ -95,8 +95,8 @@ class Socket: public jsg::Object { JsReadableStream getReadable(jsg::Lock& js) { return readable.addRef(js); } - jsg::Ref getWritable() { - return writable.addRef(); + JsWritableStream getWritable(jsg::Lock& js) { + return writable.addRef(js); } jsg::MemoizedIdentity>& getClosed() { return closedPromise; @@ -169,7 +169,7 @@ class Socket: public jsg::Object { void visitForMemoryInfo(jsg::MemoryTracker& tracker) const { tracker.trackFieldWithSize("connectionData", sizeof(IoOwn)); readable.visitForMemoryInfo(tracker); - tracker.trackField("writable", writable); + writable.visitForMemoryInfo(tracker); tracker.trackField("closedResolver", closedResolver); tracker.trackField("closedPromiseCopy", closedPromiseCopy); tracker.trackField("closedPromise", closedPromise); @@ -197,7 +197,7 @@ class Socket: public jsg::Object { kj::Maybe> connectionData; JsReadableStream readable; - jsg::Ref writable; + JsWritableStream writable; // This fulfiller is used to resolve the `closedPromise` below. jsg::Promise::Resolver closedResolver; // Copy kept so that it can be returned from `close`. From 6ca22b21e9c368045e36e9fae49f9729c650cb88 Mon Sep 17 00:00:00 2001 From: James M Snell Date: Tue, 7 Jul 2026 14:27:20 -0700 Subject: [PATCH 066/101] Use JsWritableStream in containers code --- src/workerd/api/AGENTS.md | 2 +- src/workerd/api/container.c++ | 10 +++++----- src/workerd/api/container.h | 12 +++++++----- src/workerd/api/streams/AGENTS.md | 10 ++++++---- 4 files changed, 19 insertions(+), 15 deletions(-) diff --git a/src/workerd/api/AGENTS.md b/src/workerd/api/AGENTS.md index 5b515ea085a..53922c6ff96 100644 --- a/src/workerd/api/AGENTS.md +++ b/src/workerd/api/AGENTS.md @@ -35,7 +35,7 @@ tests/ # JS integration tests (238 entries); each test = .js + .wd | KV / SyncKV | `kv.h`, `sync-kv.h` | | SQL (DO) | `sql.h` | | Body mixin (shared Req/Res) | `http.h` — `Body` class, `Body::Initializer` OneOf | -| ReadableStream from C++ code | `js-readable-stream.{h,c++}` — `JsReadableStream` abstraction; ALWAYS use it (not `jsg::Ref`) outside `streams/` | +| Readable/WritableStream from C++ | `js-readable-stream.{h,c++}` + `js-writable-stream.{h,c++}` — `JsReadableStream`/`JsWritableStream` abstractions, plus `JsReadableWritablePair` and `pipeTo`/`pipeThrough`; ALWAYS use these (not `jsg::Ref`/`jsg::Ref`) outside `streams/` | ## CONVENTIONS diff --git a/src/workerd/api/container.c++ b/src/workerd/api/container.c++ index 79029d2cb9c..8c29b08fa5e 100644 --- a/src/workerd/api/container.c++ +++ b/src/workerd/api/container.c++ @@ -80,7 +80,7 @@ jsg::JsArrayBuffer ExecOutput::getStderr(jsg::Lock& js) { ExecProcess::ExecProcess(jsg::Lock& js, IoContext& ioContext, - jsg::Optional> stdinStream, + jsg::Optional stdinStream, jsg::Optional stdoutStream, jsg::Optional stderrStream, int pid, @@ -117,8 +117,8 @@ ExecProcess::ExecProcess(jsg::Lock& js, } } -jsg::Optional> ExecProcess::getStdin() { - return stdinStream.map([](jsg::Ref& stream) { return stream.addRef(); }); +jsg::Optional ExecProcess::getStdin(jsg::Lock& js) { + return stdinStream.map([&](JsWritableStream& stream) { return stream.addRef(js); }); } jsg::Optional ExecProcess::getStdout(jsg::Lock& js) { @@ -577,7 +577,7 @@ jsg::Promise> Container::exec( stderrStream = JsReadableStream::create(js, ioContext, kj::mv(source)); } - jsg::Optional> stdinStream = kj::none; + jsg::Optional stdinStream = kj::none; // If stdin is undefined, the JS API promises immediate EOF. We still use the pipelined stdin() // capability so exec() doesn't wait on an extra round-trip. @@ -601,7 +601,7 @@ jsg::Promise> Container::exec( JSG_REQUIRE( mode == "pipe", TypeError, "stdin must be a ReadableStream or the string \"pipe\"."); auto sink = newSystemStream(kj::mv(stdinWriter), StreamEncoding::IDENTITY, ioContext); - auto writable = js.alloc(ioContext, kj::mv(sink), + auto writable = JsWritableStream::create(js, ioContext, kj::mv(sink), ioContext.getMetrics().tryCreateWritableByteStreamObserver()); stdinStream = kj::mv(writable); } diff --git a/src/workerd/api/container.h b/src/workerd/api/container.h index 54b77743ec2..09a68aa0b13 100644 --- a/src/workerd/api/container.h +++ b/src/workerd/api/container.h @@ -7,7 +7,7 @@ // #include #include -#include +#include #include #include #include @@ -80,14 +80,14 @@ class ExecProcess: public jsg::Object { public: ExecProcess(jsg::Lock& js, IoContext& ioContext, - jsg::Optional> stdinStream, + jsg::Optional stdinStream, jsg::Optional stdoutStream, jsg::Optional stderrStream, int pid, rpc::Container::ProcessHandle::Client handle, kj::Maybe> abortSignal = kj::none); - jsg::Optional> getStdin(); + jsg::Optional getStdin(jsg::Lock& js); jsg::Optional getStdout(jsg::Lock& js); jsg::Optional getStderr(jsg::Lock& js); int getPid() const { @@ -119,7 +119,9 @@ class ExecProcess: public jsg::Object { } void visitForMemoryInfo(jsg::MemoryTracker& tracker) const { - tracker.trackField("stdin", stdinStream); + KJ_IF_SOME(s, stdinStream) { + s.visitForMemoryInfo(tracker); + } KJ_IF_SOME(s, stdoutStream) { s.visitForMemoryInfo(tracker); } @@ -138,7 +140,7 @@ class ExecProcess: public jsg::Object { // the AbortSignal handler. void sendKill(int signo); - jsg::Optional> stdinStream; + jsg::Optional stdinStream; jsg::Optional stdoutStream; jsg::Optional stderrStream; int pid; diff --git a/src/workerd/api/streams/AGENTS.md b/src/workerd/api/streams/AGENTS.md index 8820fb88469..118947c6d34 100644 --- a/src/workerd/api/streams/AGENTS.md +++ b/src/workerd/api/streams/AGENTS.md @@ -6,10 +6,12 @@ See `docs/streams.md` for narrative tutorial. ## ARCHITECTURE -NOTE: C++ code outside this directory does not use `jsg::Ref` directly; it goes -through the `JsReadableStream` abstraction in `src/workerd/api/js-readable-stream.{h,c++}`, which -hides which ReadableStream implementation backs a given stream. New C++ consumers of readable -streams should use that abstraction, not the types defined here. +NOTE: C++ code outside this directory does not use `jsg::Ref` or +`jsg::Ref` directly; it goes through the `JsReadableStream` / `JsWritableStream` +abstractions in `src/workerd/api/js-{readable,writable}-stream.{h,c++}`, which hide which stream +implementation backs a given stream (and provide `JsReadableWritablePair` + +`pipeTo`/`pipeThrough` for abstraction-level pipelines). New C++ consumers of streams should use +those abstractions, not the types defined here. Dual implementation behind unified API: From 94ac2bf14bad65827c50587aaadb78e696431490 Mon Sep 17 00:00:00 2001 From: James M Snell Date: Tue, 7 Jul 2026 14:42:22 -0700 Subject: [PATCH 067/101] Fix copyright header years --- src/workerd/api/js-readable-stream-test.c++ | 2 +- src/workerd/api/js-readable-stream.c++ | 2 +- src/workerd/api/js-readable-stream.h | 2 +- src/workerd/api/js-writable-stream-test.c++ | 11 +++++++---- src/workerd/api/js-writable-stream.c++ | 2 +- src/workerd/api/js-writable-stream.h | 2 +- 6 files changed, 12 insertions(+), 9 deletions(-) diff --git a/src/workerd/api/js-readable-stream-test.c++ b/src/workerd/api/js-readable-stream-test.c++ index 0db77198112..d70f0208673 100644 --- a/src/workerd/api/js-readable-stream-test.c++ +++ b/src/workerd/api/js-readable-stream-test.c++ @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2022 Cloudflare, Inc. +// Copyright (c) 2026 Cloudflare, Inc. // Licensed under the Apache 2.0 license found in the LICENSE file or at: // https://opensource.org/licenses/Apache-2.0 diff --git a/src/workerd/api/js-readable-stream.c++ b/src/workerd/api/js-readable-stream.c++ index 3eac34142c0..3a61f5a6b9b 100644 --- a/src/workerd/api/js-readable-stream.c++ +++ b/src/workerd/api/js-readable-stream.c++ @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2022 Cloudflare, Inc. +// Copyright (c) 2026 Cloudflare, Inc. // Licensed under the Apache 2.0 license found in the LICENSE file or at: // https://opensource.org/licenses/Apache-2.0 diff --git a/src/workerd/api/js-readable-stream.h b/src/workerd/api/js-readable-stream.h index 69b8303dd35..cddd983afd6 100644 --- a/src/workerd/api/js-readable-stream.h +++ b/src/workerd/api/js-readable-stream.h @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2022 Cloudflare, Inc. +// Copyright (c) 2026 Cloudflare, Inc. // Licensed under the Apache 2.0 license found in the LICENSE file or at: // https://opensource.org/licenses/Apache-2.0 diff --git a/src/workerd/api/js-writable-stream-test.c++ b/src/workerd/api/js-writable-stream-test.c++ index 5914c46dbd0..b778b4611e1 100644 --- a/src/workerd/api/js-writable-stream-test.c++ +++ b/src/workerd/api/js-writable-stream-test.c++ @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2022 Cloudflare, Inc. +// Copyright (c) 2026 Cloudflare, Inc. // Licensed under the Apache 2.0 license found in the LICENSE file or at: // https://opensource.org/licenses/Apache-2.0 @@ -310,7 +310,9 @@ KJ_TEST("JsReadableStream pipeTo pipes into a JsWritableStream and closes it") { JsReadableStream source(js, kj::str(kData)); auto destination = JsWritableStream::create(js, env.context, state.makeSink(), kj::none); - auto promise = source.pipeTo(js, destination); + // source and destination must outlive the pipe: capture them into the continuation so they + // are not destroyed when the lambda returns (the pipe's write loop still references them). + auto promise = source.pipeTo(js, destination).then(js, JSG_VISITABLE_LAMBDA((source = kj::mv(source), destination = kj::mv(destination)), (source, destination), (jsg::Lock& js){})); return env.context.awaitJs(js, kj::mv(promise)); }); KJ_EXPECT(state.written.asPtr() == kData.asBytes()); @@ -325,7 +327,7 @@ KJ_TEST("JsReadableStream pipeTo with preventClose leaves the destination open") JsReadableStream source(js, kj::str(kData)); auto destination = JsWritableStream::create(js, env.context, state.makeSink(), kj::none); - auto promise = source.pipeTo(js, destination, PipeToOptions{.preventClose = true}); + auto promise = source.pipeTo(js, destination, PipeToOptions{.preventClose = true}).then(js, JSG_VISITABLE_LAMBDA((source = kj::mv(source), destination = kj::mv(destination)), (source, destination), (jsg::Lock& js){})); return env.context.awaitJs(js, kj::mv(promise)); }); KJ_EXPECT(state.written.asPtr() == kData.asBytes()); @@ -413,7 +415,8 @@ KJ_TEST("JsReadableStream pipeThrough composes with pipeTo") { JsReadableStream source(js, kj::str(kData)); auto destination = JsWritableStream::create(js, env.context, state.makeSink(), kj::none); - auto promise = source.pipeThrough(js, makeIdentityPair(js)).pipeTo(js, destination); + auto piped = source.pipeThrough(js, makeIdentityPair(js)); + auto promise = piped.pipeTo(js, destination).then(js, JSG_VISITABLE_LAMBDA((source = kj::mv(source), piped = kj::mv(piped), destination = kj::mv(destination)), (source, piped, destination), (jsg::Lock& js){})); return env.context.awaitJs(js, kj::mv(promise)); }); KJ_EXPECT(state.written.asPtr() == kData.asBytes()); diff --git a/src/workerd/api/js-writable-stream.c++ b/src/workerd/api/js-writable-stream.c++ index 61382dba6d6..efa0e6457c7 100644 --- a/src/workerd/api/js-writable-stream.c++ +++ b/src/workerd/api/js-writable-stream.c++ @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2022 Cloudflare, Inc. +// Copyright (c) 2026 Cloudflare, Inc. // Licensed under the Apache 2.0 license found in the LICENSE file or at: // https://opensource.org/licenses/Apache-2.0 diff --git a/src/workerd/api/js-writable-stream.h b/src/workerd/api/js-writable-stream.h index a73aafcfb6d..37a66ff8b65 100644 --- a/src/workerd/api/js-writable-stream.h +++ b/src/workerd/api/js-writable-stream.h @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2022 Cloudflare, Inc. +// Copyright (c) 2026 Cloudflare, Inc. // Licensed under the Apache 2.0 license found in the LICENSE file or at: // https://opensource.org/licenses/Apache-2.0 From 015c3583d77f8d6e0e41b6c5b127e411a8f62998 Mon Sep 17 00:00:00 2001 From: James M Snell Date: Wed, 8 Jul 2026 13:27:24 -0700 Subject: [PATCH 068/101] Fixup pipeTo holding strong reference --- src/workerd/api/js-readable-stream.c++ | 11 ++- src/workerd/api/js-readable-stream.h | 11 +-- src/workerd/api/js-writable-stream-test.c++ | 85 ++++++++++++--------- 3 files changed, 62 insertions(+), 45 deletions(-) diff --git a/src/workerd/api/js-readable-stream.c++ b/src/workerd/api/js-readable-stream.c++ index 3a61f5a6b9b..6f7fae69ee9 100644 --- a/src/workerd/api/js-readable-stream.c++ +++ b/src/workerd/api/js-readable-stream.c++ @@ -381,8 +381,15 @@ jsg::Promise JsReadableStream::pipeTo( KJ_CASE_ONEOF(writable, jsg::Ref) { // Both ends are C++: delegate to ReadableStream::pipeTo, which preserves the exact // observable behavior (locked-end rejections and their texts) and internally selects - // the most efficient pump for the endpoint types. - return stream->pipeTo(js, writable.addRef(), kj::mv(options)); + // the most efficient pump for the endpoint types. The internal pipe keeps only a bare + // reference to the source (WritableStreamInternalController::PipeLocked) and the + // destination controller holds only a kj::Weak owner, so retain both + // ends across the pipe ourselves rather than making liveness the caller's + // responsibility. + // Note: there's no use for using JSG_VISITABLE_LAMBDA here since promise + // continuations are never actually visited for GC. + return stream->pipeTo(js, writable.addRef(), kj::mv(options)) + .then(js, [source = stream.addRef(), dest = writable.addRef()](jsg::Lock& js) {}); } KJ_CASE_ONEOF(obj, jsg::JsRef) { // TODO(streams-ts): TS/TS pipes go through the TS pipeTo hook; mixed-backend pipes diff --git a/src/workerd/api/js-readable-stream.h b/src/workerd/api/js-readable-stream.h index cddd983afd6..a1c380bd8aa 100644 --- a/src/workerd/api/js-readable-stream.h +++ b/src/workerd/api/js-readable-stream.h @@ -180,11 +180,12 @@ class JsReadableStream final { // pipe completes. Rejects (does not throw) if either end is already locked. By default the // destination is closed when this stream ends and aborted if it errors; see PipeToOptions. // - // The destination is borrowed, not consumed: the caller's handle remains valid and observes the - // stream's state as the pipe progresses. Unlike pumpTo(), this is a spec-level, isolate-bound - // pipe with no deferred-proxy support; when both ends are backed by the C++ implementation the - // pipe still uses the controllers' internal native fast paths. Preconditions: !isNull(), - // !destination.isNull(). + // Both the source and destination are self-retained for the duration of the pipe; the caller + // does not need to keep either alive after calling pipeTo(). The destination is borrowed, not + // consumed: the caller's handle remains valid and observes the stream's state as the pipe + // progresses. Unlike pumpTo(), this is a spec-level, isolate-bound pipe with no deferred-proxy + // support; when both ends are backed by the C++ implementation the pipe still uses the + // controllers' internal native fast paths. Preconditions: !isNull(), !destination.isNull(). jsg::Promise pipeTo( jsg::Lock& js, JsWritableStream& destination, PipeToOptions options = {}); diff --git a/src/workerd/api/js-writable-stream-test.c++ b/src/workerd/api/js-writable-stream-test.c++ index b778b4611e1..92ac6099792 100644 --- a/src/workerd/api/js-writable-stream-test.c++ +++ b/src/workerd/api/js-writable-stream-test.c++ @@ -101,9 +101,9 @@ KJ_TEST("JsWritableStream create wraps a native sink; forceClose ends it") { KJ_EXPECT(!stream.isLocked(js)); KJ_EXPECT(!stream.isClosedOrClosing(js)); - auto promise = stream.forceClose(js).then(js, - JSG_VISITABLE_LAMBDA((stream = kj::mv(stream)), (stream), - (jsg::Lock& js) { KJ_EXPECT(stream.isClosedOrClosing(js)); })); + auto promise = stream.forceClose(js).then(js, [stream = kj::mv(stream)](jsg::Lock& js) mutable { + KJ_EXPECT(stream.isClosedOrClosing(js)); + }); return env.context.awaitJs(js, kj::mv(promise)); }); KJ_EXPECT(state.ended); @@ -150,16 +150,16 @@ KJ_TEST("JsWritableStream flush rejects when a writer is held; forceFlush succee KJ_EXPECT(stream.isLocked(js)); auto promise = stream.flush(js) - .then(js, [](jsg::Lock& js) { + .then(js, [](jsg::Lock& js) mutable { KJ_FAIL_REQUIRE("expected flush() of a writer-locked stream to reject"); }, [](jsg::Lock& js, jsg::Value exception) { auto e = js.exceptionToKj(kj::mv(exception)); KJ_EXPECT( e.getDescription() == kj::str("jsg.TypeError: ", kWriterLockedError), e.getDescription()); - }).then(js, JSG_VISITABLE_LAMBDA((stream = kj::mv(stream)), (stream), (jsg::Lock& js) { - // forceFlush() bypasses the writer lock. - return stream.forceFlush(js); - })); + }).then(js, [stream = kj::mv(stream)](jsg::Lock& js) mutable { + // forceFlush() bypasses the writer lock. + return stream.forceFlush(js); + }); return env.context.awaitJs(js, kj::mv(promise)).attach(kj::mv(writer)); }); } @@ -224,13 +224,14 @@ KJ_TEST("JsWritableStream detach throws when a writer is held") { auto writer = ws->getWriter(js); JsWritableStream stream(kj::mv(ws)); - js.tryCatch([&] { + JSG_TRY(js) { stream.detach(js); KJ_FAIL_REQUIRE("expected detach() of a writer-locked stream to throw"); - }, [&](jsg::Value exception) { + } + JSG_CATCH(exception) { auto e = js.exceptionToKj(kj::mv(exception)); KJ_EXPECT(e.getDescription().contains(kWriterLockedError), e.getDescription()); - }); + }; }); } @@ -241,19 +242,18 @@ KJ_TEST("JsWritableStream detach of a closed stream throws") { auto& js = env.js; auto stream = JsWritableStream::create(js, env.context, state.makeSink(), kj::none); - auto promise = - stream.forceClose(js).then(js, - JSG_VISITABLE_LAMBDA((stream = kj::mv(stream)), (stream), (jsg::Lock& js) { - KJ_EXPECT(stream.isClosedOrClosing(js)); - js.tryCatch([&] { - stream.detach(js); - KJ_FAIL_REQUIRE("expected detach() of a closed stream to throw"); - }, [&](jsg::Value exception) { - auto e = js.exceptionToKj(kj::mv(exception)); - KJ_EXPECT(e.getDescription().contains("This WritableStream is closed."), - e.getDescription()); - }); - })); + auto promise = stream.forceClose(js).then(js, [stream = kj::mv(stream)](jsg::Lock& js) mutable { + KJ_EXPECT(stream.isClosedOrClosing(js)); + JSG_TRY(js) { + stream.detach(js); + KJ_FAIL_REQUIRE("expected detach() of a closed stream to throw"); + } + JSG_CATCH(exception) { + auto e = js.exceptionToKj(kj::mv(exception)); + KJ_EXPECT( + e.getDescription().contains("This WritableStream is closed."), e.getDescription()); + }; + }); return env.context.awaitJs(js, kj::mv(promise)); }); } @@ -270,9 +270,9 @@ KJ_TEST("JsWritableStream addRef shares the underlying stream") { KJ_EXPECT(!ref.isClosedOrClosing(js)); // Closing through the addRef closes the original: both wrap the same stream. - auto promise = ref.forceClose(js).then(js, - JSG_VISITABLE_LAMBDA((stream = kj::mv(stream)), (stream), - (jsg::Lock& js) { KJ_EXPECT(stream.isClosedOrClosing(js)); })); + auto promise = ref.forceClose(js).then(js, [stream = kj::mv(stream)](jsg::Lock& js) mutable { + KJ_EXPECT(stream.isClosedOrClosing(js)); + }); return env.context.awaitJs(js, kj::mv(promise)); }); KJ_EXPECT(state.ended); @@ -310,9 +310,11 @@ KJ_TEST("JsReadableStream pipeTo pipes into a JsWritableStream and closes it") { JsReadableStream source(js, kj::str(kData)); auto destination = JsWritableStream::create(js, env.context, state.makeSink(), kj::none); - // source and destination must outlive the pipe: capture them into the continuation so they - // are not destroyed when the lambda returns (the pipe's write loop still references them). - auto promise = source.pipeTo(js, destination).then(js, JSG_VISITABLE_LAMBDA((source = kj::mv(source), destination = kj::mv(destination)), (source, destination), (jsg::Lock& js){})); + // pipeTo() self-retains the source; the destination must still outlive the pipe because the + // pipe's write loop runs inside the writable controller. + auto promise = + source.pipeTo(js, destination).then(js, [destination = kj::mv(destination)](jsg::Lock& js) { + }); return env.context.awaitJs(js, kj::mv(promise)); }); KJ_EXPECT(state.written.asPtr() == kData.asBytes()); @@ -327,7 +329,8 @@ KJ_TEST("JsReadableStream pipeTo with preventClose leaves the destination open") JsReadableStream source(js, kj::str(kData)); auto destination = JsWritableStream::create(js, env.context, state.makeSink(), kj::none); - auto promise = source.pipeTo(js, destination, PipeToOptions{.preventClose = true}).then(js, JSG_VISITABLE_LAMBDA((source = kj::mv(source), destination = kj::mv(destination)), (source, destination), (jsg::Lock& js){})); + auto promise = source.pipeTo(js, destination, PipeToOptions{.preventClose = true}) + .then(js, [destination = kj::mv(destination)](jsg::Lock& js) {}); return env.context.awaitJs(js, kj::mv(promise)); }); KJ_EXPECT(state.written.asPtr() == kData.asBytes()); @@ -416,7 +419,11 @@ KJ_TEST("JsReadableStream pipeThrough composes with pipeTo") { JsReadableStream source(js, kj::str(kData)); auto destination = JsWritableStream::create(js, env.context, state.makeSink(), kj::none); auto piped = source.pipeThrough(js, makeIdentityPair(js)); - auto promise = piped.pipeTo(js, destination).then(js, JSG_VISITABLE_LAMBDA((source = kj::mv(source), piped = kj::mv(piped), destination = kj::mv(destination)), (source, piped, destination), (jsg::Lock& js){})); + // pipeTo() self-retains the piped source; pipeThrough() self-retains the original source + // via JSG_THIS; only the destination needs explicit keepalive. + auto promise = + piped.pipeTo(js, destination).then(js, [destination = kj::mv(destination)](jsg::Lock& js) { + }); return env.context.awaitJs(js, kj::mv(promise)); }); KJ_EXPECT(state.written.asPtr() == kData.asBytes()); @@ -432,13 +439,14 @@ KJ_TEST("JsReadableStream pipeThrough throws synchronously when the source is lo auto detached = source.detach(js); KJ_EXPECT(source.isLocked(js)); - js.tryCatch([&] { + JSG_TRY(js) { source.pipeThrough(js, makeIdentityPair(js)); KJ_FAIL_REQUIRE("expected pipeThrough() from a locked source to throw"); - }, [&](jsg::Value exception) { + } + JSG_CATCH(exception) { auto e = js.exceptionToKj(kj::mv(exception)); KJ_EXPECT(e.getDescription().contains("currently locked to a reader"), e.getDescription()); - }); + }; }); } @@ -457,15 +465,16 @@ KJ_TEST("JsReadableStream pipeThrough throws synchronously when the transform wr .writable = JsWritableStream(kj::mv(writable)), }; - js.tryCatch([&] { + JSG_TRY(js) { source.pipeThrough(js, kj::mv(pair)); KJ_FAIL_REQUIRE("expected pipeThrough() into a locked transform writable to throw"); - }, [&](jsg::Value exception) { + } + JSG_CATCH(exception) { auto e = js.exceptionToKj(kj::mv(exception)); // pipeThrough's synchronous destination-locked message carries the trailing period, // unlike pipeTo's rejection. KJ_EXPECT(e.getDescription().contains(kWriterLockedError), e.getDescription()); - }); + }; }); } From 13a2f097385776614ae787caa64333857e42c521 Mon Sep 17 00:00:00 2001 From: Nuno Pereira Date: Mon, 13 Jul 2026 18:28:16 +0100 Subject: [PATCH 069/101] WOR-000: fix types for rollback steps with delay functions --- types/defines/rpc.d.ts | 33 +++++-- .../experimental/index.d.ts | 33 +++++-- .../generated-snapshot/experimental/index.ts | 33 +++++-- types/generated-snapshot/index.d.ts | 33 +++++-- types/generated-snapshot/index.ts | 33 +++++-- types/test/types/rpc.ts | 91 +++++++++++++++++++ 6 files changed, 211 insertions(+), 45 deletions(-) diff --git a/types/defines/rpc.d.ts b/types/defines/rpc.d.ts index f9f12e13d36..fd1409e0aa6 100644 --- a/types/defines/rpc.d.ts +++ b/types/defines/rpc.d.ts @@ -386,20 +386,32 @@ declare namespace CloudflareWorkersModule { }; }; - export type WorkflowRollbackContext = { - ctx: WorkflowStepContext; + // The rollback handler receives the step context, so it mirrors the same + // delay discriminant as the step callback: when the step was configured with + // a dynamic delay function the resolved `config.retries.delay` is omitted, + // otherwise it is present. `Delay` is threaded from the `WorkflowStep.do` + // overload that matched the step config. + export type WorkflowRollbackContext< + T = unknown, + Delay = WorkflowDelayDuration | number, + > = { + ctx: WorkflowStepContext; error: Error; output: T | undefined; /** @deprecated Use `ctx.step.name` and `ctx.step.count` instead. */ stepName: string; }; - export type WorkflowRollbackHandler = ( - ctx: WorkflowRollbackContext - ) => Promise; + export type WorkflowRollbackHandler< + T = unknown, + Delay = WorkflowDelayDuration | number, + > = (ctx: WorkflowRollbackContext) => Promise; - export type WorkflowStepRollbackOptions = { - rollback: WorkflowRollbackHandler; + export type WorkflowStepRollbackOptions< + T = unknown, + Delay = WorkflowDelayDuration | number, + > = { + rollback: WorkflowRollbackHandler; rollbackConfig?: WorkflowStepRollbackConfig; }; @@ -420,7 +432,7 @@ declare namespace CloudflareWorkersModule { name: string, config: WorkflowStepConfigWithDelayFunction, callback: (ctx: WorkflowStepContext) => Promise, - rollbackOptions?: WorkflowStepRollbackOptions + rollbackOptions?: WorkflowStepRollbackOptions ): Promise; do>( name: string, @@ -428,7 +440,10 @@ declare namespace CloudflareWorkersModule { callback: ( ctx: WorkflowStepContext ) => Promise, - rollbackOptions?: WorkflowStepRollbackOptions + rollbackOptions?: WorkflowStepRollbackOptions< + T, + WorkflowDelayDuration | number + > ): Promise; do>( name: string, diff --git a/types/generated-snapshot/experimental/index.d.ts b/types/generated-snapshot/experimental/index.d.ts index 6af00edf70c..48ffdf80302 100755 --- a/types/generated-snapshot/experimental/index.d.ts +++ b/types/generated-snapshot/experimental/index.d.ts @@ -15488,18 +15488,30 @@ declare namespace CloudflareWorkersModule { sensitive?: WorkflowStepSensitivity; }; }; - export type WorkflowRollbackContext = { - ctx: WorkflowStepContext; + // The rollback handler receives the step context, so it mirrors the same + // delay discriminant as the step callback: when the step was configured with + // a dynamic delay function the resolved `config.retries.delay` is omitted, + // otherwise it is present. `Delay` is threaded from the `WorkflowStep.do` + // overload that matched the step config. + export type WorkflowRollbackContext< + T = unknown, + Delay = WorkflowDelayDuration | number, + > = { + ctx: WorkflowStepContext; error: Error; output: T | undefined; /** @deprecated Use `ctx.step.name` and `ctx.step.count` instead. */ stepName: string; }; - export type WorkflowRollbackHandler = ( - ctx: WorkflowRollbackContext, - ) => Promise; - export type WorkflowStepRollbackOptions = { - rollback: WorkflowRollbackHandler; + export type WorkflowRollbackHandler< + T = unknown, + Delay = WorkflowDelayDuration | number, + > = (ctx: WorkflowRollbackContext) => Promise; + export type WorkflowStepRollbackOptions< + T = unknown, + Delay = WorkflowDelayDuration | number, + > = { + rollback: WorkflowRollbackHandler; rollbackConfig?: WorkflowStepRollbackConfig; }; export abstract class WorkflowStep { @@ -15519,7 +15531,7 @@ declare namespace CloudflareWorkersModule { name: string, config: WorkflowStepConfigWithDelayFunction, callback: (ctx: WorkflowStepContext) => Promise, - rollbackOptions?: WorkflowStepRollbackOptions, + rollbackOptions?: WorkflowStepRollbackOptions, ): Promise; do>( name: string, @@ -15527,7 +15539,10 @@ declare namespace CloudflareWorkersModule { callback: ( ctx: WorkflowStepContext, ) => Promise, - rollbackOptions?: WorkflowStepRollbackOptions, + rollbackOptions?: WorkflowStepRollbackOptions< + T, + WorkflowDelayDuration | number + >, ): Promise; do>( name: string, diff --git a/types/generated-snapshot/experimental/index.ts b/types/generated-snapshot/experimental/index.ts index 9d15cda8da2..04be8789498 100755 --- a/types/generated-snapshot/experimental/index.ts +++ b/types/generated-snapshot/experimental/index.ts @@ -15466,18 +15466,30 @@ export declare namespace CloudflareWorkersModule { sensitive?: WorkflowStepSensitivity; }; }; - export type WorkflowRollbackContext = { - ctx: WorkflowStepContext; + // The rollback handler receives the step context, so it mirrors the same + // delay discriminant as the step callback: when the step was configured with + // a dynamic delay function the resolved `config.retries.delay` is omitted, + // otherwise it is present. `Delay` is threaded from the `WorkflowStep.do` + // overload that matched the step config. + export type WorkflowRollbackContext< + T = unknown, + Delay = WorkflowDelayDuration | number, + > = { + ctx: WorkflowStepContext; error: Error; output: T | undefined; /** @deprecated Use `ctx.step.name` and `ctx.step.count` instead. */ stepName: string; }; - export type WorkflowRollbackHandler = ( - ctx: WorkflowRollbackContext, - ) => Promise; - export type WorkflowStepRollbackOptions = { - rollback: WorkflowRollbackHandler; + export type WorkflowRollbackHandler< + T = unknown, + Delay = WorkflowDelayDuration | number, + > = (ctx: WorkflowRollbackContext) => Promise; + export type WorkflowStepRollbackOptions< + T = unknown, + Delay = WorkflowDelayDuration | number, + > = { + rollback: WorkflowRollbackHandler; rollbackConfig?: WorkflowStepRollbackConfig; }; export abstract class WorkflowStep { @@ -15497,7 +15509,7 @@ export declare namespace CloudflareWorkersModule { name: string, config: WorkflowStepConfigWithDelayFunction, callback: (ctx: WorkflowStepContext) => Promise, - rollbackOptions?: WorkflowStepRollbackOptions, + rollbackOptions?: WorkflowStepRollbackOptions, ): Promise; do>( name: string, @@ -15505,7 +15517,10 @@ export declare namespace CloudflareWorkersModule { callback: ( ctx: WorkflowStepContext, ) => Promise, - rollbackOptions?: WorkflowStepRollbackOptions, + rollbackOptions?: WorkflowStepRollbackOptions< + T, + WorkflowDelayDuration | number + >, ): Promise; do>( name: string, diff --git a/types/generated-snapshot/index.d.ts b/types/generated-snapshot/index.d.ts index 4698edbdd60..0d1dba90e0f 100755 --- a/types/generated-snapshot/index.d.ts +++ b/types/generated-snapshot/index.d.ts @@ -14840,18 +14840,30 @@ declare namespace CloudflareWorkersModule { sensitive?: WorkflowStepSensitivity; }; }; - export type WorkflowRollbackContext = { - ctx: WorkflowStepContext; + // The rollback handler receives the step context, so it mirrors the same + // delay discriminant as the step callback: when the step was configured with + // a dynamic delay function the resolved `config.retries.delay` is omitted, + // otherwise it is present. `Delay` is threaded from the `WorkflowStep.do` + // overload that matched the step config. + export type WorkflowRollbackContext< + T = unknown, + Delay = WorkflowDelayDuration | number, + > = { + ctx: WorkflowStepContext; error: Error; output: T | undefined; /** @deprecated Use `ctx.step.name` and `ctx.step.count` instead. */ stepName: string; }; - export type WorkflowRollbackHandler = ( - ctx: WorkflowRollbackContext, - ) => Promise; - export type WorkflowStepRollbackOptions = { - rollback: WorkflowRollbackHandler; + export type WorkflowRollbackHandler< + T = unknown, + Delay = WorkflowDelayDuration | number, + > = (ctx: WorkflowRollbackContext) => Promise; + export type WorkflowStepRollbackOptions< + T = unknown, + Delay = WorkflowDelayDuration | number, + > = { + rollback: WorkflowRollbackHandler; rollbackConfig?: WorkflowStepRollbackConfig; }; export abstract class WorkflowStep { @@ -14871,7 +14883,7 @@ declare namespace CloudflareWorkersModule { name: string, config: WorkflowStepConfigWithDelayFunction, callback: (ctx: WorkflowStepContext) => Promise, - rollbackOptions?: WorkflowStepRollbackOptions, + rollbackOptions?: WorkflowStepRollbackOptions, ): Promise; do>( name: string, @@ -14879,7 +14891,10 @@ declare namespace CloudflareWorkersModule { callback: ( ctx: WorkflowStepContext, ) => Promise, - rollbackOptions?: WorkflowStepRollbackOptions, + rollbackOptions?: WorkflowStepRollbackOptions< + T, + WorkflowDelayDuration | number + >, ): Promise; do>( name: string, diff --git a/types/generated-snapshot/index.ts b/types/generated-snapshot/index.ts index 8e9d132614d..58c9b015c3e 100755 --- a/types/generated-snapshot/index.ts +++ b/types/generated-snapshot/index.ts @@ -14818,18 +14818,30 @@ export declare namespace CloudflareWorkersModule { sensitive?: WorkflowStepSensitivity; }; }; - export type WorkflowRollbackContext = { - ctx: WorkflowStepContext; + // The rollback handler receives the step context, so it mirrors the same + // delay discriminant as the step callback: when the step was configured with + // a dynamic delay function the resolved `config.retries.delay` is omitted, + // otherwise it is present. `Delay` is threaded from the `WorkflowStep.do` + // overload that matched the step config. + export type WorkflowRollbackContext< + T = unknown, + Delay = WorkflowDelayDuration | number, + > = { + ctx: WorkflowStepContext; error: Error; output: T | undefined; /** @deprecated Use `ctx.step.name` and `ctx.step.count` instead. */ stepName: string; }; - export type WorkflowRollbackHandler = ( - ctx: WorkflowRollbackContext, - ) => Promise; - export type WorkflowStepRollbackOptions = { - rollback: WorkflowRollbackHandler; + export type WorkflowRollbackHandler< + T = unknown, + Delay = WorkflowDelayDuration | number, + > = (ctx: WorkflowRollbackContext) => Promise; + export type WorkflowStepRollbackOptions< + T = unknown, + Delay = WorkflowDelayDuration | number, + > = { + rollback: WorkflowRollbackHandler; rollbackConfig?: WorkflowStepRollbackConfig; }; export abstract class WorkflowStep { @@ -14849,7 +14861,7 @@ export declare namespace CloudflareWorkersModule { name: string, config: WorkflowStepConfigWithDelayFunction, callback: (ctx: WorkflowStepContext) => Promise, - rollbackOptions?: WorkflowStepRollbackOptions, + rollbackOptions?: WorkflowStepRollbackOptions, ): Promise; do>( name: string, @@ -14857,7 +14869,10 @@ export declare namespace CloudflareWorkersModule { callback: ( ctx: WorkflowStepContext, ) => Promise, - rollbackOptions?: WorkflowStepRollbackOptions, + rollbackOptions?: WorkflowStepRollbackOptions< + T, + WorkflowDelayDuration | number + >, ): Promise; do>( name: string, diff --git a/types/test/types/rpc.ts b/types/test/types/rpc.ts index e911fedd9bb..197460a6bfc 100644 --- a/types/test/types/rpc.ts +++ b/types/test/types/rpc.ts @@ -1203,6 +1203,97 @@ void delayReturningDuration; const asyncDelay: WorkflowDelayFunction = async () => 5; void asyncDelay; +// --------------------------------------------------------------------------- +// Rollback context delay narrowing. +// +// The rollback handler receives the step context, so it mirrors the same delay +// discriminant as the step callback: a static delay surfaces the resolved +// `config.retries.delay`, while a delay function omits it. This holds under an +// explicit return-type argument too. `rollbackConfig` continues to accept a +// delay function of its own. +// --------------------------------------------------------------------------- + +// A static-delay step exposes the resolved delay in the rollback context. +workflowStep.do( + 'rollback ctx static delay present', + {retries: {limit: 1, delay: 0}}, + async (): Promise => 'ok', + { + rollback: async (rollbackCtx) => { + expectTypeOf(rollbackCtx.ctx.config.retries?.delay).toEqualTypeOf< + WorkflowDelayDuration | number | undefined + >(); + }, + } +); + +// A delay-function step omits the resolved delay in the rollback context. +workflowStep.do( + 'rollback ctx dynamic delay absent', + {retries: {limit: 1, delay: (): WorkflowDelayDuration => '1 minute'}}, + async (): Promise => 'ok', + { + rollback: async (rollbackCtx) => { + expectTypeOf(rollbackCtx.ctx.config.retries).toEqualTypeOf< + {limit: number; backoff?: WorkflowBackoff} | undefined + >(); + // @ts-expect-error delay is absent when the step used a delay function + rollbackCtx.ctx.config.retries?.delay; + }, + } +); + +// The same omission holds under an explicit return-type argument. +workflowStep.do( + 'rollback ctx dynamic delay absent explicit', + {retries: {limit: 1, delay: (): WorkflowDelayDuration => '1 minute'}}, + async () => 'ok', + { + rollback: async (rollbackCtx) => { + expectTypeOf(rollbackCtx.ctx.config.retries).toEqualTypeOf< + {limit: number; backoff?: WorkflowBackoff} | undefined + >(); + // @ts-expect-error delay is absent under an explicit type argument too + rollbackCtx.ctx.config.retries?.delay; + }, + } +); + +// An async delay-function step also omits the resolved delay. +workflowStep.do( + 'rollback ctx async dynamic delay absent', + {retries: {limit: 1, delay: async (): Promise => 5}}, + async (): Promise => 'ok', + { + rollback: async (rollbackCtx) => { + expectTypeOf(rollbackCtx.ctx.config.retries).toEqualTypeOf< + {limit: number; backoff?: WorkflowBackoff} | undefined + >(); + }, + } +); + +// The no-config (2-arg) form keeps the default (static) rollback context. +workflowStep.do('rollback ctx default delay present', async (): Promise => 'ok', { + rollback: async (rollbackCtx) => { + expectTypeOf(rollbackCtx.ctx.config.retries?.delay).toEqualTypeOf< + WorkflowDelayDuration | number | undefined + >(); + }, +}); + +// `rollbackConfig` accepts an async delay function of its own. +workflowStep.do('rollback config async delay function', async (): Promise => 'ok', { + rollback: async () => {}, + rollbackConfig: {retries: {limit: 0, delay: async () => 5}}, +}); + +// @ts-expect-error a rollbackConfig delay function must return a delay duration +workflowStep.do('rollback config delay wrong return', async () => 'ok', { + rollback: async () => {}, + rollbackConfig: {retries: {limit: 0, delay: (): boolean => true}}, +}); + declare const cronSchedule: WorkflowCronSchedule; expectTypeOf(cronSchedule.cron).toEqualTypeOf(); expectTypeOf(cronSchedule.scheduledTime).toEqualTypeOf(); From 4f67ee711ebeb21ab30c8a7ebe43f87141030914 Mon Sep 17 00:00:00 2001 From: Mike Aizatsky Date: Mon, 13 Jul 2026 11:07:10 -0700 Subject: [PATCH 070/101] hardening.md --- docs/hardening.md | 48 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 docs/hardening.md diff --git a/docs/hardening.md b/docs/hardening.md new file mode 100644 index 00000000000..2caae300d6c --- /dev/null +++ b/docs/hardening.md @@ -0,0 +1,48 @@ +# Hardening + +Hardening process replaces unsafe C++ constructs with others that either completely eliminate +undefined behaviors or replace them by runtime checks. + +## Runtime Checks + +The set of `KJ_*` macros should be used to assert correct runtime behavior. + +## Bounds/Null checking + +We have `kj_enable_irequire` flag enabled in all configurations, which enables KJ bounds and null +checking in built-in data structures. There is no need to duplicate such checks in user code +unless more debug information is required. + +## Raw References + +Raw reference types are incredibly dangerous, because they don't offer any lifetime protection. + +They are especially dangerous when used as data fields, or in asynchronous context (coroutines or +lambda captures). + +The alternative is to use: + +- `kj::Ptr` for references that never outlive the object and `kj::Weak` when you can't guarantee + that. These smart pointers are obtained either from `kj::Pin` for inline storage or inheriting and + exposing `kj::PtrTarget` methods for heap-allocated objects. + +- `kj::Rc`/`kj::WeakRc` for objects that are (or should be) reference counted. + +When using these, all references in all call paths ending on smart pointers need to be adjusted to +use them as well. + +It could still be acceptable to use references as parameters when: + +- function is synchronous +- its scope is very small and clear +- it _clearly_ can't result in indirect deallocation through the use of other objects or methods + +When in doubt - use smart pointers. + +## Raw Pointers + +All Raw References advice applies to raw pointers as well. If `null` is a valid state, then weak +pointers should be used rather than wrapping smart pointers into `kj::Maybe`. + + + From db51bd41dc09916f1d028e6b41f7217636bdac12 Mon Sep 17 00:00:00 2001 From: Mike Aizatskyi Date: Mon, 13 Jul 2026 18:40:02 +0000 Subject: [PATCH 071/101] Edit AGENTS.md --- AGENTS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/AGENTS.md b/AGENTS.md index 281bfaca64a..181ba544b67 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -111,7 +111,7 @@ Key elements: `modules` (embed JS/TS files), `compatibilityFlags`, `bindings` (s ### Dependencies -- **Cap'n Proto source code** available in `external/capnp-cpp` - contains KJ C++ base library and +- **Cap'n Proto source code** available in `external/+http+capnp-cpp/` - it contains KJ C++ base library and capnproto RPC library. Consult it for all questions about `kj/` and `capnproto/` includes and `kj::` and `capnp::` namespaces. From 85a82bfc21bbf1d8c1b03986e2a42e6e38aa0c27 Mon Sep 17 00:00:00 2001 From: Mike Aizatsky Date: Mon, 13 Jul 2026 11:45:34 -0700 Subject: [PATCH 072/101] harden ReadableStreamSource reference --- src/workerd/api/streams/common.h | 4 ++-- src/workerd/api/streams/internal.c++ | 27 +++++++++++++++------------ src/workerd/api/system-streams.c++ | 6 +++--- 3 files changed, 20 insertions(+), 17 deletions(-) diff --git a/src/workerd/api/streams/common.h b/src/workerd/api/streams/common.h index 062b99bc9c6..50ea0875baf 100644 --- a/src/workerd/api/streams/common.h +++ b/src/workerd/api/streams/common.h @@ -236,7 +236,7 @@ class WritableStreamSink { // Must call to flush and finish the stream. virtual kj::Maybe>> tryPumpFrom( - ReadableStreamSource& input, bool end); + kj::Ptr input, bool end); virtual void abort(kj::Exception reason) = 0; // TODO(conform): abort() should return a promise after which closed fulfillers should be @@ -252,7 +252,7 @@ class WritableStreamSink { } }; -class ReadableStreamSource { +class ReadableStreamSource: public kj::PtrTarget { public: virtual kj::Promise tryRead(void* buffer, size_t minBytes, size_t maxBytes) = 0; diff --git a/src/workerd/api/streams/internal.c++ b/src/workerd/api/streams/internal.c++ index 7d50a8ab55a..b5fe2ed206d 100644 --- a/src/workerd/api/streams/internal.c++ +++ b/src/workerd/api/streams/internal.c++ @@ -35,11 +35,12 @@ namespace { kj::str(JSG_EXCEPTION(TypeError) ": ", message))); } -kj::Promise pumpTo(ReadableStreamSource& input, WritableStreamSink& output, bool end) { +kj::Promise pumpTo( + kj::Ptr input, WritableStreamSink& output, bool end) { kj::byte buffer[65536]{}; while (true) { - auto amount = co_await input.tryRead(buffer, 1, kj::size(buffer)); + auto amount = co_await input->tryRead(buffer, 1, kj::size(buffer)); if (amount == 0) { if (end) { @@ -55,9 +56,11 @@ kj::Promise pumpTo(ReadableStreamSource& input, WritableStreamSink& output // Modified from AllReader in kj/async-io.c++. class AllReader final { public: - explicit AllReader(ReadableStreamSource& input, uint64_t limit): input(input), limit(limit) { + explicit AllReader(kj::Ptr input, uint64_t limit) + : input(kj::mv(input)), + limit(limit) { JSG_REQUIRE(limit > 0, TypeError, "Memory limit exceeded before EOF."); - KJ_IF_SOME(length, input.tryGetLength(StreamEncoding::IDENTITY)) { + KJ_IF_SOME(length, this->input->tryGetLength(StreamEncoding::IDENTITY)) { // Oh hey, we might be able to bail early. JSG_REQUIRE(length < limit, TypeError, "Memory limit would be exceeded before EOF."); } @@ -75,7 +78,7 @@ class AllReader final { } private: - ReadableStreamSource& input; + kj::Ptr input; uint64_t limit; template @@ -137,7 +140,7 @@ class AllReader final { // optimize the loop here by setting the value specifically so we are only // allocating at most twice. But, to be safe, let's enforce an upper bound on each // allocation even if we do know the total. - kj::Maybe maybeLength = input.tryGetLength(StreamEncoding::IDENTITY); + kj::Maybe maybeLength = input->tryGetLength(StreamEncoding::IDENTITY); // The amountToRead is the regular allocation size we'll use right up until we've // read the number of expected bytes (if known). This number is calculated as the @@ -157,7 +160,7 @@ class AllReader final { // Note that we're passing amountToRead as the *minBytes* here so the tryRead should // attempt to fill the entire buffer. If it doesn't, the implication is that we read // everything. - uint64_t amount = co_await input.tryRead(bytes.begin(), amountToRead, amountToRead); + uint64_t amount = co_await input->tryRead(bytes.begin(), amountToRead, amountToRead); KJ_DASSERT(amount <= amountToRead); runningTotal += amount; @@ -375,13 +378,13 @@ class TeeBranch final: public ReadableStreamSource { kj::Promise> ReadableStreamSource::pumpTo( WritableStreamSink& output, bool end) { - KJ_IF_SOME(p, output.tryPumpFrom(*this, end)) { + KJ_IF_SOME(p, output.tryPumpFrom(addPtrToThis(), end)) { return kj::mv(p); } // Non-optimized pumpTo() is presumed to require the IoContext to remain live, so don't do // anything in the deferred proxy part. - return addNoopDeferredProxy(api::pumpTo(*this, output, end)); + return addNoopDeferredProxy(api::pumpTo(addPtrToThis(), output, end)); } kj::Maybe ReadableStreamSource::tryGetLength(StreamEncoding encoding) { @@ -390,7 +393,7 @@ kj::Maybe ReadableStreamSource::tryGetLength(StreamEncoding encoding) kj::Promise> ReadableStreamSource::readAllBytes(uint64_t limit) { try { - AllReader allReader(*this, limit); + AllReader allReader(addPtrToThis(), limit); co_return co_await allReader.readAllBytes(); } catch (...) { // TODO(soon): Temporary logging. @@ -405,7 +408,7 @@ kj::Promise> ReadableStreamSource::readAllBytes(uint64_t limit) kj::Promise ReadableStreamSource::readAllText( uint64_t limit, ReadAllTextOption option) { try { - AllReader allReader(*this, limit); + AllReader allReader(addPtrToThis(), limit); co_return co_await allReader.readAllText(option); } catch (...) { // TODO(soon): Temporary logging. @@ -424,7 +427,7 @@ kj::Maybe ReadableStreamSource::tryTee(uint64_t limit } kj::Maybe>> WritableStreamSink::tryPumpFrom( - ReadableStreamSource& input, bool end) { + kj::Ptr input, bool end) { return kj::none; } diff --git a/src/workerd/api/system-streams.c++ b/src/workerd/api/system-streams.c++ index 7702be2241a..e2f1b3661bb 100644 --- a/src/workerd/api/system-streams.c++ +++ b/src/workerd/api/system-streams.c++ @@ -166,7 +166,7 @@ class EncodedAsyncOutputStream final: public WritableStreamSink { kj::Promise write(kj::ArrayPtr> pieces) override; kj::Maybe>> tryPumpFrom( - ReadableStreamSource& input, bool end) override; + kj::Ptr input, bool end) override; kj::Promise end() override; @@ -224,7 +224,7 @@ kj::Promise EncodedAsyncOutputStream::write( } kj::Maybe>> EncodedAsyncOutputStream::tryPumpFrom( - ReadableStreamSource& input, bool end) { + kj::Ptr input, bool end) { // If this output stream has already been ended, then there's nothing more to // pump into it, just return an immediately resolved promise. Alternatively @@ -233,7 +233,7 @@ kj::Maybe>> EncodedAsyncOutputStream::tryPumpFro return kj::Promise>(DeferredProxy{kj::READY_NOW}); } - KJ_IF_SOME(nativeInput, kj::dynamicDowncastIfAvailable(input)) { + KJ_IF_SOME(nativeInput, kj::dynamicDowncastIfAvailable(*input.get())) { // We can avoid putting our inner streams into identity encoding if the input and output both // have the same encoding. Since ReadableStreamSource/WritableStreamSink always pump everything // (there is no `amount` parameter like in the KJ equivalents), we can assume that we will From d201678d2c35568334479f75d201969e55121aac Mon Sep 17 00:00:00 2001 From: Dan Lapid Date: Mon, 13 Jul 2026 19:33:17 +0000 Subject: [PATCH 073/101] Fix workerd Date.now() to more closely mimic production behavior. In production, Date.now() returns 0 when called outside of a request handler, but workerd was returning clock time instead. This caused subtle bugs in libraries that precompute a time origin at module load (see cloudflare/workerd#6839). --- src/workerd/server/BUILD.bazel | 1 + src/workerd/server/v8-platform-impl.c++ | 4 ++-- src/workerd/server/v8-platform-impl.h | 4 +++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/workerd/server/BUILD.bazel b/src/workerd/server/BUILD.bazel index ec5d50dbd54..7d2a945a0a9 100644 --- a/src/workerd/server/BUILD.bazel +++ b/src/workerd/server/BUILD.bazel @@ -319,6 +319,7 @@ wd_cc_library( "v8-platform-impl.h", ], deps = [ + "//src/workerd/io", "//src/workerd/jsg", "@capnp-cpp//src/kj", ], diff --git a/src/workerd/server/v8-platform-impl.c++ b/src/workerd/server/v8-platform-impl.c++ index 6ce30e471a0..302ec7070e0 100644 --- a/src/workerd/server/v8-platform-impl.c++ +++ b/src/workerd/server/v8-platform-impl.c++ @@ -4,12 +4,12 @@ #include "v8-platform-impl.h" -#include +#include namespace workerd::server { double WorkerdPlatform::CurrentClockTimeMillis() noexcept { - return (kj::systemPreciseCalendarClock().now() - kj::UNIX_EPOCH) / kj::MILLISECONDS; + return dateNow(); } } // namespace workerd::server diff --git a/src/workerd/server/v8-platform-impl.h b/src/workerd/server/v8-platform-impl.h index d4259b107be..09ee81ac34b 100644 --- a/src/workerd/server/v8-platform-impl.h +++ b/src/workerd/server/v8-platform-impl.h @@ -68,7 +68,9 @@ class WorkerdPlatform final: public v8::Platform { return inner.MonotonicallyIncreasingTime(); } - // Overridden to return KJ time + // Overridden to defer to `workerd::dateNow()`, which returns the current IoContext's time + // when inside a request handler and 0 otherwise. This matches the behavior of production + // Cloudflare Workers, where `Date.now()` in top-level (module-scope) code returns 0. double CurrentClockTimeMillis() noexcept override; v8::TracingController* GetTracingController() noexcept override { From addb54a3fecc2ef44c63f92203a8f67a547b406f Mon Sep 17 00:00:00 2001 From: Dan Lapid Date: Mon, 13 Jul 2026 19:37:23 +0000 Subject: [PATCH 074/101] Just format --- src/cloudflare/internal/d1-api.ts | 36 ++++++++++--------- .../test/d1/d1-api-instrumentation-test.js | 8 ++--- .../internal/test/d1/d1-api-test.js | 5 ++- src/cloudflare/internal/test/d1/d1-mock.js | 10 ++++-- src/cloudflare/internal/workflows-api.ts | 5 ++- src/node/internal/internal_dns.ts | 4 ++- src/workerd/api/tests/js-rpc-test.js | 10 +++--- .../api/tests/new-module-registry-test.js | 5 ++- .../api/tests/ts-identity-streams-test.js | 2 +- src/workerd/api/worker-rpc.c++ | 9 +++-- src/workerd/io/worker-entrypoint.c++ | 3 +- src/workerd/jsg/modules-new.c++ | 3 +- src/workerd/jsg/util.c++ | 4 +-- src/workerd/jsg/util.h | 3 +- .../server/tests/container-client/test.js | 9 +++-- 15 files changed, 69 insertions(+), 47 deletions(-) diff --git a/src/cloudflare/internal/d1-api.ts b/src/cloudflare/internal/d1-api.ts index 0b195310c66..0c672d103eb 100644 --- a/src/cloudflare/internal/d1-api.ts +++ b/src/cloudflare/internal/d1-api.ts @@ -295,23 +295,25 @@ class D1DatabaseSession { this.getBookmark() ?? undefined ); - const exec = (d1BindingJsrpc - ? await this._queryOrThrow( - { - queries: statements.map((s) => ({ - sql: s.statement, - params: s.params, - })), - }, - span - ) - : await this._sendOrThrow( - '/query', - statements.map((s) => s.statement), - statements.map((s) => s.params), - 'ROWS_AND_COLUMNS', - span - )) as D1UpstreamSuccess[]; + const exec = ( + d1BindingJsrpc + ? await this._queryOrThrow( + { + queries: statements.map((s) => ({ + sql: s.statement, + params: s.params, + })), + }, + span + ) + : await this._sendOrThrow( + '/query', + statements.map((s) => s.statement), + statements.map((s) => s.params), + 'ROWS_AND_COLUMNS', + span + ) + ) as D1UpstreamSuccess[]; span.setAttribute( 'cloudflare.d1.response.bookmark', diff --git a/src/cloudflare/internal/test/d1/d1-api-instrumentation-test.js b/src/cloudflare/internal/test/d1/d1-api-instrumentation-test.js index 1cda894f8c0..6a7671cf9c1 100644 --- a/src/cloudflare/internal/test/d1/d1-api-instrumentation-test.js +++ b/src/cloudflare/internal/test/d1/d1-api-instrumentation-test.js @@ -1087,8 +1087,8 @@ const allExpectedSpans = [ ]; const expectedSpans = d1BindingJsrpc - // The JSRPC binding path does not issue internal HTTP requests to the D1 - // service, so it should not emit the transport-level fetch spans that the - // legacy path produces. The public D1 operation spans are still asserted. - ? allExpectedSpans.filter((span) => span.name !== 'fetch') + ? // The JSRPC binding path does not issue internal HTTP requests to the D1 + // service, so it should not emit the transport-level fetch spans that the + // legacy path produces. The public D1 operation spans are still asserted. + allExpectedSpans.filter((span) => span.name !== 'fetch') : allExpectedSpans; diff --git a/src/cloudflare/internal/test/d1/d1-api-test.js b/src/cloudflare/internal/test/d1/d1-api-test.js index c71650e6b31..1e7c3831ae8 100644 --- a/src/cloudflare/internal/test/d1/d1-api-test.js +++ b/src/cloudflare/internal/test/d1/d1-api-test.js @@ -46,6 +46,9 @@ export const testDirectQuery = { }, }); assert.equal(typeof response.results.bookmark, 'string'); - assert.equal(typeof response.results.queryResults[0].meta.duration, 'number'); + assert.equal( + typeof response.results.queryResults[0].meta.duration, + 'number' + ); }, }; diff --git a/src/cloudflare/internal/test/d1/d1-mock.js b/src/cloudflare/internal/test/d1/d1-mock.js index a97c3146515..9d8de2714b9 100644 --- a/src/cloudflare/internal/test/d1/d1-mock.js +++ b/src/cloudflare/internal/test/d1/d1-mock.js @@ -29,7 +29,10 @@ export class D1MockDO extends DurableObject { return this.runQuery(query, resultsFormat); } catch (e) { // Reproduce the production behavior by catching any error and returning a V4Failure - return { success: false, error: String(e instanceof Error ? e.message : e) }; + return { + success: false, + error: String(e instanceof Error ? e.message : e), + }; } }; return Response.json( @@ -48,7 +51,10 @@ export class D1MockDO extends DurableObject { return this.runQuery(query, 'ROWS_AND_COLUMNS'); } catch (e) { // Reproduce the production behavior by catching any error and returning a V4Failure - return { success: false, error: String(e instanceof Error ? e.message : e) }; + return { + success: false, + error: String(e instanceof Error ? e.message : e), + }; } }); const failure = results.find((result) => !result.success); diff --git a/src/cloudflare/internal/workflows-api.ts b/src/cloudflare/internal/workflows-api.ts index 2c3109885f1..af7d9eda789 100644 --- a/src/cloudflare/internal/workflows-api.ts +++ b/src/cloudflare/internal/workflows-api.ts @@ -22,7 +22,10 @@ interface Fetcher { pause(id: string): Promise; resume(id: string): Promise; - terminate(id: string, options?: WorkflowInstanceTerminateOptions): Promise; + terminate( + id: string, + options?: WorkflowInstanceTerminateOptions + ): Promise; restart(id: string, options?: WorkflowInstanceRestartOptions): Promise; status(id: string): Promise; sendEvent( diff --git a/src/node/internal/internal_dns.ts b/src/node/internal/internal_dns.ts index a856dba4ea5..9672489389d 100644 --- a/src/node/internal/internal_dns.ts +++ b/src/node/internal/internal_dns.ts @@ -327,7 +327,9 @@ export function resolve4( const overrideIp = getMagicHostOverride(name); if (overrideIp != null) { - return Promise.resolve([ttl ? { ttl: 0, address: overrideIp } : overrideIp]); + return Promise.resolve([ + ttl ? { ttl: 0, address: overrideIp } : overrideIp, + ]); } // Validation errors needs to be sync. diff --git a/src/workerd/api/tests/js-rpc-test.js b/src/workerd/api/tests/js-rpc-test.js index 6d01fa145d0..d97d57699a3 100644 --- a/src/workerd/api/tests/js-rpc-test.js +++ b/src/workerd/api/tests/js-rpc-test.js @@ -575,7 +575,7 @@ export class MyActor extends DurableObject { } async throwingMethod() { - throw new Error("ACTOR METHOD THREW"); + throw new Error('ACTOR METHOD THREW'); } async doCallbackBlockingConcurrency() { @@ -1825,19 +1825,19 @@ export let testExceptionProperties = { export let testDurableObjectExceptionProperties = { async test(controller, env, ctx) { - let id = env.MyActor.idFromName("exception-properties"); + let id = env.MyActor.idFromName('exception-properties'); try { await env.MyActor.get(id).throwingMethod(); - assert.fail("expected actor RPC to throw"); + assert.fail('expected actor RPC to throw'); } catch (e) { assert.strictEqual(e.remote, true); - assert.strictEqual(e.message, "ACTOR METHOD THREW"); + assert.strictEqual(e.message, 'ACTOR METHOD THREW'); assert.strictEqual(e.durableObjectId, id.toString()); } try { await env.MyService.throwingMethod(); - assert.fail("expected service RPC to throw"); + assert.fail('expected service RPC to throw'); } catch (e) { assert.strictEqual(e.remote, true); assert.strictEqual(e.durableObjectId, undefined); diff --git a/src/workerd/api/tests/new-module-registry-test.js b/src/workerd/api/tests/new-module-registry-test.js index 9d2191f1cd6..2d464b91c88 100644 --- a/src/workerd/api/tests/new-module-registry-test.js +++ b/src/workerd/api/tests/new-module-registry-test.js @@ -366,7 +366,10 @@ export const invalidModuleSpecifier = { () => null, (e) => e ); - ok(stat instanceof TypeError, `expected TypeError, got ${stat && stat.name}`); + ok( + stat instanceof TypeError, + `expected TypeError, got ${stat && stat.name}` + ); ok(/Invalid module specifier/.test(stat.message), stat.message); }, }; diff --git a/src/workerd/api/tests/ts-identity-streams-test.js b/src/workerd/api/tests/ts-identity-streams-test.js index 5c43860016e..17761a996e5 100644 --- a/src/workerd/api/tests/ts-identity-streams-test.js +++ b/src/workerd/api/tests/ts-identity-streams-test.js @@ -744,7 +744,7 @@ export const flsUnderwriteZeroBytesThrows = { await rejects( () => writer.close(), (err) => { - ok(err instanceof RangeError); + ok(err instanceof RangeError); ok(err.message.includes('did not see all expected bytes')); return true; } diff --git a/src/workerd/api/worker-rpc.c++ b/src/workerd/api/worker-rpc.c++ index 21bb51f99b1..1f3f7082f2d 100644 --- a/src/workerd/api/worker-rpc.c++ +++ b/src/workerd/api/worker-rpc.c++ @@ -1055,7 +1055,8 @@ class JsRpcTargetBase: public rpc::JsRpcTarget::Server { // Try to execute the requested method. co_return co_await enterIsolateAndCall(callContext).catch_([this](kj::Exception&& e) { - maybeAddDurableObjectId(e, durableObjectId.map([](const kj::String& id) { return id.asPtr(); })); + maybeAddDurableObjectId( + e, durableObjectId.map([](const kj::String& id) { return id.asPtr(); })); if (jsg::isTunneledException(e.getDescription())) { // Annotate exceptions in RPC worker calls as remote exceptions. auto description = jsg::stripRemoteExceptionPrefix(e.getDescription()); @@ -1229,9 +1230,11 @@ class JsRpcTargetBase: public rpc::JsRpcTarget::Server { // as a result of the promise being rejected). This will implicitly dispose the param // stubs. }), - ctx.addFunctor([callPipelineFulfillerRef, durableObjectId = getCurrentDurableObjectId()]( + ctx.addFunctor( + [callPipelineFulfillerRef, durableObjectId = getCurrentDurableObjectId()]( jsg::Lock& js, jsg::Value&& error) { - maybeAddDurableObjectId(js, error, durableObjectId.map([](const kj::String& id) { return id.asPtr(); })); + maybeAddDurableObjectId( + js, error, durableObjectId.map([](const kj::String& id) { return id.asPtr(); })); // If we set up a `callPipeline` early, we have to make sure it propagates the error. // (Otherwise we get a PromiseFulfiller error instead, which is pretty useless...) KJ_IF_SOME(cpf, callPipelineFulfillerRef) { diff --git a/src/workerd/io/worker-entrypoint.c++ b/src/workerd/io/worker-entrypoint.c++ index dd70f47d3d8..47ef90251b9 100644 --- a/src/workerd/io/worker-entrypoint.c++ +++ b/src/workerd/io/worker-entrypoint.c++ @@ -571,8 +571,7 @@ kj::Promise WorkerEntrypoint::requestImpl(kj::HttpMethod method, // exception types need no annotation. Set before exceptionToPropagate() so it survives the // internal-exception description rewrite; the detail serializes back across the RPC boundary. if (exception.getType() == kj::Exception::Type::DISCONNECTED) { - exception.setDetail( - jsg::REQUEST_DELIVERED_TO_ACTOR_DETAIL_ID, kj::heapArray(0)); + exception.setDetail(jsg::REQUEST_DELIVERED_TO_ACTOR_DETAIL_ID, kj::heapArray(0)); } // TODO(cleanup): We'd really like to tunnel exceptions any time a worker is calling another // worker, not just for actors (and W2W below), but getting that right will require cleaning diff --git a/src/workerd/jsg/modules-new.c++ b/src/workerd/jsg/modules-new.c++ index d3426ec9764..22f08dd1ab5 100644 --- a/src/workerd/jsg/modules-new.c++ +++ b/src/workerd/jsg/modules-new.c++ @@ -1188,8 +1188,7 @@ v8::MaybeLocal> resolv // A circular dependency is a module-graph/loading error, not a type // error, so this is an Error (matching the require path and Node's // ERR_REQUIRE_CYCLE_MODULE which extends Error). - js.throwException( - js.error(kj::str("Circular dependency when resolving module: ", spec))); + js.throwException(js.error(kj::str("Circular dependency when resolving module: ", spec))); return v8::MaybeLocal(); } diff --git a/src/workerd/jsg/util.c++ b/src/workerd/jsg/util.c++ index 2ff326c8f47..699cb05820c 100644 --- a/src/workerd/jsg/util.c++ +++ b/src/workerd/jsg/util.c++ @@ -454,8 +454,8 @@ void addExceptionDetail(Lock& js, kj::Exception& exception, v8::Local } void addDurableObjectId(kj::Exception& exception, kj::StringPtr durableObjectId) { - exception.setDetail(DURABLE_OBJECT_EXCEPTION_METADATA_DETAIL_ID, - kj::heapArray(durableObjectId.asBytes())); + exception.setDetail( + DURABLE_OBJECT_EXCEPTION_METADATA_DETAIL_ID, kj::heapArray(durableObjectId.asBytes())); } kj::Maybe getDurableObjectId(const kj::Exception& exception) { diff --git a/src/workerd/jsg/util.h b/src/workerd/jsg/util.h index 8a3f00ded48..851c1b1d110 100644 --- a/src/workerd/jsg/util.h +++ b/src/workerd/jsg/util.h @@ -106,8 +106,7 @@ constexpr kj::Exception::DetailTypeId REQUEST_NOT_DELIVERED_TO_ACTOR_DETAIL_ID = // have run). Must not be retried as a delivery failure. Set at the receiving entrypoint's actor // catch so it survives the internal-exception description rewrite and serializes back across the RPC // boundary. The payload is a zero-length array (marker only). -constexpr kj::Exception::DetailTypeId REQUEST_DELIVERED_TO_ACTOR_DETAIL_ID = - 0x7f6e0bece261e8eeull; +constexpr kj::Exception::DetailTypeId REQUEST_DELIVERED_TO_ACTOR_DETAIL_ID = 0x7f6e0bece261e8eeull; // Detail type for Durable Object metadata on exceptions propagated out of actor execution. constexpr kj::Exception::DetailTypeId DURABLE_OBJECT_EXCEPTION_METADATA_DETAIL_ID = diff --git a/src/workerd/server/tests/container-client/test.js b/src/workerd/server/tests/container-client/test.js index dd8e494a7ac..c3ffeeeaa6e 100644 --- a/src/workerd/server/tests/container-client/test.js +++ b/src/workerd/server/tests/container-client/test.js @@ -309,9 +309,12 @@ export class DurableObjectExample extends DurableObject { { const ac = new AbortController(); ac.abort(); - assert.throws(() => container.exec(['echo', 'hello'], { signal: ac.signal }), { - name: 'AbortError', - }); + assert.throws( + () => container.exec(['echo', 'hello'], { signal: ac.signal }), + { + name: 'AbortError', + } + ); } // 14. Aborting the signal while the process is running kills it (SIGKILL). From b41154b8f034b9bec26efca661d69c0f644c6337 Mon Sep 17 00:00:00 2001 From: Mike Aizatsky Date: Mon, 13 Jul 2026 11:30:26 -0700 Subject: [PATCH 075/101] Harden WritableStreamSink reference --- src/workerd/api/streams/common.h | 12 +++++-- .../api/streams/identity-transform-stream.c++ | 8 ++--- .../api/streams/identity-transform-stream.h | 2 +- src/workerd/api/streams/internal.c++ | 35 ++++++++++--------- src/workerd/api/streams/internal.h | 2 +- src/workerd/api/streams/readable-source.c++ | 6 ++-- src/workerd/api/streams/readable.c++ | 4 +-- src/workerd/api/streams/standard.c++ | 4 +-- src/workerd/api/streams/writable.c++ | 10 +++--- 9 files changed, 46 insertions(+), 37 deletions(-) diff --git a/src/workerd/api/streams/common.h b/src/workerd/api/streams/common.h index 50ea0875baf..f614c8e7b83 100644 --- a/src/workerd/api/streams/common.h +++ b/src/workerd/api/streams/common.h @@ -226,8 +226,14 @@ struct Transformer { // Likewise, when creating a new kind of *internal* WritableStream, where the data destination is // a kj stream, you will implement the WritableStreamSink API. -class WritableStreamSink { +class WritableStreamSink: public kj::PtrTarget { public: + // Obtain a strong pointer to this sink. Callers must ensure the sink outlives the returned + // kj::Ptr (see docs/hardening.md). + kj::Ptr getPtr() { + return addPtrToThis(); + } + virtual kj::Promise write(kj::ArrayPtr buffer) KJ_WARN_UNUSED_RESULT = 0; virtual kj::Promise write( kj::ArrayPtr> pieces) KJ_WARN_UNUSED_RESULT = 0; @@ -262,7 +268,7 @@ class ReadableStreamSource: public kj::PtrTarget { // If `end` is true, then `output.end()` will be called after pumping. Note that it's especially // important to take advantage of this when using deferred proxying since calling `end()` // directly might attempt to use the `IoContext` to call `registerPendingEvent()`. - virtual kj::Promise> pumpTo(WritableStreamSink& output, bool end); + virtual kj::Promise> pumpTo(kj::Ptr output, bool end); // If pumpTo() pumps to a system stream, what is the best encoding for that system stream to // use? This is just a hint. @@ -491,7 +497,7 @@ class ReadableStreamController { virtual void close(jsg::Lock& js) = 0; virtual void error(jsg::Lock& js, jsg::JsValue reason) = 0; virtual void release(jsg::Lock& js, kj::Maybe maybeError = kj::none) = 0; - virtual kj::Maybe> tryPumpTo(WritableStreamSink& sink, bool end) = 0; + virtual kj::Maybe> tryPumpTo(kj::Ptr sink, bool end) = 0; virtual jsg::Promise read(jsg::Lock& js) = 0; }; diff --git a/src/workerd/api/streams/identity-transform-stream.c++ b/src/workerd/api/streams/identity-transform-stream.c++ index a5b31ed7ef5..0b187cfee57 100644 --- a/src/workerd/api/streams/identity-transform-stream.c++ +++ b/src/workerd/api/streams/identity-transform-stream.c++ @@ -125,7 +125,7 @@ class IdentityTransformStreamImpl final: public kj::Refcounted, return promise; } - kj::Promise> pumpTo(WritableStreamSink& output, bool end) override { + kj::Promise> pumpTo(kj::Ptr output, bool end) override { #ifdef KJ_NO_RTTI // Yes, I'm paranoid. static_assert(!KJ_NO_RTTI, "Need RTTI for correctness"); @@ -136,7 +136,7 @@ class IdentityTransformStreamImpl final: public kj::Refcounted, JSG_REQUIRE(!isIdentityTransformStream(output), TypeError, "Inter-TransformStream ReadableStream.pipeTo() is not implemented."); - return ReadableStreamSource::pumpTo(output, end); + return ReadableStreamSource::pumpTo(kj::mv(output), end); } kj::Maybe tryGetLength(StreamEncoding encoding) override { @@ -391,8 +391,8 @@ OneWayPipe newIdentityPipe(kj::Maybe expectedLength) { return OneWayPipe{.in = kj::mv(pair.readable), .out = kj::mv(pair.writable)}; } -bool isIdentityTransformStream(WritableStreamSink& sink) { - return kj::dynamicDowncastIfAvailable(sink) != kj::none; +bool isIdentityTransformStream(kj::Ptr sink) { + return kj::dynamicDowncastIfAvailable(sink.asRef()) != kj::none; } } // namespace workerd::api diff --git a/src/workerd/api/streams/identity-transform-stream.h b/src/workerd/api/streams/identity-transform-stream.h index fe4b880598b..a6035a95247 100644 --- a/src/workerd/api/streams/identity-transform-stream.h +++ b/src/workerd/api/streams/identity-transform-stream.h @@ -58,6 +58,6 @@ struct OneWayPipe { OneWayPipe newIdentityPipe(kj::Maybe expectedLength = kj::none); -bool isIdentityTransformStream(WritableStreamSink& sink); +bool isIdentityTransformStream(kj::Ptr sink); } // namespace workerd::api diff --git a/src/workerd/api/streams/internal.c++ b/src/workerd/api/streams/internal.c++ index b5fe2ed206d..e3c3365d75d 100644 --- a/src/workerd/api/streams/internal.c++ +++ b/src/workerd/api/streams/internal.c++ @@ -36,7 +36,7 @@ namespace { } kj::Promise pumpTo( - kj::Ptr input, WritableStreamSink& output, bool end) { + kj::Ptr input, kj::Ptr output, bool end) { kj::byte buffer[65536]{}; while (true) { @@ -44,12 +44,12 @@ kj::Promise pumpTo( if (amount == 0) { if (end) { - co_await output.end(); + co_await output->end(); } co_return; } - co_await output.write(kj::arrayPtr(buffer, amount)); + co_await output->write(kj::arrayPtr(buffer, amount)); } } @@ -295,7 +295,7 @@ class TeeBranch final: public ReadableStreamSource { return inner->tryRead(buffer, minBytes, maxBytes); } - kj::Promise> pumpTo(WritableStreamSink& output, bool end) override { + kj::Promise> pumpTo(kj::Ptr output, bool end) override { #ifdef KJ_NO_RTTI // Yes, I'm paranoid. static_assert(!KJ_NO_RTTI, "Need RTTI for correctness"); @@ -315,7 +315,7 @@ class TeeBranch final: public ReadableStreamSource { co_await inner->pumpTo(outputAdapter); if (end) { - co_await output.end(); + co_await output->end(); } // We only use `TeeBranch` when a locally-sourced stream was tee'd (because system streams @@ -353,21 +353,24 @@ class TeeBranch final: public ReadableStreamSource { // read logic. class PumpAdapter final: public kj::AsyncOutputStream { public: - explicit PumpAdapter(WritableStreamSink& inner): inner(inner) {} + explicit PumpAdapter(kj::Ptr inner): inner(kj::mv(inner)) {} kj::Promise write(kj::ArrayPtr buffer) override { - return inner.write(buffer); + return inner->write(buffer); } kj::Promise write(kj::ArrayPtr> pieces) override { - return inner.write(pieces); + return inner->write(pieces); } kj::Promise whenWriteDisconnected() override { KJ_UNIMPLEMENTED("whenWriteDisconnected() not expected on PumpAdapter"); } - WritableStreamSink& inner; + // Non-owning pointer to the sink being pumped to. The caller of `TeeBranch::pumpTo()` owns the + // sink and must keep it alive for the entire duration of the pump; this PumpAdapter (and the + // kj::Ptr it holds) only lives within that pump and must not outlive the sink. + kj::Ptr inner; }; kj::Own inner; @@ -377,14 +380,14 @@ class TeeBranch final: public ReadableStreamSource { // ======================================================================================= kj::Promise> ReadableStreamSource::pumpTo( - WritableStreamSink& output, bool end) { - KJ_IF_SOME(p, output.tryPumpFrom(addPtrToThis(), end)) { + kj::Ptr output, bool end) { + KJ_IF_SOME(p, output->tryPumpFrom(addPtrToThis(), end)) { return kj::mv(p); } // Non-optimized pumpTo() is presumed to require the IoContext to remain live, so don't do // anything in the deferred proxy part. - return addNoopDeferredProxy(api::pumpTo(addPtrToThis(), output, end)); + return addNoopDeferredProxy(api::pumpTo(addPtrToThis(), kj::mv(output), end)); } kj::Maybe ReadableStreamSource::tryGetLength(StreamEncoding encoding) { @@ -1939,7 +1942,7 @@ jsg::Promise WritableStreamInternalController::writeLoopAfterFrontOutputLo // error. But in that case, the queue would already have been drained and we wouldn't be here. return KJ_ASSERT_NONNULL( state.whenActive([&](IoOwn& writable) mutable -> jsg::Promise { - KJ_IF_SOME(promise, sourceRef.tryPumpTo(*writable->sink, !preventClose)) { + KJ_IF_SOME(promise, sourceRef.tryPumpTo(writable->sink->getPtr(), !preventClose)) { return handlePromise(js, ioContext.awaitIo(js, writable->canceler.wrap( @@ -2461,11 +2464,11 @@ void ReadableStreamInternalController::PipeLocked::release( } kj::Maybe> ReadableStreamInternalController::PipeLocked::tryPumpTo( - WritableStreamSink& sink, bool end) { + kj::Ptr sink, bool end) { // This is safe because the caller should have already checked isClosed and tryGetErrored // and handled those before calling tryPumpTo. auto& readable = KJ_ASSERT_NONNULL(inner.state.tryGetUnsafe()); - return IoContext::current().waitForDeferredProxy(readable->pumpTo(sink, end)); + return IoContext::current().waitForDeferredProxy(readable->pumpTo(kj::mv(sink), end)); } jsg::Promise ReadableStreamInternalController::PipeLocked::read(jsg::Lock& js) { @@ -2589,7 +2592,7 @@ kj::Promise> ReadableStreamInternalController::pumpTo( }; auto holder = kj::rc(kj::mv(sink), kj::mv(source)); - return holder->source->pumpTo(*holder->sink, end) + return holder->source->pumpTo(holder->sink->getPtr(), end) .then([holder = holder.addRef()](DeferredProxy proxy) mutable -> DeferredProxy { proxy.proxyTask = proxy.proxyTask.attach(holder.addRef()); holder->done = true; diff --git a/src/workerd/api/streams/internal.h b/src/workerd/api/streams/internal.h index e45daa7becd..a75a88dde62 100644 --- a/src/workerd/api/streams/internal.h +++ b/src/workerd/api/streams/internal.h @@ -143,7 +143,7 @@ class ReadableStreamInternalController: public ReadableStreamController { void release(jsg::Lock& js, kj::Maybe maybeError = kj::none) override; - kj::Maybe> tryPumpTo(WritableStreamSink& sink, bool end) override; + kj::Maybe> tryPumpTo(kj::Ptr sink, bool end) override; jsg::Promise read(jsg::Lock& js) override; diff --git a/src/workerd/api/streams/readable-source.c++ b/src/workerd/api/streams/readable-source.c++ index 6c1b84fe04d..b763171f04c 100644 --- a/src/workerd/api/streams/readable-source.c++ +++ b/src/workerd/api/streams/readable-source.c++ @@ -824,17 +824,17 @@ class MemoryInputStream final: public ReadableStreamSource { return kj::none; } - kj::Promise> pumpTo(WritableStreamSink& output, bool end) override { + kj::Promise> pumpTo(kj::Ptr output, bool end) override { // Explicitly NOT using KJ_CO_MAGIC BEGIN_DEFERRED_PROXYING here! // The backing memory may be tied to V8 heap (e.g., ArrayBuffer, Blob data), // so we must complete all I/O before the IoContext can be released. if (unread.size() > 0) { auto data = unread; unread = nullptr; - co_await output.write(data); + co_await output->write(data); } if (end) { - co_await output.end(); + co_await output->end(); } co_return; } diff --git a/src/workerd/api/streams/readable.c++ b/src/workerd/api/streams/readable.c++ index 2d9c7efe8d0..9d0729a1149 100644 --- a/src/workerd/api/streams/readable.c++ +++ b/src/workerd/api/streams/readable.c++ @@ -642,12 +642,12 @@ class NoDeferredProxyReadableStream final: public ReadableStreamSource { return inner->tryRead(buffer, minBytes, maxBytes); } - kj::Promise> pumpTo(WritableStreamSink& output, bool end) override { + kj::Promise> pumpTo(kj::Ptr output, bool end) override { // Move the deferred proxy part of the task over to the non-deferred part. To do this, // we use `ioctx.waitForDeferredProxy()`, which returns a single promise covering both parts // (and, importantly, registering pending events where needed). Then, we add a noop deferred // proxy to the end of that. - return addNoopDeferredProxy(ioctx.waitForDeferredProxy(inner->pumpTo(output, end))); + return addNoopDeferredProxy(ioctx.waitForDeferredProxy(inner->pumpTo(kj::mv(output), end))); } StreamEncoding getPreferredEncoding() override { diff --git a/src/workerd/api/streams/standard.c++ b/src/workerd/api/streams/standard.c++ index 51bae723c37..f66a544f5c9 100644 --- a/src/workerd/api/streams/standard.c++ +++ b/src/workerd/api/streams/standard.c++ @@ -123,7 +123,7 @@ class ReadableLockImpl { inner.lock.state.template transitionTo(); } - kj::Maybe> tryPumpTo(WritableStreamSink& sink, bool end) override; + kj::Maybe> tryPumpTo(kj::Ptr sink, bool end) override; jsg::Promise read(jsg::Lock& js) override; @@ -352,7 +352,7 @@ void ReadableLockImpl::onError(jsg::Lock& js, jsg::JsValue reason) { template kj::Maybe> ReadableLockImpl::PipeLocked::tryPumpTo( - WritableStreamSink& sink, bool end) { + kj::Ptr sink, bool end) { // We return nullptr here because this controller does not support kj's pumpTo. return kj::none; } diff --git a/src/workerd/api/streams/writable.c++ b/src/workerd/api/streams/writable.c++ index ac04b9399cb..663c2fa0209 100644 --- a/src/workerd/api/streams/writable.c++ +++ b/src/workerd/api/streams/writable.c++ @@ -288,10 +288,10 @@ class WritableStreamRpcAdapter final: public capnp::ExplicitEndOutputStream { } kj::Promise write(kj::ArrayPtr buffer) override { - return canceler.wrap(getInner().write(buffer)); + return canceler.wrap(getInner()->write(buffer)); } kj::Promise write(kj::ArrayPtr> pieces) override { - return canceler.wrap(getInner().write(pieces)); + return canceler.wrap(getInner()->write(pieces)); } // TODO(perf): We can't properly implement tryPumpFrom(), which means that Cap'n Proto will @@ -305,7 +305,7 @@ class WritableStreamRpcAdapter final: public capnp::ExplicitEndOutputStream { } kj::Promise end() override { - return canceler.wrap(getInner().end()); + return canceler.wrap(getInner()->end()); } private: @@ -316,8 +316,8 @@ class WritableStreamRpcAdapter final: public capnp::ExplicitEndOutputStream { kj::refcounted>( kj::Badge(), *this); - WritableStreamSink& getInner() { - return *KJ_UNWRAP_OR(inner, { kj::throwFatalException(cancellationException()); }); + kj::Ptr getInner() { + return KJ_UNWRAP_OR(inner, { kj::throwFatalException(cancellationException()); })->getPtr(); } static kj::Exception cancellationException() { From 581a2687f1de5e59974de254db11eccacd82f5f0 Mon Sep 17 00:00:00 2001 From: Dan Lapid Date: Fri, 3 Jul 2026 04:48:41 +0000 Subject: [PATCH 076/101] Add `bytes` variant to Frankenvalue Frankenvalue could previously only carry an ArrayBuffer via V8 serialization, which requires a live JS context. This makes it impossible to place binary data (e.g. a Workers+Assets manifest) into `ctx.props` from config or a control plane. Add a `bytes` member to the Frankenvalue union that materializes as a JS ArrayBuffer via jsg::Lock::wrapBytes() -- the same path as a `data` binding -- plus a matching `arrayBuffer` field in the capnp union and a `Frankenvalue::fromBytes()` factory. --- src/workerd/io/frankenvalue-test.c++ | 51 ++++++++++++++++++++++++++++ src/workerd/io/frankenvalue.c++ | 21 ++++++++++++ src/workerd/io/frankenvalue.capnp | 4 +++ src/workerd/io/frankenvalue.h | 9 ++++- 4 files changed, 84 insertions(+), 1 deletion(-) diff --git a/src/workerd/io/frankenvalue-test.c++ b/src/workerd/io/frankenvalue-test.c++ index 6a4dcdcf686..0ca57f259bb 100644 --- a/src/workerd/io/frankenvalue-test.c++ +++ b/src/workerd/io/frankenvalue-test.c++ @@ -190,6 +190,57 @@ KJ_TEST("Frankenvalue capability value") { }); } +KJ_TEST("Frankenvalue bytes value") { + jsg::test::Evaluator e(v8System); + + e.run([&](auto& lock) { + jsg::Lock& js = lock; + + static constexpr kj::byte kBytes[] = {0x01, 0x02, 0x03, 0x04, 0xff}; + auto bytesPtr = kj::arrayPtr(kBytes); + + // Construct a Frankenvalue via fromBytes(), then round-trip it through capnp and clone(), + // exercising the Bytes variant on the toCapnp / fromCapnp / clone paths. + auto value = Frankenvalue::fromBytes(kj::heapArray(bytesPtr)); + KJ_EXPECT(value.estimateSize() == bytesPtr.size()); + + // estimateSize() includes stitched-in properties. + value.setProperty(kj::str("nested"), Frankenvalue::fromBytes(kj::heapArray(bytesPtr))); + KJ_EXPECT(value.estimateSize() == 2 * bytesPtr.size() + "nested"_kjc.size()); + + // Round-trip through capnp. + { + capnp::MallocMessageBuilder message; + auto builder = message.initRoot(); + value.toCapnp(builder); + + // Sanity-check the on-wire representation: root and nested are both arrayBuffer variants. + auto reader = builder.asReader(); + KJ_EXPECT(reader.which() == rpc::Frankenvalue::ARRAY_BUFFER); + KJ_EXPECT(reader.getArrayBuffer().asChars() == bytesPtr.asChars()); + KJ_EXPECT(reader.getProperties().size() == 1); + KJ_EXPECT(reader.getProperties()[0].which() == rpc::Frankenvalue::ARRAY_BUFFER); + + value = Frankenvalue::fromCapnp(reader); + } + + // Exercise clone(). + value = value.clone(); + + // toJs() produces an ArrayBuffer holding the original bytes; the nested property is an + // ArrayBuffer on `nested`. + auto jsValue = value.toJs(js); + v8::Local v8Value = jsValue; + KJ_ASSERT(v8Value->IsObject()); + auto obj = v8Value.As(); + auto nested = jsg::check(obj->Get(js.v8Context(), js.str("nested"_kj))); + KJ_ASSERT(nested->IsArrayBuffer()); + auto nestedBuf = nested.As(); + KJ_EXPECT(nestedBuf->ByteLength() == bytesPtr.size()); + KJ_EXPECT(memcmp(nestedBuf->Data(), bytesPtr.begin(), bytesPtr.size()) == 0); + }); +} + KJ_TEST("Frankenvalue fromCapnp rejects capability index out of range") { // Security: a `capability` value must not reference a cap table index beyond this node's base // caps, or toJs() would read out of bounds. diff --git a/src/workerd/io/frankenvalue.c++ b/src/workerd/io/frankenvalue.c++ index 8cd3f0ac8d4..eec2dbdec67 100644 --- a/src/workerd/io/frankenvalue.c++ +++ b/src/workerd/io/frankenvalue.c++ @@ -18,6 +18,9 @@ Frankenvalue Frankenvalue::cloneImpl() const { KJ_CASE_ONEOF(v8Serialized, V8Serialized) { result.value = V8Serialized{kj::heapArray(v8Serialized.data.asPtr())}; } + KJ_CASE_ONEOF(bytes, Bytes) { + result.value = Bytes{kj::heapArray(bytes.data.asPtr())}; + } KJ_CASE_ONEOF(capability, Capability) { result.value = capability; } @@ -88,6 +91,9 @@ void Frankenvalue::toCapnpImpl(rpc::Frankenvalue::Builder builder, size_t capTab KJ_CASE_ONEOF(v8Serialized, V8Serialized) { builder.setV8Serialized(v8Serialized.data); } + KJ_CASE_ONEOF(bytes, Bytes) { + builder.setArrayBuffer(bytes.data); + } KJ_CASE_ONEOF(capability, Capability) { // Defense-in-depth: the cap index must reference one of this node's base caps. fromCapnp() // enforces the same invariant on the decode side (see fromCapnpImpl()). @@ -142,6 +148,9 @@ size_t Frankenvalue::fromCapnpImpl( case rpc::Frankenvalue::V8_SERIALIZED: this->value = V8Serialized{kj::heapArray(reader.getV8Serialized())}; break; + case rpc::Frankenvalue::ARRAY_BUFFER: + this->value = Bytes{kj::heapArray(reader.getArrayBuffer())}; + break; case rpc::Frankenvalue::CAPABILITY: { auto cap = reader.getCapability(); // The tag is untrusted, but we can't validate it here: resolving it to a deserializer @@ -212,6 +221,9 @@ jsg::JsValue Frankenvalue::toJsImpl(jsg::Lock& js, kj::ArrayPtr data) { + Frankenvalue result; + result.value = Bytes{kj::mv(data)}; + return result; +} + size_t Frankenvalue::estimateSize() const { size_t result = 0; @@ -309,6 +327,9 @@ size_t Frankenvalue::estimateSize() const { KJ_CASE_ONEOF(v8Serialized, V8Serialized) { result += v8Serialized.data.size(); } + KJ_CASE_ONEOF(bytes, Bytes) { + result += bytes.data.size(); + } KJ_CASE_ONEOF(capability, Capability) { result += sizeof(Capability); } diff --git a/src/workerd/io/frankenvalue.capnp b/src/workerd/io/frankenvalue.capnp index 1652d9cbaf6..49fbe757aa3 100644 --- a/src/workerd/io/frankenvalue.capnp +++ b/src/workerd/io/frankenvalue.capnp @@ -21,6 +21,10 @@ struct Frankenvalue { v8Serialized @2 :Data; # Parse these V8-serialized bytes to compute the value. + arrayBuffer @9 :Data; + # The value is an `ArrayBuffer` wrapping these bytes (no V8 serialization). Lets a control + # plane place binary data into `ctx.props` without a JavaScript context. + capability :group { # The value is a single capability (e.g. a service binding / Fetcher), taken directly from # this value's cap table. This allows a capability to be expressed without going through V8 diff --git a/src/workerd/io/frankenvalue.h b/src/workerd/io/frankenvalue.h index 7304383c437..cb8f25f9c31 100644 --- a/src/workerd/io/frankenvalue.h +++ b/src/workerd/io/frankenvalue.h @@ -62,6 +62,10 @@ class Frankenvalue { // then JSON-stringifying from there.) static Frankenvalue fromJson(kj::String json); + // Construct a Frankenvalue whose value is an `ArrayBuffer` wrapping `data`, without going + // through V8 serialization. For placing binary data into `ctx.props` where no JS context exists. + static Frankenvalue fromBytes(kj::Array data); + // Construct a Frankenvalue whose value is a single capability (cap table entry), without going // through V8 serialization. When converted to JS, the capability is materialized using the // deserializer registered for `tag` (a `workerd::rpc::SerializationTag` value, e.g. @@ -184,6 +188,9 @@ class Frankenvalue { struct V8Serialized { kj::Array data; }; + struct Bytes { + kj::Array data; + }; struct Capability { // Index into this value's base cap table (the caps referenced by the union, before property // caps). @@ -194,7 +201,7 @@ class Frankenvalue { // need not depend on the `SerializationTag` schema. uint16_t tag; }; - kj::OneOf value; + kj::OneOf value; struct Property; kj::Vector properties; From 28ba2b6e8feaa8346c5a6569ead1a2e0789e56d2 Mon Sep 17 00:00:00 2001 From: Dan Lapid Date: Fri, 3 Jul 2026 04:48:51 +0000 Subject: [PATCH 077/101] Add decompileCompatibilityFlags for the full flag set decompileCompatibilityFlagsForFl only returns flags annotated with `$neededByFl`. Callers that need to propagate a worker's complete effective compatibility to another component (e.g. the Workers+Assets asset-worker, which takes an explicit `compatibility_flags` list) need every set flag. Add decompileCompatibilityFlags() returning the enable-flag for every set field, sharing the field-table plumbing with the FL variant. --- src/workerd/io/compatibility-date-test.c++ | 65 ++++++++++++++++++++++ src/workerd/io/compatibility-date.c++ | 43 ++++++++++---- src/workerd/io/compatibility-date.h | 9 ++- 3 files changed, 103 insertions(+), 14 deletions(-) diff --git a/src/workerd/io/compatibility-date-test.c++ b/src/workerd/io/compatibility-date-test.c++ index 774409626ef..582ed147795 100644 --- a/src/workerd/io/compatibility-date-test.c++ +++ b/src/workerd/io/compatibility-date-test.c++ @@ -398,6 +398,71 @@ KJ_TEST("encode to flag list for FL") { } } +KJ_TEST("encode to full flag list") { + // Same setup as "encode to flag list for FL", but exercises decompileCompatibilityFlags -- which + // must return EVERY set flag, not just the $neededByFl subset. + + capnp::MallocMessageBuilder message; + auto orphanage = message.getOrphanage(); + + auto compileOwnFeatureFlags = [&](kj::StringPtr compatDate, + kj::ArrayPtr featureFlags) { + auto flagListOrphan = orphanage.newOrphan>(featureFlags.size()); + auto flagList = flagListOrphan.get(); + for (auto i: kj::indices(featureFlags)) { + flagList.set(i, featureFlags.begin()[i]); + } + + auto outputOrphan = orphanage.newOrphan(); + auto output = outputOrphan.get(); + + SimpleWorkerErrorReporter errorReporter; + compileCompatibilityFlags(compatDate, flagList.asReader(), output, errorReporter, + /*experimental=*/false, CompatibilityDateValidation::FUTURE_FOR_TEST, nullptr); + KJ_ASSERT(errorReporter.errors.empty()); + + return kj::mv(outputOrphan); + }; + + auto contains = [](kj::ArrayPtr flags, kj::StringPtr needle) { + for (auto& f: flags) { + if (f == needle) return true; + } + return false; + }; + + { + // At 2021-11-10, several non-$neededByFl flags are enabled by date. The full variant returns + // them; -ForFl returns none. Note that the full variant may return additional flags that lack + // a $compatEnableDate but are on by default -- test explicit membership rather than size. + auto featureFlagsOrphan = compileOwnFeatureFlags("2021-11-10", {}); + auto featureFlags = featureFlagsOrphan.get(); + auto forFl = decompileCompatibilityFlagsForFl(featureFlags); + auto full = decompileCompatibilityFlags(featureFlags); + KJ_EXPECT(forFl.size() == 0); + KJ_EXPECT(contains(full, "formdata_parser_supports_files"_kj)); + KJ_EXPECT(contains(full, "fetch_refuses_unknown_protocols"_kj)); + // A flag whose $compatEnableDate is after this compat date is NOT set. + KJ_EXPECT(!contains(full, "minimal_subrequests"_kj)); + } + + { + // Explicit enable-flag on a date that would otherwise leave it off. + auto featureFlagsOrphan = compileOwnFeatureFlags("2021-05-17", {"minimal_subrequests"_kj}); + auto strings = decompileCompatibilityFlags(featureFlagsOrphan.get()); + KJ_EXPECT(contains(strings, "minimal_subrequests"_kj)); + } + + { + // Explicit disable-flag suppresses a date-enabled flag in the full variant too. + auto featureFlagsOrphan = compileOwnFeatureFlags("2022-07-01", {"cots_on_external_fetch"}); + auto strings = decompileCompatibilityFlags(featureFlagsOrphan.get()); + KJ_EXPECT(!contains(strings, "no_cots_on_external_fetch"_kj)); + // But other date-enabled flags still appear. + KJ_EXPECT(contains(strings, "minimal_subrequests"_kj)); + } +} + KJ_TEST("compatibility dates must be Tuesday, Wednesday, or Thursday") { // List of specific flags that are allowed to use non-conformant dates // (already deployed and can't be changed for compatibility reasons) diff --git a/src/workerd/io/compatibility-date.c++ b/src/workerd/io/compatibility-date.c++ index 8128802d85d..1a266ec4d43 100644 --- a/src/workerd/io/compatibility-date.c++ +++ b/src/workerd/io/compatibility-date.c++ @@ -315,7 +315,10 @@ struct ParsedField { capnp::StructSchema::Field field; }; -kj::Array makeFieldTable(capnp::StructSchema::FieldList fields) { +// When `onlyNeededByFl` is true, only fields annotated `$neededByFl` are included; otherwise every +// field carrying a `$compatEnableFlag` is. +kj::Array makeFieldTable( + capnp::StructSchema::FieldList fields, bool onlyNeededByFl) { kj::Vector table(fields.size()); for (auto field: fields) { @@ -330,34 +333,50 @@ kj::Array makeFieldTable(capnp::StructSchema::FieldList field } } - if (neededByFl) { + if (onlyNeededByFl) { + if (!neededByFl) continue; table.add(ParsedField{ .enableFlag = KJ_REQUIRE_NONNULL(enableFlag), .field = field, }); + } else { + KJ_IF_SOME(flag, enableFlag) { + table.add(ParsedField{ + .enableFlag = flag, + .field = field, + }); + } } } return table.releaseAsArray(); } -} // namespace - -kj::Array decompileCompatibilityFlagsForFl(CompatibilityFlags::Reader input) { - static const auto fieldTable = - makeFieldTable(capnp::Schema::from().getFields()); - - kj::Vector enableFlags; - enableFlags.reserve(fieldTable.size()); - for (auto field: fieldTable) { +kj::Array decompileFlags( + CompatibilityFlags::Reader input, kj::ArrayPtr fieldTable) { + kj::Vector enableFlags(fieldTable.size()); + for (auto& field: fieldTable) { if (capnp::toDynamic(input).get(field.field).as()) { enableFlags.add(field.enableFlag); } } - return enableFlags.releaseAsArray(); } +} // namespace + +kj::Array decompileCompatibilityFlagsForFl(CompatibilityFlags::Reader input) { + static const auto fieldTable = makeFieldTable( + capnp::Schema::from().getFields(), /*onlyNeededByFl=*/true); + return decompileFlags(input, fieldTable); +} + +kj::Array decompileCompatibilityFlags(CompatibilityFlags::Reader input) { + static const auto fieldTable = makeFieldTable( + capnp::Schema::from().getFields(), /*onlyNeededByFl=*/false); + return decompileFlags(input, fieldTable); +} + kj::Maybe normalizeCompatDate(kj::StringPtr date) { return CompatDate::parse(date).map([](auto v) { return v.toString(); }); } diff --git a/src/workerd/io/compatibility-date.h b/src/workerd/io/compatibility-date.h index b7b62009657..7d405d4178e 100644 --- a/src/workerd/io/compatibility-date.h +++ b/src/workerd/io/compatibility-date.h @@ -48,9 +48,14 @@ void compileCompatibilityFlags(kj::StringPtr compatDate, // Return an array of compatibility enable-flags which express the given FeatureFlags. The returned // StringPtrs point to FeatureFlags annotation parameters, which live in static storage. +// +// TODO(soon): Introduce a codec which can minimize the number of flags generated by either variant +// below by choosing a good compatibility date. kj::Array decompileCompatibilityFlagsForFl(CompatibilityFlags::Reader input); -// TODO(soon): Introduce a codec which can minimize the number of flags generated by choosing a good -// compatibility date. + +// Like decompileCompatibilityFlagsForFl, but returns EVERY set flag, not just those annotated +// `$neededByFl`. Use when propagating a worker's full effective compatibility to another component. +kj::Array decompileCompatibilityFlags(CompatibilityFlags::Reader input); // Exposed to unit test the parser. kj::Maybe normalizeCompatDate(kj::StringPtr date); From da2cfe1330adbb940d112d53f551be8512efe6ba Mon Sep 17 00:00:00 2001 From: Dan Lapid Date: Fri, 3 Jul 2026 21:52:11 +0000 Subject: [PATCH 078/101] Implement WorkerInterface Rust->C++ bridge --- build/deps/gen/deps.MODULE.bazel | 6 +- src/rust/kj/http.rs | 12 +- src/rust/worker/BUILD | 1 + src/rust/worker/bridge.h | 7 +- src/rust/worker/cxx_worker.rs | 194 +++++++++++++++++++++ src/rust/worker/error.rs | 2 +- src/rust/worker/ffi.c++ | 83 +++++++++ src/rust/worker/ffi.h | 52 ++++++ src/rust/worker/ffi.rs | 90 +++++++++- src/rust/worker/kill_switch.rs | 5 +- src/rust/worker/lib.rs | 7 +- src/rust/worker/ok.rs | 2 +- src/rust/worker/test.c++ | 170 ++++++++++++++++++ src/workerd/io/compatibility-date-test.c++ | 2 +- 14 files changed, 613 insertions(+), 20 deletions(-) create mode 100644 src/rust/worker/cxx_worker.rs create mode 100644 src/rust/worker/ffi.c++ diff --git a/build/deps/gen/deps.MODULE.bazel b/build/deps/gen/deps.MODULE.bazel index 9213a5cb63c..0fa66e6cb3e 100644 --- a/build/deps/gen/deps.MODULE.bazel +++ b/build/deps/gen/deps.MODULE.bazel @@ -136,10 +136,10 @@ bazel_dep(name = "tcmalloc", version = "0.0.0-20250927-12f2552") # workerd-cxx http.archive( name = "workerd-cxx", - sha256 = "d4db1075562a134a19dee7751adfc79bba95cad7740bb42d5621c4dfd78158d3", - strip_prefix = "cloudflare-workerd-cxx-5cb35c5", + sha256 = "f3213d568e0d135753da8e3f3c16c5a4f2c508c567d567a608e85ba61d98aa2c", + strip_prefix = "cloudflare-workerd-cxx-61285bf", type = "tgz", - url = "https://github.com/cloudflare/workerd-cxx/tarball/5cb35c57a904f3c949b13583a66a4ae55ebb6eda", + url = "https://github.com/cloudflare/workerd-cxx/tarball/61285bf53ffed51e4bc47d795f0e92461ce41ab4", ) use_repo(http, "workerd-cxx") diff --git a/src/rust/kj/http.rs b/src/rust/kj/http.rs index a04a122c1d6..3c4764d1868 100644 --- a/src/rust/kj/http.rs +++ b/src/rust/kj/http.rs @@ -271,6 +271,14 @@ impl HeadersRef<'_> { } } +impl<'a> HeadersRef<'a> { + /// The underlying `kj::HttpHeaders`, for passing to FFI functions that take a `const&`. + #[must_use] + pub fn as_ffi(self) -> &'a ffi::HttpHeaders { + self.0 + } +} + impl<'a> From<&'a ffi::HttpHeaders> for HeadersRef<'a> { fn from(value: &'a ffi::HttpHeaders) -> Self { HeadersRef(value) @@ -335,7 +343,7 @@ impl<'a> ServiceResponse<'a> { .into()) } - pub(crate) fn into_ffi(self) -> Pin<&'a mut ffi::HttpServiceResponse> { + pub fn into_ffi(self) -> Pin<&'a mut ffi::HttpServiceResponse> { self.0 } } @@ -383,7 +391,7 @@ impl<'a> ConnectResponse<'a> { .into()) } - pub(crate) fn into_ffi(self) -> Pin<&'a mut ffi::ConnectResponse> { + pub fn into_ffi(self) -> Pin<&'a mut ffi::ConnectResponse> { self.0 } } diff --git a/src/rust/worker/BUILD b/src/rust/worker/BUILD index 0c0f4d53909..747af2d138b 100644 --- a/src/rust/worker/BUILD +++ b/src/rust/worker/BUILD @@ -40,6 +40,7 @@ wd_rust_crate( wd_cc_library( name = "bridge", + srcs = ["ffi.c++"], hdrs = ["bridge.h"], deps = [ ":ffi.rs@cxx", diff --git a/src/rust/worker/bridge.h b/src/rust/worker/bridge.h index e9bdf57b024..7e4b4bb5c91 100644 --- a/src/rust/worker/bridge.h +++ b/src/rust/worker/bridge.h @@ -55,9 +55,8 @@ inline workerd::WorkerInterface::AlarmResult fromImpl( .retry = result.retry, .retryCountsAgainstLimit = result.retry_counts_against_limit, .outcome = kj::from(result.outcome), - .errorDescription = result.error_description.empty() - ? kj::none - : kj::Maybe(kj::str(result.error_description)), + .errorDescription = + result.error_description.map([](const ::rust::String& desc) { return kj::str(desc); }), }; } @@ -107,7 +106,7 @@ class RustWorkerInterface final: public workerd::WorkerInterface { } kj::Promise customEvent(kj::Own event) override { - co_return kj::from(co_await impl->custom_event(*event)); + co_return kj::from(co_await impl->custom_event(kj::mv(event))); } kj::Promise test() override { diff --git a/src/rust/worker/cxx_worker.rs b/src/rust/worker/cxx_worker.rs new file mode 100644 index 00000000000..0e32cc45d2f --- /dev/null +++ b/src/rust/worker/cxx_worker.rs @@ -0,0 +1,194 @@ +// Copyright (c) 2026 Cloudflare, Inc. +// Licensed under the Apache 2.0 license found in the LICENSE file or at: +// https://opensource.org/licenses/Apache-2.0 + +//! `CxxWorkerInterface`: wraps a C++ `workerd::WorkerInterface` and exposes it as a Rust +//! `worker::Interface`. +//! +//! Lets Rust code delegate to (decorate) a C++ worker. This is the reverse of `RustWorkerInterface` +//! (bridge.h), which exposes a Rust `worker::Interface` to C++. + +use std::pin::Pin; +use std::time::SystemTime; + +use kj::http::ConnectResponse; +use kj::http::ConnectSettings; +use kj::http::HeadersRef; +use kj::http::Method; +use kj::http::Service; +use kj::http::ServiceResponse; +use kj::io::AsyncInputStream; +use kj::io::AsyncIoStream; +use kj_rs::KjDate; +use kj_rs::KjOwn; +use outcome_capnp::EventOutcome; + +use crate::AlarmResult; +use crate::CustomEvent; +use crate::CustomEventResult; +use crate::Interface; +use crate::Result; +use crate::ScheduledResult; +use crate::ffi::bridge; + +/// Wraps an owned C++ `WorkerInterface`, delegating every event to it. +pub struct CxxWorkerInterface { + inner: KjOwn, +} + +impl CxxWorkerInterface { + #[must_use] + pub fn new(inner: KjOwn) -> Self { + Self { inner } + } +} + +#[async_trait::async_trait(?Send)] +impl Service for CxxWorkerInterface { + async fn request<'a>( + &'a mut self, + method: Method, + url: &'a [u8], + headers: HeadersRef<'a>, + request_body: Pin<&'a mut AsyncInputStream>, + response: ServiceResponse<'a>, + ) -> Result<()> { + // SAFETY: the returned future borrows self.inner and the arguments for its duration. + unsafe { + bridge::worker_request( + self.inner.as_mut(), + method, + url, + headers.as_ffi(), + request_body, + response.into_ffi(), + ) + } + .await?; + Ok(()) + } + + async fn connect<'a>( + &'a mut self, + host: &'a [u8], + headers: HeadersRef<'a>, + connection: Pin<&'a mut AsyncIoStream>, + response: ConnectResponse<'a>, + settings: ConnectSettings<'a>, + ) -> Result<()> { + // SAFETY: the returned future borrows self.inner and the arguments for its duration. + unsafe { + bridge::worker_connect( + self.inner.as_mut(), + host, + headers.as_ffi(), + connection, + response.into_ffi(), + settings, + ) + } + .await?; + Ok(()) + } +} + +#[async_trait::async_trait(?Send)] +impl Interface for CxxWorkerInterface { + async fn prewarm(&mut self, url: &str) -> Result<()> { + // SAFETY: the returned future borrows self.inner for its duration. + unsafe { bridge::worker_prewarm(self.inner.as_mut(), url.as_bytes()) }.await?; + Ok(()) + } + + async fn run_scheduled( + &mut self, + scheduled_time: &SystemTime, + cron: &str, + ) -> Result { + let nanos = KjDate::from(*scheduled_time).nanoseconds(); + // SAFETY: the returned future borrows self.inner for its duration. + let result = + unsafe { bridge::worker_run_scheduled(self.inner.as_mut(), nanos, cron.as_bytes()) } + .await?; + Ok(result.into()) + } + + async fn run_alarm( + &mut self, + scheduled_time: &SystemTime, + retry_count: u32, + ) -> Result { + let nanos = KjDate::from(*scheduled_time).nanoseconds(); + // SAFETY: the returned future borrows self.inner for its duration. + let result = + unsafe { bridge::worker_run_alarm(self.inner.as_mut(), nanos, retry_count) }.await?; + Ok(result.into()) + } + + async fn custom_event(&mut self, event: KjOwn) -> Result { + // SAFETY: the returned future borrows self.inner for its duration; event is moved in. + let result = unsafe { bridge::worker_custom_event(self.inner.as_mut(), event) }.await?; + Ok(result.into()) + } + + async fn test(&mut self) -> Result { + // SAFETY: the returned future borrows self.inner for its duration. + Ok(unsafe { bridge::worker_test(self.inner.as_mut()) }.await?) + } +} + +// Reverse of the conversions in ffi.rs: the bridge's shared result structs -> the crate's owned +// result types. +impl From for ScheduledResult { + fn from(value: bridge::ScheduledResult) -> Self { + Self { + retry: value.retry, + outcome: value.outcome.into(), + } + } +} + +impl From for AlarmResult { + fn from(value: bridge::AlarmResult) -> Self { + Self { + retry: value.retry, + retry_counts_against_limit: value.retry_counts_against_limit, + outcome: value.outcome.into(), + error_description: value.error_description.into(), + } + } +} + +impl From for CustomEventResult { + fn from(value: bridge::CustomEventResult) -> Self { + Self { + outcome: value.outcome.into(), + } + } +} + +impl From for EventOutcome { + fn from(value: bridge::EventOutcome) -> Self { + match value { + bridge::EventOutcome::Unknown => Self::Unknown, + bridge::EventOutcome::Ok => Self::Ok, + bridge::EventOutcome::Exception => Self::Exception, + bridge::EventOutcome::ExceededCpu => Self::ExceededCpu, + bridge::EventOutcome::KillSwitch => Self::KillSwitch, + bridge::EventOutcome::DaemonDown => Self::DaemonDown, + bridge::EventOutcome::ScriptNotFound => Self::ScriptNotFound, + bridge::EventOutcome::Canceled => Self::Canceled, + bridge::EventOutcome::ExceededMemory => Self::ExceededMemory, + bridge::EventOutcome::LoadShed => Self::LoadShed, + bridge::EventOutcome::ResponseStreamDisconnected => Self::ResponseStreamDisconnected, + bridge::EventOutcome::InternalError => Self::InternalError, + bridge::EventOutcome::ExceededWallTime => Self::ExceededWallTime, + other => { + unreachable!( + "unknown WorkerInterface::EventOutcome discriminant: {}", + other.repr + ) + } + } + } +} diff --git a/src/rust/worker/error.rs b/src/rust/worker/error.rs index 3cd109f26f4..9cf153ebed9 100644 --- a/src/rust/worker/error.rs +++ b/src/rust/worker/error.rs @@ -96,7 +96,7 @@ impl Interface for Worker { async fn custom_event( &mut self, - _event: Pin<&mut crate::CustomEvent>, + _event: crate::KjOwn, ) -> crate::Result { Err(self.error(file!(), line!())) } diff --git a/src/rust/worker/ffi.c++ b/src/rust/worker/ffi.c++ new file mode 100644 index 00000000000..f73fe11ae7e --- /dev/null +++ b/src/rust/worker/ffi.c++ @@ -0,0 +1,83 @@ +// Copyright (c) 2026 Cloudflare, Inc. +// Licensed under the Apache 2.0 license found in the LICENSE file or at: +// https://opensource.org/licenses/Apache-2.0 + +#include "ffi.h" + +#include + +#include + +// Out-of-line reverse-direction shims. These return the cxx shared result structs, which are only +// complete in the generated bridge header (ffi.rs.h) -- and that header includes ffi.h, so ffi.h +// itself cannot include it. See kj/ffi.c++ for the same pattern with HttpConnectSettings. + +namespace workerd::rust::worker { + +// C++ EventOutcome -> the bridge's shared EventOutcome enum (reverse of bridge.h's fromImpl). +static EventOutcome toRustOutcome(workerd::EventOutcome outcome) { + switch (outcome) { + case workerd::EventOutcome::UNKNOWN: + return EventOutcome::Unknown; + case workerd::EventOutcome::OK: + return EventOutcome::Ok; + case workerd::EventOutcome::EXCEPTION: + return EventOutcome::Exception; + case workerd::EventOutcome::EXCEEDED_CPU: + return EventOutcome::ExceededCpu; + case workerd::EventOutcome::KILL_SWITCH: + return EventOutcome::KillSwitch; + case workerd::EventOutcome::DAEMON_DOWN: + return EventOutcome::DaemonDown; + case workerd::EventOutcome::SCRIPT_NOT_FOUND: + return EventOutcome::ScriptNotFound; + case workerd::EventOutcome::CANCELED: + return EventOutcome::Canceled; + case workerd::EventOutcome::EXCEEDED_MEMORY: + return EventOutcome::ExceededMemory; + case workerd::EventOutcome::LOAD_SHED: + return EventOutcome::LoadShed; + case workerd::EventOutcome::RESPONSE_STREAM_DISCONNECTED: + return EventOutcome::ResponseStreamDisconnected; + case workerd::EventOutcome::INTERNAL_ERROR: + return EventOutcome::InternalError; + case workerd::EventOutcome::EXCEEDED_WALL_TIME: + return EventOutcome::ExceededWallTime; + } + KJ_UNREACHABLE; +} + +kj::Promise worker_run_scheduled( + WorkerInterface& worker, int64_t scheduledTimeNanos, ::rust::Slice cron) { + auto cronStr = kj::str(kj::from(cron).asChars()); + auto result = co_await worker.runScheduled(kj_rs::repr::fromNanos(scheduledTimeNanos), cronStr); + co_return ScheduledResult{ + .retry = result.retry, + .outcome = toRustOutcome(result.outcome), + }; +} + +kj::Promise worker_run_alarm( + WorkerInterface& worker, int64_t scheduledTimeNanos, uint32_t retryCount) { + auto result = co_await worker.runAlarm(kj_rs::repr::fromNanos(scheduledTimeNanos), retryCount); + kj::Maybe<::rust::String> errorDescription; + KJ_IF_SOME(desc, result.errorDescription) { + errorDescription = ::rust::String(desc.begin(), desc.size()); + } + co_return AlarmResult{ + .retry = result.retry, + .retry_counts_against_limit = result.retryCountsAgainstLimit, + .outcome = toRustOutcome(result.outcome), + .error_description = kj::mv(errorDescription), + }; +} + +kj::Promise worker_custom_event( + WorkerInterface& worker, kj::Own event) { + auto result = co_await worker.customEvent(kj::mv(event)); + co_return CustomEventResult{ + .outcome = toRustOutcome(result.outcome), + }; +} + +} // namespace workerd::rust::worker diff --git a/src/rust/worker/ffi.h b/src/rust/worker/ffi.h index aeb954e9837..90a43fbd61e 100644 --- a/src/rust/worker/ffi.h +++ b/src/rust/worker/ffi.h @@ -20,5 +20,57 @@ using AsyncIoStream = kj::AsyncIoStream; using ConnectResponse = kj::HttpService::ConnectResponse; using CustomEvent = workerd::WorkerInterface::CustomEvent; +using WorkerInterface = workerd::WorkerInterface; + +// Shared result structs, defined in the generated bridge header (which includes this file). Forward +// declared here so the out-of-line shim declarations below can name them. +struct ScheduledResult; +struct AlarmResult; +struct CustomEventResult; + +// Reverse-direction shims: call a C++ WorkerInterface from Rust. Methods returning shared result +// structs (run_scheduled / run_alarm / custom_event) are declared here but defined out-of-line in +// ffi.c++, since those structs are only complete in the generated bridge header. + +inline kj::Promise worker_request(WorkerInterface& worker, + HttpMethod method, + ::rust::Slice url, + const HttpHeaders& headers, + AsyncInputStream& requestBody, + HttpServiceResponse& response) { + auto urlStr = kj::str(kj::from(url).asChars()); + co_await worker.request(method, urlStr, headers, requestBody, response); +} + +inline kj::Promise worker_connect(WorkerInterface& worker, + ::rust::Slice host, + const HttpHeaders& headers, + AsyncIoStream& connection, + ConnectResponse& response, + kj::rust::HttpConnectSettings settings) { + auto hostStr = kj::str(kj::from(host).asChars()); + co_await worker.connect(hostStr, headers, connection, response, + { + .useTls = settings.use_tls, + .tlsStarter = settings.tls_starter, + }); +} + +inline kj::Promise worker_prewarm( + WorkerInterface& worker, ::rust::Slice url) { + auto urlStr = kj::str(kj::from(url).asChars()); + co_await worker.prewarm(urlStr); +} + +inline kj::Promise worker_test(WorkerInterface& worker) { + return worker.test(); +} + +kj::Promise worker_run_scheduled( + WorkerInterface& worker, int64_t scheduledTimeNanos, ::rust::Slice cron); +kj::Promise worker_run_alarm( + WorkerInterface& worker, int64_t scheduledTimeNanos, uint32_t retryCount); +kj::Promise worker_custom_event( + WorkerInterface& worker, kj::Own event); } // namespace workerd::rust::worker diff --git a/src/rust/worker/ffi.rs b/src/rust/worker/ffi.rs index 9a5670c130c..b48b30981c1 100644 --- a/src/rust/worker/ffi.rs +++ b/src/rust/worker/ffi.rs @@ -6,6 +6,14 @@ use std::pin::Pin; use kj::http::ConnectSettings; use kj_rs::KjDate; +// `KjMaybe` is only referenced inside the cxx bridge module below, but the macro expansion needs +// the type in scope. +#[expect( + unused_imports, + reason = "referenced inside the #[cxx::bridge] macro expansion" +)] +use kj_rs::KjMaybe; +use kj_rs::KjOwn; use crate::CustomEvent; use crate::Result; @@ -14,6 +22,10 @@ use crate::ffi::bridge::CustomEventResult; use crate::ffi::bridge::ScheduledResult; #[cxx::bridge(namespace = "workerd::rust::worker")] +#[expect( + clippy::missing_safety_doc, + reason = "cxx bridge extern decls; safety is uniform" +)] pub mod bridge { #[namespace = "kj::rust"] unsafe extern "C++" { @@ -49,13 +61,13 @@ pub mod bridge { pub outcome: EventOutcome, } - #[derive(Debug, Clone, PartialEq, Eq)] + #[derive(Debug)] // keep in sync with `src/workerd/io/worker-interface.h` pub struct AlarmResult { pub retry: bool, pub retry_counts_against_limit: bool, pub outcome: EventOutcome, - pub error_description: String, + pub error_description: KjMaybe, } #[derive(Debug, Clone, PartialEq, Eq)] @@ -72,6 +84,15 @@ pub mod bridge { extern "Rust" { type Wrapper; + // Wrap a C++ WorkerInterface as a Rust worker::Interface (via CxxWorkerInterface), then + // re-expose it to C++ as a WorkerInterface. On its own this is an identity decorator; it + // exists so C++ can hand a worker to Rust for decoration and get a WorkerInterface back. + #[expect( + clippy::unnecessary_box_returns, + reason = "c++ expects heap-allocation" + )] + fn new_cxx_worker(inner: KjOwn) -> Box; + async unsafe fn request<'a>( self: &'a mut Wrapper, method: HttpMethod, @@ -106,7 +127,7 @@ pub mod bridge { async unsafe fn custom_event<'a>( self: &'a mut Wrapper, - event: Pin<&'a mut CustomEvent>, + event: KjOwn, ) -> Result; async unsafe fn test<'a>(self: &'a mut Wrapper) -> Result; @@ -120,6 +141,57 @@ pub mod bridge { type CustomEvent; } + // Reverse direction: call a C++ `workerd::WorkerInterface` from Rust. Lets a Rust + // `worker::Interface` decorate/delegate to a C++ worker (see `CxxWorkerInterface`). + unsafe extern "C++" { + type WorkerInterface; + + async unsafe fn worker_request<'a>( + worker: Pin<&'a mut WorkerInterface>, + method: HttpMethod, + url: &'a [u8], + headers: &'a HttpHeaders, + request_body: Pin<&'a mut AsyncInputStream>, + response: Pin<&'a mut HttpServiceResponse>, + ) -> Result<()>; + + async unsafe fn worker_connect<'a>( + worker: Pin<&'a mut WorkerInterface>, + host: &'a [u8], + headers: &'a HttpHeaders, + connection: Pin<&'a mut AsyncIoStream>, + response: Pin<&'a mut ConnectResponse>, + settings: HttpConnectSettings<'a>, + ) -> Result<()>; + + async unsafe fn worker_prewarm<'a>( + worker: Pin<&'a mut WorkerInterface>, + url: &'a [u8], + ) -> Result<()>; + + // Dates cross as i64 nanoseconds since the Unix epoch (kj_rs::repr::{to,from}Nanos), to + // avoid marshalling KjDate through an extern "C++" boundary. + async unsafe fn worker_run_scheduled<'a>( + worker: Pin<&'a mut WorkerInterface>, + scheduled_time_nanos: i64, + cron: &'a [u8], + ) -> Result; + + async unsafe fn worker_run_alarm<'a>( + worker: Pin<&'a mut WorkerInterface>, + scheduled_time_nanos: i64, + retry_count: u32, + ) -> Result; + + // Takes ownership of the event and forwards it to the C++ worker. + async unsafe fn worker_custom_event( + worker: Pin<&mut WorkerInterface>, + event: KjOwn, + ) -> Result; + + async unsafe fn worker_test<'a>(worker: Pin<&'a mut WorkerInterface>) -> Result; + } + impl Box {} } @@ -127,6 +199,14 @@ pub struct Wrapper { worker: Box, } +#[expect( + clippy::unnecessary_box_returns, + reason = "c++ expects heap-allocation" +)] +fn new_cxx_worker(inner: kj_rs::KjOwn) -> Box { + crate::Interface::into_ffi(crate::CxxWorkerInterface::new(inner)) +} + impl Wrapper { pub(crate) fn new(worker: Box) -> Self { Self { worker } @@ -186,7 +266,7 @@ impl Wrapper { Ok(result.into()) } - async fn custom_event(&mut self, event: Pin<&mut CustomEvent>) -> Result { + async fn custom_event(&mut self, event: KjOwn) -> Result { let result = self.worker.custom_event(event).await?; Ok(result.into()) } @@ -212,7 +292,7 @@ impl From for bridge::AlarmResult { retry: value.retry, retry_counts_against_limit: value.retry_counts_against_limit, outcome: value.outcome.into(), - error_description: value.error_description.unwrap_or_default(), + error_description: value.error_description.into(), } } } diff --git a/src/rust/worker/kill_switch.rs b/src/rust/worker/kill_switch.rs index 307aab33031..df7d08e0202 100644 --- a/src/rust/worker/kill_switch.rs +++ b/src/rust/worker/kill_switch.rs @@ -102,7 +102,10 @@ impl Interface for Worker { }) } - async fn custom_event(&mut self, _event: Pin<&mut CustomEvent>) -> Result { + async fn custom_event( + &mut self, + _event: crate::KjOwn, + ) -> Result { Ok(CustomEventResult { outcome: EventOutcome::KillSwitch, }) diff --git a/src/rust/worker/lib.rs b/src/rust/worker/lib.rs index 9bf8e9c8b01..1d50d23906f 100644 --- a/src/rust/worker/lib.rs +++ b/src/rust/worker/lib.rs @@ -13,13 +13,13 @@ //! Current limitations: //! - most parameters are opaque c++ types +pub mod cxx_worker; pub mod error; pub mod exception; pub mod ffi; pub mod kill_switch; pub mod ok; -use std::pin::Pin; use std::time::SystemTime; use cxx::KjError; @@ -33,10 +33,13 @@ pub use kj::http::Service; pub use kj::http::ServiceResponse; pub use kj::io::AsyncInputStream; pub use kj::io::AsyncIoStream; +pub use kj_rs::KjOwn; use outcome_capnp::EventOutcome; +pub use crate::cxx_worker::CxxWorkerInterface; pub use crate::ffi::Wrapper; pub use crate::ffi::bridge::CustomEvent; +pub use crate::ffi::bridge::WorkerInterface; pub type Result = std::result::Result; @@ -90,7 +93,7 @@ pub trait Interface: kj::http::Service { /// immediately; if its callbacks have not run yet, they will not run at all. So, a `CustomEvent` /// implementation can hold references to objects it doesn't own as long as the returned promise /// will be canceled before those objects go away. - async fn custom_event(&mut self, event: Pin<&mut CustomEvent>) -> Result; + async fn custom_event(&mut self, event: KjOwn) -> Result; /// Convert `self` into the structure suitable for passing to C++ through FFI layer /// To obtain `workerd::WorkerInterface` on C++ side finish wrapping with `fromRust()` diff --git a/src/rust/worker/ok.rs b/src/rust/worker/ok.rs index 965bc7dd609..82a8e1b8f0e 100644 --- a/src/rust/worker/ok.rs +++ b/src/rust/worker/ok.rs @@ -93,7 +93,7 @@ impl Interface for Worker { async fn custom_event( &mut self, - _event: Pin<&mut crate::CustomEvent>, + _event: crate::KjOwn, ) -> crate::Result { Err(Self::not_implemented("custom_event")) } diff --git a/src/rust/worker/test.c++ b/src/rust/worker/test.c++ index 9b34ffdd651..1bffe545cd3 100644 --- a/src/rust/worker/test.c++ +++ b/src/rust/worker/test.c++ @@ -323,5 +323,175 @@ KJ_TEST("error worker test") { KJ_ASSERT(!worker->test().wait(waitScope)); } +// ====================================================================================== +// Reverse direction: a Rust CxxWorkerInterface wrapping (delegating to) a C++ WorkerInterface. + +namespace { + +// Records the calls it receives, and returns distinctive values so the test can confirm the call +// travelled C++ stub -> Rust CxxWorkerInterface -> back to C++. +class StubWorker final: public WorkerInterface { + public: + bool prewarmCalled = false; + kj::String prewarmUrl; + bool runScheduledCalled = false; + bool runAlarmCalled = false; + bool customEventCalled = false; + bool testCalled = false; + bool connectCalled = false; + kj::String connectHost; + + kj::Promise request(kj::HttpMethod method, + kj::StringPtr url, + const kj::HttpHeaders& headers, + kj::AsyncInputStream& requestBody, + Response& response) override { + auto body = response.send(200, "OK", headers, kj::str("STUB").size()); + return body->write("STUB"_kjb).attach(kj::mv(body)); + } + + kj::Promise connect(kj::StringPtr host, + const kj::HttpHeaders& headers, + kj::AsyncIoStream& connection, + ConnectResponse& response, + kj::HttpConnectSettings settings) override { + connectCalled = true; + connectHost = kj::str(host); + response.accept(200, "OK", headers); + return kj::READY_NOW; + } + + kj::Promise prewarm(kj::StringPtr url) override { + prewarmCalled = true; + prewarmUrl = kj::str(url); + return kj::READY_NOW; + } + + kj::Promise runScheduled(kj::Date scheduledTime, kj::StringPtr cron) override { + runScheduledCalled = true; + return ScheduledResult{.retry = true, .outcome = workerd::EventOutcome::OK}; + } + + kj::Promise runAlarm(kj::Date scheduledTime, uint32_t retryCount) override { + runAlarmCalled = true; + return AlarmResult{ + .retry = true, .retryCountsAgainstLimit = false, .outcome = workerd::EventOutcome::OK}; + } + + kj::Promise customEvent(kj::Own event) override { + customEventCalled = true; + return CustomEvent::Result{.outcome = workerd::EventOutcome::OK}; + } + + kj::Promise test() override { + testCalled = true; + return true; + } +}; + +} // namespace + +KJ_TEST("cxx_worker delegates non-HTTP events to the wrapped C++ worker") { + kj::EventLoop loop; + kj::WaitScope waitScope(loop); + + auto stub = kj::heap(); + auto& stubRef = *stub; + // Wrap the C++ stub as a Rust worker::Interface, then re-expose it to C++. + auto worker = kj::from(new_cxx_worker(kj::mv(stub))); + + worker->prewarm("/warm"_kj).wait(waitScope); + KJ_ASSERT(stubRef.prewarmCalled); + KJ_ASSERT(stubRef.prewarmUrl == "/warm"); + + auto scheduled = + worker->runScheduled(kj::UNIX_EPOCH + 1000 * kj::SECONDS, "0 0 * * *"_kj).wait(waitScope); + KJ_ASSERT(stubRef.runScheduledCalled); + KJ_ASSERT(scheduled.retry == true); + KJ_ASSERT(scheduled.outcome == workerd::EventOutcome::OK); + + auto alarm = worker->runAlarm(kj::UNIX_EPOCH + 2000 * kj::SECONDS, 3).wait(waitScope); + KJ_ASSERT(stubRef.runAlarmCalled); + KJ_ASSERT(alarm.retry == true); + KJ_ASSERT(alarm.retryCountsAgainstLimit == false); + KJ_ASSERT(alarm.outcome == workerd::EventOutcome::OK); + + KJ_ASSERT(worker->test().wait(waitScope)); + KJ_ASSERT(stubRef.testCalled); +} + +KJ_TEST("cxx_worker delegates customEvent (ownership transfer) to the wrapped C++ worker") { + kj::EventLoop loop; + kj::WaitScope waitScope(loop); + + class TestCustomEvent: public workerd::WorkerInterface::CustomEvent { + public: + virtual ~TestCustomEvent() = default; + + kj::Promise run(kj::Own incomingRequest, + kj::Maybe entrypointName, + kj::Maybe versionInfo, + workerd::Frankenvalue props, + kj::TaskSet& waitUntilTasks, + bool) override { + KJ_UNIMPLEMENTED(); + } + kj::Promise sendRpc(capnp::HttpOverCapnpFactory& httpOverCapnpFactory, + capnp::ByteStreamFactory& byteStreamFactory, + workerd::FrankenvalueHandler& frankenvalueHandler, + workerd::rpc::EventDispatcher::Client dispatcher) override { + KJ_UNIMPLEMENTED(); + } + kj::Promise notSupported() override { + KJ_UNIMPLEMENTED(); + } + uint16_t getType() override { + return 42; + } + workerd::tracing::EventInfo getEventInfo() const override { + return workerd::tracing::CustomEventInfo(); + } + }; + + auto stub = kj::heap(); + auto& stubRef = *stub; + auto worker = kj::from(new_cxx_worker(kj::mv(stub))); + + auto result = worker->customEvent(kj::heap()).wait(waitScope); + KJ_ASSERT(stubRef.customEventCalled); + KJ_ASSERT(result.outcome == workerd::EventOutcome::OK); +} + +KJ_TEST("cxx_worker delegates request to the wrapped C++ worker") { + kj::EventLoop loop; + kj::WaitScope waitScope(loop); + + auto client = newClient(new_cxx_worker(kj::heap())); + kj::HttpHeaderTable headerTable; + kj::HttpHeaders headers(headerTable); + + auto response = + client->request(kj::HttpMethod::GET, "http://test/"_kj, headers).response.wait(waitScope); + KJ_ASSERT(response.statusCode == 200); + KJ_ASSERT(response.body->readAllText().wait(waitScope) == "STUB"); +} + +KJ_TEST("cxx_worker delegates connect to the wrapped C++ worker") { + kj::EventLoop loop; + kj::WaitScope waitScope(loop); + + auto stub = kj::heap(); + auto& stubRef = *stub; + auto client = newClient(new_cxx_worker(kj::mv(stub))); + kj::HttpHeaderTable headerTable; + kj::HttpHeaders headers(headerTable); + + auto response = client->connect("example.com:443"_kj, headers, kj::HttpConnectSettings{}) + .status.wait(waitScope); + KJ_ASSERT(stubRef.connectCalled); + KJ_ASSERT(stubRef.connectHost == "example.com:443"); + KJ_ASSERT(response.statusCode == 200); +} + } // namespace } // namespace workerd diff --git a/src/workerd/io/compatibility-date-test.c++ b/src/workerd/io/compatibility-date-test.c++ index 582ed147795..8e27bcb4315 100644 --- a/src/workerd/io/compatibility-date-test.c++ +++ b/src/workerd/io/compatibility-date-test.c++ @@ -418,7 +418,7 @@ KJ_TEST("encode to full flag list") { SimpleWorkerErrorReporter errorReporter; compileCompatibilityFlags(compatDate, flagList.asReader(), output, errorReporter, - /*experimental=*/false, CompatibilityDateValidation::FUTURE_FOR_TEST, nullptr); + /*allowExperimentalFeatures=*/false, CompatibilityDateValidation::FUTURE_FOR_TEST, nullptr); KJ_ASSERT(errorReporter.errors.empty()); return kj::mv(outputOrphan); From a30d6b9e83df59b8b3746cef0d8e1c18030d354e Mon Sep 17 00:00:00 2001 From: Felix Hanau Date: Thu, 7 May 2026 20:46:58 -0400 Subject: [PATCH 079/101] EW-9327 Implement connection string override support Allow connectionStringOverride for regular fetchers Implement ctx.mapVirtualHost Add tests for connectionStringOverride, serialization Use .workers.alt host Gate new mapVirtualHost API behind experimental flag --- src/node/internal/internal_dns.ts | 5 ++- src/workerd/api/global-scope.c++ | 12 +++++ src/workerd/api/global-scope.h | 7 +++ src/workerd/api/http.c++ | 33 ++++++++++++++ src/workerd/api/http.h | 65 +++++++++++++++++++++++++++- src/workerd/api/sockets.c++ | 7 +++ src/workerd/api/tests/js-rpc-test.js | 64 +++++++++++++++++++++++++++ 7 files changed, 191 insertions(+), 2 deletions(-) diff --git a/src/node/internal/internal_dns.ts b/src/node/internal/internal_dns.ts index 9672489389d..f34abee85fe 100644 --- a/src/node/internal/internal_dns.ts +++ b/src/node/internal/internal_dns.ts @@ -56,7 +56,10 @@ let defaultDnsOrder: DnsOrder = 'verbatim'; // override. Gate on the suffix so ordinary lookups stay entirely in JS rather than crossing into // C++ on every resolution. function getMagicHostOverride(hostname: string): string | undefined { - if (!hostname.endsWith('.hyperdrive.local')) { + if ( + !hostname.endsWith('.hyperdrive.local') && + !hostname.endsWith('.workers.alt') + ) { return undefined; } return inner.getCallerDnsOverride(hostname); diff --git a/src/workerd/api/global-scope.c++ b/src/workerd/api/global-scope.c++ index fca7885d930..ec8589f46f6 100644 --- a/src/workerd/api/global-scope.c++ +++ b/src/workerd/api/global-scope.c++ @@ -147,6 +147,18 @@ jsg::Optional> ExecutionContext::getAccess(jsg::Lock& js } return kj::none; } + +kj::String ExecutionContext::mapVirtualHost( + jsg::Lock& js, jsg::Ref fetcher, uint16_t port) { + // Set up override, or return existing magic hostname if override already exists. Overrides + // created using mapVirtualHost() should never use the legacy hyperdrive hostname, so the only way + // for that hostname to be used is if an override was already registered earlier using + // ExtendedFetcher. + fetcher->registerOverride(js, IsHyperdrive::NO, port); + return kj::str(KJ_ASSERT_NONNULL(fetcher->getHostInternal()), ":", + KJ_ASSERT_NONNULL(fetcher->getPortInternal())); +} + void ExecutionContext::abort(jsg::Lock& js, jsg::Optional reason) { KJ_IF_SOME(r, reason) { IoContext::current().abort(js.exceptionToKj(kj::mv(r))); diff --git a/src/workerd/api/global-scope.h b/src/workerd/api/global-scope.h index a4ce5dd46e0..91a73b19e38 100644 --- a/src/workerd/api/global-scope.h +++ b/src/workerd/api/global-scope.h @@ -371,6 +371,10 @@ class ExecutionContext: public jsg::Object { // Called by the runtime to provide Cloudflare Access authentication context. jsg::Optional> getAccess(jsg::Lock& js); + // Maps a virtual host to the given fetcher on the specified port. This is a no-op if an override + // has already been installed. Returns the string for the override. + kj::String mapVirtualHost(jsg::Lock& js, jsg::Ref fetcher, uint16_t port); + JSG_RESOURCE_TYPE(ExecutionContext, CompatibilityFlags::Reader flags) { JSG_METHOD(waitUntil); JSG_METHOD(passThroughOnException); @@ -386,6 +390,9 @@ class ExecutionContext: public jsg::Object { JSG_LAZY_INSTANCE_PROPERTY(version, getVersion); } JSG_LAZY_INSTANCE_PROPERTY(access, getAccess); + if (flags.getWorkerdExperimental()) { + JSG_METHOD(mapVirtualHost); + } // ctx.tracing - user tracing API. Always available; the Tracing object is stateless // and enterSpan() is a no-op when called outside a traced request. diff --git a/src/workerd/api/http.c++ b/src/workerd/api/http.c++ index 69d3cac247b..449e1fd63eb 100644 --- a/src/workerd/api/http.c++ +++ b/src/workerd/api/http.c++ @@ -14,6 +14,7 @@ #include "worker-rpc.h" #include "workerd/jsg/jsvalue.h" +#include #include #include #include @@ -2027,6 +2028,38 @@ jsg::Ref Fetcher::connect( return connectImpl(js, JSG_THIS, kj::mv(address), kj::mv(options)); } +void Fetcher::registerOverride(jsg::Lock& js, IsHyperdrive isHyperdrive, uint16_t overridePort) { + if (registeredConnectOverride) { + return; + } + KJ_DASSERT(host == kj::none); + kj::FixedArray randomBytes; + workerd::getEntropy(randomBytes); + host = kj::str(kj::encodeHex(randomBytes), isHyperdrive ? ".hyperdrive.local" : ".workers.alt"); + + // Register an entry in the global scope connectOverrides HashMap so that cloudflare:sockets's + // connect() will route connections to this magic hostname through the fetcher. + auto& globalScope = IoContext::current().getCurrentLock().getGlobalScope(); + globalScope.setConnectOverride( + kj::str(KJ_ASSERT_NONNULL(host), ":", overridePort), + [self = JSG_THIS](jsg::Lock& js) mutable { + return self->connect( + js, + kj::str(KJ_ASSERT_NONNULL(self->host), ":", KJ_ASSERT_NONNULL(self->port)), + kj::none); + }); + // port may already be set for extended fetchers, in that case this is a no-op + port = overridePort; + registeredConnectOverride = true; +} + +kj::StringPtr ExtendedFetcher::getHost(jsg::Lock& js) { + // Ensures the connect override is registered on the global scope using registerOverride(), then + // returns the random hostname. + registerOverride(js, isHyperdrive, KJ_ASSERT_NONNULL(port)); + return KJ_ASSERT_NONNULL(host); +} + jsg::Promise> Fetcher::fetch(jsg::Lock& js, kj::OneOf, kj::String> requestOrUrl, jsg::Optional>> requestInit) { diff --git a/src/workerd/api/http.h b/src/workerd/api/http.h index 5c97f5b4da1..37a1b3e34aa 100644 --- a/src/workerd/api/http.h +++ b/src/workerd/api/http.h @@ -19,11 +19,14 @@ #include #include #include +#include #include namespace workerd::api { +WD_STRONG_BOOL(IsHyperdrive); + // Base class for Request and Response. In JavaScript, this class is a mixin, meaning no one will // be instantiating objects of this type -- it exists solely to house body-related functionality // common to both Requests and Responses. @@ -447,6 +450,28 @@ class Fetcher: public JsRpcClientProvider { JSG_SERIALIZABLE(rpc::SerializationTag::SERVICE_STUB); + // Set up a connection string override for a random host and the given port. isHyperdrive + // signifies whether the legacy hostname for hyperdrive should be used (only used with + // ExtendedFetcher). + void registerOverride(jsg::Lock& js, IsHyperdrive isHyperdrive, uint16_t overridePort); + + kj::Maybe getPortInternal() const { + return port; + } + kj::Maybe getHostInternal() const { + return host; + } + bool hasConnectOverride() const { + return registeredConnectOverride; + } + + protected: + // The connection string override is lazily initialized (when ctx.mapVirtualHost is invoked for + // the fetcher or when host is accessed for ExtendedFetcher), so the host/port may be none. + bool registeredConnectOverride = false; + kj::Maybe host; + kj::Maybe port; + private: kj::OneOf, @@ -457,6 +482,44 @@ class Fetcher: public JsRpcClientProvider { bool isInHouse; }; +// Extended fetcher type that makes the getters for host/port available using the JSG API. Allows +// us to support connection string overrides in a backwards-compatible way. +class ExtendedFetcher final: public Fetcher { + public: + explicit ExtendedFetcher(uint channel, + RequiresHostAndProtocol requiresHost, + uint16_t connectionStringOverridePort, + IsHyperdrive isHyperdrive, + bool isInHouse = false) + : Fetcher(channel, requiresHost, isInHouse), + isHyperdrive(isHyperdrive) { + port = connectionStringOverridePort; + } + + // Get port – this is guaranteed to be present for extended fetchers + uint16_t getPort() const { + return KJ_ASSERT_NONNULL(port); + } + // Get host and set up override, if needed. + kj::StringPtr getHost(jsg::Lock& js); + + JSG_RESOURCE_TYPE(ExtendedFetcher) { + JSG_INHERIT(Fetcher); + + JSG_LAZY_READONLY_INSTANCE_PROPERTY(host, getHost); + JSG_LAZY_READONLY_INSTANCE_PROPERTY(port, getPort); + } + + void serialize(jsg::Lock& js, jsg::Serializer& serializer) { + return Fetcher::serialize(js, serializer); + } + + JSG_ONEWAY_SERIALIZABLE(rpc::SerializationTag::SERVICE_STUB); + + private: + IsHyperdrive isHyperdrive; +}; + // Type of the second parameter to Request's constructor. Also the type of the second parameter // to fetch(). // @@ -1153,7 +1216,7 @@ jsg::Ref makeHttpResponse(jsg::Lock& js, api::Headers::ValueIterator::Next, api::Body, api::Response, api::Response::InitializerDict, \ api::Request, api::Request::InitializerDict, api::Fetcher, api::Fetcher::PutOptions, \ api::Fetcher::ScheduledOptions, api::Fetcher::ScheduledResult, api::Fetcher::QueueResult, \ - api::Fetcher::ServiceBindingQueueMessage + api::Fetcher::ServiceBindingQueueMessage, api::ExtendedFetcher // The list of http.h types that are added to worker.c++'s JSG_DECLARE_ISOLATE_TYPE } // namespace workerd::api diff --git a/src/workerd/api/sockets.c++ b/src/workerd/api/sockets.c++ index e39294637f8..aaae9b78de8 100644 --- a/src/workerd/api/sockets.c++ +++ b/src/workerd/api/sockets.c++ @@ -551,6 +551,13 @@ void Socket::handleProxyStatus( LOG_WARNING_PERIODICALLY( "attempt to connect to Hyperdrive failed to trigger connectOverride", self->remoteAddress, status.statusCode, status.statusText); + } else if (self->remoteAddress.orDefault(kj::String()).contains(".workers.alt"_kj)) { + // No attempts to connect to Hyperdrive should end up here, since they go through the other + // version of handleProxyStatus. If they end up here somehow, log about it to get some + // context that can aid in debugging. + LOG_WARNING_PERIODICALLY( + "attempt to use connectionStringOverride failed to trigger connectOverride", + self->remoteAddress, status.statusCode, status.statusText); } self->handleProxyError(js, JSG_KJ_EXCEPTION(FAILED, Error, msg)); } else { diff --git a/src/workerd/api/tests/js-rpc-test.js b/src/workerd/api/tests/js-rpc-test.js index d97d57699a3..43ef048c4d9 100644 --- a/src/workerd/api/tests/js-rpc-test.js +++ b/src/workerd/api/tests/js-rpc-test.js @@ -12,6 +12,7 @@ import { RpcTarget, ServiceStub, } from 'cloudflare:workers'; +import { connect } from 'cloudflare:sockets'; try { waitUntil(null); @@ -2160,6 +2161,16 @@ export class Greeter extends WorkerEntrypoint { async greet(name) { return `${this.ctx.props.greeting}, ${name}!`; } + // Expose ctx.mapVirtualHost to callers so we can test it on fetchers received over RPC. + async mapVirtualHost(fetcher, port) { + return this.ctx.mapVirtualHost(fetcher, port); + } + + // Make an RPC call through the provided fetcher to verify it still works after being mapped + // (or after being received over RPC). + async greetThrough(fetcher) { + return await fetcher.greet('foo'); + } } export class GreeterFactory extends WorkerEntrypoint { @@ -2275,3 +2286,56 @@ export let stubDepthLimitTest = { ); }, }; + +// Test connection string override via mapVirtualHost(). +export let mapVirtualHostIdempotent = { + async test(controller, env, ctx) { + let host1 = ctx.mapVirtualHost(env.MyService, 5432); + assert.ok(typeof host1 === 'string'); + // Overrides that are first created using mapVirtualHost() always use .workers.alt. + assert.match(host1, /^[0-9a-f]{32}\.workers\.alt:5432$/); + + // Once the override is installed, it doesn't change in future calls. + let host2 = ctx.mapVirtualHost(env.MyService, 1234); + assert.strictEqual(host1, host2); + + // Different fetchers should be assigned distinct virtual hosts. + let host3 = ctx.mapVirtualHost(env.self, 5432); + assert.notStrictEqual(host1, host3); + }, +}; + +export let mapVirtualHostConnect = { + async test(controller, env, ctx) { + // Test that we can connect to the magic host obtained via mapVirtualHost(). + let magicHost = ctx.mapVirtualHost(env.MyService, 5432); + assert.ok(typeof magicHost === 'string'); + assert.match(magicHost, /^[0-9a-f]{32}\.workers\.alt:5432$/); + + let socket = await connect(magicHost); + await socket.opened; + const dec = new TextDecoder(); + let result = ''; + for await (const chunk of socket.readable) { + result += dec.decode(chunk, { stream: true }); + } + result += dec.decode(); + assert.strictEqual(result, 'hello'); + await socket.closed; + }, +}; + +export let mapVirtualHostSerialization = { + async test(controller, env, ctx) { + // A fetcher with a connection-string override can still be serialized over RPC and remains + // usable for RPC on the other side. + let greeter = await env.GreeterFactory.makeGreeter('Yo'); + let magicHost = ctx.mapVirtualHost(greeter, 5432); + assert.strictEqual(await greeter.greetThrough(greeter), 'Yo, foo!'); + + // A fetcher that was serialized and passed over jsRpc does not remember the magic host it was + // set up with before + let magicHostSerial = await greeter.mapVirtualHost(greeter, 5432); + assert.notStrictEqual(magicHost, magicHostSerial); + }, +}; From fec7456d8cb4dd8001d07e06b9e1aea4769ec75e Mon Sep 17 00:00:00 2001 From: Dan Carney Date: Tue, 14 Jul 2026 12:04:42 +0000 Subject: [PATCH 080/101] Upgrade to V8 15.1 * Update V8 to 15.1.206.7 Bump workerd's pinned V8 from 15.0.245.14 to 15.1.206.7 (the head of the 15.1 release branch), matching the version edgeworker now uses. - Rebased the V8 patches onto 15.1.206.7. Notable conflict resolutions: - Promise Context Tagging: CreateJSPromiseObject moved from maglev-graph-builder.cc to maglev-reducer-inl.h in 15.1; applied the context-tag slot there. - runtime.h: dropped the PromiseRejectAfterResolved/PromiseResolveAfterResolved intrinsics (removed upstream in 15.1) while keeping workerd's PromiseContextInit/PromiseContextCheck. - Dropped 0036-Fix-non-portable-std-atomic_flag-construction: V8 15.1 upstream now initializes with ATOMIC_FLAG_INIT, which is already portable, so the patch is obsolete. Remaining patches renumbered accordingly. - VERSION + INTEGRITY updated; strip_prefix follows. - Bumped deps to match V8 DEPS for version parity: icu (ee5f27a -> d578f2e, ICU 77.1 -> 78.2), fast_float (05087a3 -> 34164f5), fp16 (3d2de18 -> 782eea1), including regenerated integrity in build/deps/gen/deps.MODULE.bazel. * jsg: adapt to V8 15.1 API changes Adapt workerd to the V8 15.1 API: - CppHeapPointerTag: kDefaultTag no longer applies; use kFirstObjectWrappableTag (the first embedder-assignable wrappable tag, valid in both sandbox modes). - DictionaryTemplate::New now takes std::span (v8::MemorySpan became a deprecated alias); pass std::span directly, and add an explicit include. - Migrate other newly-deprecated V8 15.1 APIs (v8_imminent_deprecation_warnings is enabled by default, so these fail -Werror / clang-tidy): - v8::MemorySpan -> std::span in modules.c++, modules-new.c++, unsafe.c++. - Isolate::EnqueueMicrotask -> Context::GetMicrotaskQueue()->EnqueueMicrotask() in global-scope.c++ and worker-fs.c++. See merge request cloudflare/ew/workerd!471 --- build/deps/deps.jsonc | 4 +- build/deps/gen/deps.MODULE.bazel | 12 +-- build/deps/v8.MODULE.bazel | 13 ++- ...etting-ValueDeserializer-format-vers.patch | 4 +- ...etting-ValueSerializer-format-versio.patch | 12 +-- ...003-Allow-Windows-builds-under-Bazel.patch | 20 ++-- ...zel-build-by-always-using-target-cfg.patch | 16 +-- ...06-Implement-Promise-Context-Tagging.patch | 102 +++++++++--------- ...itial-ExecutionContextId-used-by-the.patch | 2 +- ...lizer-SetTreatFunctionsAsHostObjects.patch | 10 +- ...look-for-fp16-dependency.-This-depen.patch | 4 +- ...masm-specific-unwinding-annotations-.patch | 6 +- ...legal-invocation-error-message-in-v8.patch | 8 +- ...request-context-promise-resolve-hand.patch | 76 ++++++------- ...her-slot-in-the-isolate-for-embedder.patch | 4 +- ...ializer-SetTreatProxiesAsHostObjects.patch | 12 +-- .../v8/0017-Enable-V8-shared-linkage.patch | 24 ++--- ...e-to-look-for-fast_float-and-simdutf.patch | 6 +- ...9-Remove-unneded-latomic-linker-flag.patch | 4 +- ...et-heap-and-external-memory-sizes-di.patch | 8 +- ...1-Port-concurrent-mksnapshot-support.patch | 6 +- .../v8/0022-Port-V8_USE_ZLIB-support.patch | 14 +-- ...3-Modify-where-to-look-for-dragonbox.patch | 4 +- .../v8/0024-Disable-slow-handle-check.patch | 2 +- ...ional-Exception-construction-methods.patch | 6 +- .../v8/0028-bind-icu-to-googlesource.patch | 6 +- .../v8/0029-Add-v8-String-IsFlat-API.patch | 4 +- ...untOfExternalAllocatedMemoryImpl-as-.patch | 6 +- ...e_barriers-flag-in-V8-s-bazel-config.patch | 4 +- ...nature-to-get-around-windows-build-f.patch | 4 +- ...Object.hasOwnProperty-with-intercept.patch | 4 +- ....bazel-llvm-toolchain-and-libcxx-rep.patch | 2 +- ...p-from-defs.bzl-not-resolvable-via-h.patch | 4 +- ...6-Fix-ExtendedMap-layout-on-Windows.patch} | 8 +- ...-std-atomic_flag-construction-in-run.patch | 24 ----- ...-MemorySpan-declarations-on-Windows.patch} | 6 +- ...> 0038-Properly-depend-on-llvm-libc.patch} | 6 +- src/workerd/api/global-scope.c++ | 4 +- src/workerd/api/unsafe.c++ | 4 +- src/workerd/io/worker-fs.c++ | 4 +- src/workerd/jsg/jsg.h | 4 +- src/workerd/jsg/modules-new.c++ | 4 +- src/workerd/jsg/modules.c++ | 5 +- src/workerd/jsg/wrappable.h | 5 +- 44 files changed, 236 insertions(+), 251 deletions(-) rename patches/v8/{0037-Fix-ExtendedMap-layout-on-Windows.patch => 0036-Fix-ExtendedMap-layout-on-Windows.patch} (89%) delete mode 100644 patches/v8/0036-Fix-non-portable-std-atomic_flag-construction-in-run.patch rename patches/v8/{0038-Fix-CFunction-MemorySpan-declarations-on-Windows.patch => 0037-Fix-CFunction-MemorySpan-declarations-on-Windows.patch} (87%) rename patches/v8/{0039-Properly-depend-on-llvm-libc.patch => 0038-Properly-depend-on-llvm-libc.patch} (84%) diff --git a/build/deps/deps.jsonc b/build/deps/deps.jsonc index b127866e403..908986f0239 100644 --- a/build/deps/deps.jsonc +++ b/build/deps/deps.jsonc @@ -56,7 +56,7 @@ "owner": "fastfloat", "repo": "fast_float", "branch": "main", - "freeze_commit": "05087a303dad9c98768b33c829d398223a649bc6", + "freeze_commit": "34164f547b7df3f5d794ff67e9f885c36819ebfc", "build_file_content": "cc_library(name = 'fast_float', hdrs = glob(['include/fast_float/*.h']), visibility = ['//visibility:public'], include_prefix = 'third_party/fast_float/src')", "use_module_bazel_from_bcr": "8.0.2" }, @@ -67,7 +67,7 @@ "use_bazel_dep": true, "owner": "Maratyszcza", "repo": "FP16", - "freeze_commit": "3d2de1816307bac63c16a297e8c4dc501b4076df", + "freeze_commit": "782eea126dc5c755827be751a099eb01826175cf", "build_file_content": "exports_files(glob(['**']))", "use_module_bazel_from_bcr": "0.0.0-20210320-0a92994" }, diff --git a/build/deps/gen/deps.MODULE.bazel b/build/deps/gen/deps.MODULE.bazel index 0fa66e6cb3e..7dfd58add5d 100644 --- a/build/deps/gen/deps.MODULE.bazel +++ b/build/deps/gen/deps.MODULE.bazel @@ -52,10 +52,10 @@ archive_override( build_file_content = "cc_library(name = 'fast_float', hdrs = glob(['include/fast_float/*.h']), visibility = ['//visibility:public'], include_prefix = 'third_party/fast_float/src')", remote_file_integrity = {"MODULE.bazel": "sha256-Q1BGZO/fpMbPE0libIcTXJuHkmMlxyBFjzlu7iVWjto="}, remote_file_urls = {"MODULE.bazel": ["https://raw.githubusercontent.com/bazelbuild/bazel-central-registry/refs/heads/main/modules/fast_float/8.0.2/MODULE.bazel"]}, - sha256 = "aa2ab8d370d1011a7a6d4ab90589b298b7e5973fe00041c06ecc7298328c25b4", - strip_prefix = "fastfloat-fast_float-05087a3", + sha256 = "9b239eaa54c58fc3a1bb7e1a11030b67211e38ce7f4781b41b004f26f6fd1528", + strip_prefix = "fastfloat-fast_float-34164f5", type = "tgz", - url = "https://github.com/fastfloat/fast_float/tarball/05087a303dad9c98768b33c829d398223a649bc6", + url = "https://github.com/fastfloat/fast_float/tarball/34164f547b7df3f5d794ff67e9f885c36819ebfc", ) # fp16 @@ -65,10 +65,10 @@ archive_override( build_file_content = "exports_files(glob(['**']))", remote_file_integrity = {"MODULE.bazel": "sha256-2YAHXAyoWo6FapWo2MBLpWyewByY+F4tSWUMbYt8gmg="}, remote_file_urls = {"MODULE.bazel": ["https://raw.githubusercontent.com/bazelbuild/bazel-central-registry/refs/heads/main/modules/fp16/0.0.0-20210320-0a92994/MODULE.bazel"]}, - sha256 = "81686dc45a7897cf958ee99f655879a066dde41db5fc1379a1e8db338ad17589", - strip_prefix = "Maratyszcza-FP16-3d2de18", + sha256 = "ee835b205fabb4035d2c64a2dda54f2e8ff720f9f77a1945b8fa71ba61290c70", + strip_prefix = "Maratyszcza-FP16-782eea1", type = "tgz", - url = "https://github.com/Maratyszcza/FP16/tarball/3d2de1816307bac63c16a297e8c4dc501b4076df", + url = "https://github.com/Maratyszcza/FP16/tarball/782eea126dc5c755827be751a099eb01826175cf", ) # highway diff --git a/build/deps/v8.MODULE.bazel b/build/deps/v8.MODULE.bazel index a77f77b6ba7..15015fda9ac 100644 --- a/build/deps/v8.MODULE.bazel +++ b/build/deps/v8.MODULE.bazel @@ -18,9 +18,9 @@ http_archive = use_repo_rule("@bazel_tools//tools/build_defs/repo:http.bzl", "ht git_repository = use_repo_rule("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository") -VERSION = "15.0.245.14" +VERSION = "15.1.206.7" -INTEGRITY = "sha256-6Txsw77ZX2BQoqTdA0WQQ5C2EmgUu6lYQAkT/wJ2UUM=" +INTEGRITY = "sha256-BToQQPeqVh//fw86DQYHhNVBzyfJ7f+Wlwlfz4nRBZs=" PATCHES = [ "0001-Allow-manually-setting-ValueDeserializer-format-vers.patch", @@ -58,10 +58,9 @@ PATCHES = [ "0033-Return-false-on-Object.hasOwnProperty-with-intercept.patch", "0034-Remove-V8-MODULE.bazel-llvm-toolchain-and-libcxx-rep.patch", "0035-Remove-libcxx-dep-from-defs.bzl-not-resolvable-via-h.patch", - "0036-Fix-non-portable-std-atomic_flag-construction-in-run.patch", - "0037-Fix-ExtendedMap-layout-on-Windows.patch", - "0038-Fix-CFunction-MemorySpan-declarations-on-Windows.patch", - "0039-Properly-depend-on-llvm-libc.patch", + "0036-Fix-ExtendedMap-layout-on-Windows.patch", + "0037-Fix-CFunction-MemorySpan-declarations-on-Windows.patch", + "0038-Properly-depend-on-llvm-libc.patch", ] http_archive( @@ -83,7 +82,7 @@ git_repository( git_repository( name = "com_googlesource_chromium_icu", build_file = "@v8//:bazel/BUILD.icu", - commit = "ee5f27adc28bd3f15b2c293f726d14d2e336cbd5", + commit = "d578f2e8b7bd5938e21cfb6bf15c079e0aa5b738", patch_cmds = ["find source -name BUILD.bazel | xargs rm"], patch_cmds_win = ["Get-ChildItem -Path source -File -Include BUILD.bazel -Recurse | Remove-Item"], remote = "https://chromium.googlesource.com/chromium/deps/icu.git", diff --git a/patches/v8/0001-Allow-manually-setting-ValueDeserializer-format-vers.patch b/patches/v8/0001-Allow-manually-setting-ValueDeserializer-format-vers.patch index 147fc462ad5..ecd877bd4f8 100644 --- a/patches/v8/0001-Allow-manually-setting-ValueDeserializer-format-vers.patch +++ b/patches/v8/0001-Allow-manually-setting-ValueDeserializer-format-vers.patch @@ -37,10 +37,10 @@ index 0cb3e045bc46ec732956318b980e749d1847d06d..40ad805c7970cc9379e69f046205836d * Reads raw data in various common formats to the buffer. * Note that integer types are read in base-128 varint format, not with a diff --git a/src/api/api.cc b/src/api/api.cc -index 92690e59e96140664538fb84116dfca4dc597e4b..19cdfb8a7939ffb9678c881b40dea91b72373f3f 100644 +index 9ba67d3388d8328c93309138bbf8b0954a0850aa..cbe74a9614c347753561b45b9c07598b4263bf09 100644 --- a/src/api/api.cc +++ b/src/api/api.cc -@@ -3701,6 +3701,10 @@ uint32_t ValueDeserializer::GetWireFormatVersion() const { +@@ -3731,6 +3731,10 @@ uint32_t ValueDeserializer::GetWireFormatVersion() const { return private_->deserializer.GetWireFormatVersion(); } diff --git a/patches/v8/0002-Allow-manually-setting-ValueSerializer-format-versio.patch b/patches/v8/0002-Allow-manually-setting-ValueSerializer-format-versio.patch index b7d2894c465..9b5c9cb796d 100644 --- a/patches/v8/0002-Allow-manually-setting-ValueSerializer-format-versio.patch +++ b/patches/v8/0002-Allow-manually-setting-ValueSerializer-format-versio.patch @@ -23,10 +23,10 @@ index 40ad805c7970cc9379e69f046205836dbd760373..596be18adeb3a5a81794aaa44b1d347d * Writes out a header, which includes the format version. */ diff --git a/src/api/api.cc b/src/api/api.cc -index 19cdfb8a7939ffb9678c881b40dea91b72373f3f..ab81034ff4b0696ffaed83949b1707ac54856be1 100644 +index cbe74a9614c347753561b45b9c07598b4263bf09..a03b14ae35a30306fcaf0526cd5dd2f34d812c88 100644 --- a/src/api/api.cc +++ b/src/api/api.cc -@@ -3573,6 +3573,10 @@ ValueSerializer::ValueSerializer(Isolate* v8_isolate, Delegate* delegate) +@@ -3603,6 +3603,10 @@ ValueSerializer::ValueSerializer(Isolate* v8_isolate, Delegate* delegate) ValueSerializer::~ValueSerializer() { delete private_; } @@ -38,10 +38,10 @@ index 19cdfb8a7939ffb9678c881b40dea91b72373f3f..ab81034ff4b0696ffaed83949b1707ac void ValueSerializer::SetTreatArrayBufferViewsAsHostObjects(bool mode) { diff --git a/src/objects/value-serializer.cc b/src/objects/value-serializer.cc -index 359d353797e59b71eb8a71f7f2e9ae0db6e9f30b..804bd5264043790ba70ed19e2dfc06c4ef8c2e3d 100644 +index 06d1861adceae7283938849d0ca0ac9413a87119..1063a750c3b3ccb8f10127d77237880a6f8a15cf 100644 --- a/src/objects/value-serializer.cc +++ b/src/objects/value-serializer.cc -@@ -303,6 +303,7 @@ ValueSerializer::ValueSerializer(Isolate* isolate, +@@ -302,6 +302,7 @@ ValueSerializer::ValueSerializer(Isolate* isolate, : isolate_(isolate), delegate_(delegate), zone_(isolate->allocator(), ZONE_NAME), @@ -49,7 +49,7 @@ index 359d353797e59b71eb8a71f7f2e9ae0db6e9f30b..804bd5264043790ba70ed19e2dfc06c4 id_map_(isolate->heap(), ZoneAllocationPolicy(&zone_)), array_buffer_transfer_map_(isolate->heap(), ZoneAllocationPolicy(&zone_)) { -@@ -322,9 +323,17 @@ ValueSerializer::~ValueSerializer() { +@@ -321,9 +322,17 @@ ValueSerializer::~ValueSerializer() { } } @@ -68,7 +68,7 @@ index 359d353797e59b71eb8a71f7f2e9ae0db6e9f30b..804bd5264043790ba70ed19e2dfc06c4 } void ValueSerializer::SetTreatArrayBufferViewsAsHostObjects(bool mode) { -@@ -1107,10 +1116,12 @@ Maybe ValueSerializer::WriteJSArrayBufferView( +@@ -1106,10 +1115,12 @@ Maybe ValueSerializer::WriteJSArrayBufferView( WriteVarint(static_cast(tag)); WriteVarint(view->byte_offset()); WriteVarint(view->byte_length()); diff --git a/patches/v8/0003-Allow-Windows-builds-under-Bazel.patch b/patches/v8/0003-Allow-Windows-builds-under-Bazel.patch index c0e8c19fa01..de2056cdf3d 100644 --- a/patches/v8/0003-Allow-Windows-builds-under-Bazel.patch +++ b/patches/v8/0003-Allow-Windows-builds-under-Bazel.patch @@ -6,10 +6,10 @@ Subject: Allow Windows builds under Bazel Signed-off-by: James M Snell diff --git a/BUILD.bazel b/BUILD.bazel -index 38c0bf152ebffe97b31ed7999366182680dae82d..539001e56b09feff43d47e9dda308854d68b5627 100644 +index decf32a574257963d4f30f1bc40ddc6a22c071cb..6be6167f027a77ffc28c59827eb261eeaf31ef77 100644 --- a/BUILD.bazel +++ b/BUILD.bazel -@@ -4130,6 +4130,8 @@ filegroup( +@@ -4157,6 +4157,8 @@ filegroup( "@v8//bazel/config:is_inline_asm_x64": ["src/heap/base/asm/x64/push_registers_asm.cc"], "@v8//bazel/config:is_inline_asm_arm": ["src/heap/base/asm/arm/push_registers_asm.cc"], "@v8//bazel/config:is_inline_asm_arm64": ["src/heap/base/asm/arm64/push_registers_asm.cc"], @@ -19,10 +19,10 @@ index 38c0bf152ebffe97b31ed7999366182680dae82d..539001e56b09feff43d47e9dda308854 "@v8//bazel/config:is_inline_asm_riscv64": ["src/heap/base/asm/riscv64/push_registers_asm.cc"], "@v8//bazel/config:is_inline_asm_ppc64le": ["src/heap/base/asm/ppc/push_registers_asm.cc"], diff --git a/bazel/config/BUILD.bazel b/bazel/config/BUILD.bazel -index 17e379b8e27baaa33f58ee852cfd919a9b39d729..7c2154b8ac2e817abebf89f5fa7d30354591d381 100644 +index 4d002e3659aabe48faf357ba07a3f6c689a30e67..c6741a87758ec3c88807800eca9dca17454e7f0c 100644 --- a/bazel/config/BUILD.bazel +++ b/bazel/config/BUILD.bazel -@@ -270,6 +270,7 @@ selects.config_setting_group( +@@ -271,6 +271,7 @@ selects.config_setting_group( match_all = [ ":is_windows", ":is_x64", @@ -30,7 +30,7 @@ index 17e379b8e27baaa33f58ee852cfd919a9b39d729..7c2154b8ac2e817abebf89f5fa7d3035 ], ) -@@ -278,6 +279,7 @@ selects.config_setting_group( +@@ -279,6 +280,7 @@ selects.config_setting_group( match_all = [ ":is_windows", ":is_ia32", @@ -38,7 +38,7 @@ index 17e379b8e27baaa33f58ee852cfd919a9b39d729..7c2154b8ac2e817abebf89f5fa7d3035 ], ) -@@ -286,6 +288,34 @@ selects.config_setting_group( +@@ -287,6 +289,34 @@ selects.config_setting_group( match_all = [ ":is_windows", ":is_arm64", @@ -73,7 +73,7 @@ index 17e379b8e27baaa33f58ee852cfd919a9b39d729..7c2154b8ac2e817abebf89f5fa7d3035 ], ) -@@ -319,6 +349,13 @@ config_setting( +@@ -320,6 +350,13 @@ config_setting( }, ) @@ -88,10 +88,10 @@ index 17e379b8e27baaa33f58ee852cfd919a9b39d729..7c2154b8ac2e817abebf89f5fa7d3035 name = "is_clang", match_any = [ diff --git a/bazel/defs.bzl b/bazel/defs.bzl -index cbe27e0c23488416993b4196cd2c3ee4ffacfc71..f9a883940d15c66e80fdd56e42dbddbd60566c9f 100644 +index ff118a6311e55e7de385f441ef1752ad97e4ec84..c56c0c2c3fbf3529b5dcf5d54921d4fd0b1f1b2a 100644 --- a/bazel/defs.bzl +++ b/bazel/defs.bzl -@@ -125,6 +125,24 @@ def _default_args(): +@@ -134,6 +134,24 @@ def _default_args(): "-Wno-non-virtual-dtor", "-isystem .", ], @@ -116,7 +116,7 @@ index cbe27e0c23488416993b4196cd2c3ee4ffacfc71..f9a883940d15c66e80fdd56e42dbddbd "//conditions:default": [], }) + select({ "@v8//bazel/config:is_clang": [ -@@ -174,13 +192,23 @@ def _default_args(): +@@ -183,13 +201,23 @@ def _default_args(): ], "//conditions:default": [ ], diff --git a/patches/v8/0005-Speed-up-V8-bazel-build-by-always-using-target-cfg.patch b/patches/v8/0005-Speed-up-V8-bazel-build-by-always-using-target-cfg.patch index d515f1b3910..390170371ae 100644 --- a/patches/v8/0005-Speed-up-V8-bazel-build-by-always-using-target-cfg.patch +++ b/patches/v8/0005-Speed-up-V8-bazel-build-by-always-using-target-cfg.patch @@ -10,7 +10,7 @@ both target and exec configurations as generator tools depend on them. Signed-off-by: James M Snell diff --git a/BUILD.bazel b/BUILD.bazel -index 539001e56b09feff43d47e9dda308854d68b5627..e46b1dcfd207f5eb50f6d7e4493d58e2f92030b1 100644 +index 6be6167f027a77ffc28c59827eb261eeaf31ef77..37a39771739b475e2a367d4f4b3b0b68aacfe6ec 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -19,6 +19,7 @@ load( @@ -21,7 +21,7 @@ index 539001e56b09feff43d47e9dda308854d68b5627..e46b1dcfd207f5eb50f6d7e4493d58e2 ) load(":bazel/v8-non-pointer-compression.bzl", "v8_binary_non_pointer_compression") -@@ -4519,22 +4520,20 @@ filegroup( +@@ -4548,22 +4549,20 @@ filegroup( ], ) @@ -50,7 +50,7 @@ index 539001e56b09feff43d47e9dda308854d68b5627..e46b1dcfd207f5eb50f6d7e4493d58e2 ) v8_mksnapshot( -@@ -4763,7 +4762,6 @@ v8_binary( +@@ -4794,7 +4793,6 @@ v8_binary( srcs = [ "src/regexp/gen-regexp-special-case.cc", "src/regexp/special-case.h", @@ -59,10 +59,10 @@ index 539001e56b09feff43d47e9dda308854d68b5627..e46b1dcfd207f5eb50f6d7e4493d58e2 copts = ["-Wno-implicit-fallthrough"], defines = [ diff --git a/bazel/defs.bzl b/bazel/defs.bzl -index f9a883940d15c66e80fdd56e42dbddbd60566c9f..1a790af6467107e2536690abf52b5230d0d78ce8 100644 +index c56c0c2c3fbf3529b5dcf5d54921d4fd0b1f1b2a..c395faec16e9264a066355c17daf46003937b182 100644 --- a/bazel/defs.bzl +++ b/bazel/defs.bzl -@@ -347,6 +347,15 @@ def v8_library( +@@ -356,6 +356,15 @@ def v8_library( **kwargs ) @@ -78,7 +78,7 @@ index f9a883940d15c66e80fdd56e42dbddbd60566c9f..1a790af6467107e2536690abf52b5230 # Use a single generator target for torque definitions and initializers. We can # split the set of outputs by using OutputGroupInfo, that way we do not need to # run the torque generator twice. -@@ -415,7 +424,7 @@ _v8_torque_files = rule( +@@ -422,7 +431,7 @@ _v8_torque_files = rule( "tool": attr.label( allow_files = True, executable = True, @@ -87,7 +87,7 @@ index f9a883940d15c66e80fdd56e42dbddbd60566c9f..1a790af6467107e2536690abf52b5230 ), "args": attr.string_list(), }, -@@ -517,13 +526,16 @@ _v8_mksnapshot = rule( +@@ -524,13 +533,16 @@ _v8_mksnapshot = rule( mandatory = True, allow_files = True, executable = True, @@ -106,7 +106,7 @@ index f9a883940d15c66e80fdd56e42dbddbd60566c9f..1a790af6467107e2536690abf52b5230 ) def v8_mksnapshot(name, args, suffix = ""): -@@ -654,3 +666,34 @@ def v8_build_config(name, arch): +@@ -661,3 +673,34 @@ def v8_build_config(name, arch): outs = ["icu/" + name + ".json"], cmd = "echo '" + build_config_content(cpu, "true") + "' > \"$@\"", ) diff --git a/patches/v8/0006-Implement-Promise-Context-Tagging.patch b/patches/v8/0006-Implement-Promise-Context-Tagging.patch index e95ef5aa038..2c04a55b879 100644 --- a/patches/v8/0006-Implement-Promise-Context-Tagging.patch +++ b/patches/v8/0006-Implement-Promise-Context-Tagging.patch @@ -24,10 +24,10 @@ index 456af07fe1d89feeb03daabc869072220808460d..2feae234f77b60de3220804edc6d2ccd #endif // INCLUDE_V8_ISOLATE_CALLBACKS_H_ diff --git a/include/v8-isolate.h b/include/v8-isolate.h -index e75551b212b30916ba6acabc9ba5957da0b9eaa5..4f1690bbba7d3c32085932cb536f1acd1c62bc2e 100644 +index f8395c96f14bc3d8a07399a65fd077e03d4da578..f1484dc626b6c21d57e97695eb576706c0c725f3 100644 --- a/include/v8-isolate.h +++ b/include/v8-isolate.h -@@ -1878,6 +1878,9 @@ class V8_EXPORT Isolate { +@@ -1880,6 +1880,9 @@ class V8_EXPORT Isolate { */ uint64_t GetHashSeed(); @@ -37,7 +37,7 @@ index e75551b212b30916ba6acabc9ba5957da0b9eaa5..4f1690bbba7d3c32085932cb536f1acd Isolate() = delete; ~Isolate() = delete; Isolate(const Isolate&) = delete; -@@ -1924,6 +1927,19 @@ MaybeLocal Isolate::GetDataFromSnapshotOnce(size_t index) { +@@ -1926,6 +1929,19 @@ MaybeLocal Isolate::GetDataFromSnapshotOnce(size_t index) { return {}; } @@ -58,10 +58,10 @@ index e75551b212b30916ba6acabc9ba5957da0b9eaa5..4f1690bbba7d3c32085932cb536f1acd #endif // INCLUDE_V8_ISOLATE_H_ diff --git a/src/api/api.cc b/src/api/api.cc -index ab81034ff4b0696ffaed83949b1707ac54856be1..eda12d8f7d1468fc57d53ec5c4f0268d2f28c0ec 100644 +index a03b14ae35a30306fcaf0526cd5dd2f34d812c88..e97e02becb356808b6f5026bb9fa65e278dab155 100644 --- a/src/api/api.cc +++ b/src/api/api.cc -@@ -12712,6 +12712,25 @@ std::string SourceLocation::ToString() const { +@@ -12679,6 +12679,25 @@ std::string SourceLocation::ToString() const { .str(); } @@ -88,10 +88,10 @@ index ab81034ff4b0696ffaed83949b1707ac54856be1..eda12d8f7d1468fc57d53ec5c4f0268d #ifdef ENABLE_SLOW_DCHECKS diff --git a/src/builtins/promise-abstract-operations.tq b/src/builtins/promise-abstract-operations.tq -index b95d45fa9ae170f341c5e160b91cc8dcba7fe2cd..59b8d8d5e243cf46a8093c76613ae2ce420e22e8 100644 +index 9265da58004ce7ae00d6ef30daae453db841d13a..5eda0cd4e0cd4d7d8ad4cda6c26e0b6645ac7cba 100644 --- a/src/builtins/promise-abstract-operations.tq +++ b/src/builtins/promise-abstract-operations.tq -@@ -20,6 +20,9 @@ extern transitioning runtime PromiseResolveAfterResolved( +@@ -14,6 +14,9 @@ extern transitioning runtime PromiseRevokeReject( extern transitioning runtime PromiseRejectEventFromStack( implicit context: Context)(JSPromise, JSAny): JSAny; @@ -101,7 +101,7 @@ index b95d45fa9ae170f341c5e160b91cc8dcba7fe2cd..59b8d8d5e243cf46a8093c76613ae2ce } // https://tc39.es/ecma262/#sec-promise-abstract-operations -@@ -470,12 +473,14 @@ transitioning macro PerformPromiseThenImpl( +@@ -471,12 +474,14 @@ transitioning macro PerformPromiseThenImpl( // PromiseReaction holding both the onFulfilled and onRejected callbacks. // Once the {promise} is resolved we decide on the concrete handler to // push onto the microtask queue. @@ -118,7 +118,7 @@ index b95d45fa9ae170f341c5e160b91cc8dcba7fe2cd..59b8d8d5e243cf46a8093c76613ae2ce } else { const reactionsOrResult = promise.reactions_or_result; let microtask: PromiseReactionJobTask; -@@ -497,8 +502,8 @@ transitioning macro PerformPromiseThenImpl( +@@ -498,8 +503,8 @@ transitioning macro PerformPromiseThenImpl( } } EnqueueMicrotask(handlerContext, microtask); @@ -151,7 +151,7 @@ index 50677631b5399453eebc6b149272431f74b1fce6..c652bd836b27805865e0a902ef9cf7c1 } diff --git a/src/builtins/promise-misc.tq b/src/builtins/promise-misc.tq -index f82a7c29779f8047be5377cede911036e3ce2dc5..89e4e8e7a2b7c00eaf9d71c21dd95160082b1528 100644 +index a3a56395edb81c104fe617665e7234982e419ccc..d2f271727c480ed814472a16d95d6aa6fb0c692f 100644 --- a/src/builtins/promise-misc.tq +++ b/src/builtins/promise-misc.tq @@ -54,6 +54,7 @@ macro PromiseInit(promise: JSPromise): void { @@ -170,8 +170,8 @@ index f82a7c29779f8047be5377cede911036e3ce2dc5..89e4e8e7a2b7c00eaf9d71c21dd95160 return promise; } -@@ -273,6 +275,7 @@ transitioning macro NewJSPromise(implicit context: Context)( - parent: Object): JSPromise { +@@ -273,6 +275,7 @@ transitioning macro NewJSPromise(implicit context: Context)(parent: Object): + JSPromise { const instance = InnerNewJSPromise(); PromiseInit(instance); + runtime::PromiseContextInit(instance); @@ -205,10 +205,10 @@ index 2a28acade0ff9d01d89ea33fe086880ed711457a..1df4a87e67153ef9fb2409cd6bcb0bb3 offset < static_cast(sizeof(JSPromise)) + v8::Promise::kEmbedderFieldCount * kEmbedderDataSlotSize; diff --git a/src/diagnostics/objects-printer.cc b/src/diagnostics/objects-printer.cc -index 2819bd94f103e52802c1252f46957056f0dbd76f..677727d8034504ac98e1ceb5cd2bff8ec5fdadfa 100644 +index 233b4513a16ce62a20b9f047db162f5e979bd9ee..77b384998ee5c60dc266cddaab35d052fe42f0ff 100644 --- a/src/diagnostics/objects-printer.cc +++ b/src/diagnostics/objects-printer.cc -@@ -1015,6 +1015,7 @@ void JSPromise::JSPromisePrint(std::ostream& os) { +@@ -1019,6 +1019,7 @@ void JSPromise::JSPromisePrint(std::ostream& os) { } os << "\n - has_handler: " << has_handler(); os << "\n - is_silent: " << is_silent(); @@ -217,10 +217,10 @@ index 2819bd94f103e52802c1252f46957056f0dbd76f..677727d8034504ac98e1ceb5cd2bff8e } diff --git a/src/execution/isolate-inl.h b/src/execution/isolate-inl.h -index 8aee199688f60374443e071db7a44700afbd9f1a..3fe3b5e3607d02466dbb972d2dcaf32cad473ca0 100644 +index 6b704e0f13da98e8a2900757b6cddb37dc3e7554..89bfb809f427851fcddf26e3bbf644c2495a9af0 100644 --- a/src/execution/isolate-inl.h +++ b/src/execution/isolate-inl.h -@@ -124,6 +124,25 @@ bool Isolate::is_execution_terminating() { +@@ -125,6 +125,25 @@ bool Isolate::is_execution_terminating() { i::ReadOnlyRoots(this).termination_exception(); } @@ -247,19 +247,19 @@ index 8aee199688f60374443e071db7a44700afbd9f1a..3fe3b5e3607d02466dbb972d2dcaf32c Tagged Isolate::VerifyBuiltinsResult(Tagged result) { if (is_execution_terminating() && !v8_flags.strict_termination_checks) { diff --git a/src/execution/isolate.cc b/src/execution/isolate.cc -index 80e39c7c3260ef9d005f45e5ddba302fb490e18e..a8c10b94c664b2fc68312a0bd3089affaee261ec 100644 +index d5b352a620a95bc41680b0222be9184ec726e5c1..a75b94aa2a39d5d0e5dbad86891a2d3461d2873f 100644 --- a/src/execution/isolate.cc +++ b/src/execution/isolate.cc -@@ -682,6 +682,8 @@ void Isolate::Iterate(RootVisitor* v, ThreadLocalTop* thread) { - FullObjectSlot(&thread->pending_message_)); - v->VisitRootPointer(Root::kStackRoots, nullptr, +@@ -678,6 +678,8 @@ void Isolate::Iterate(RootVisitor* v, ThreadLocalTop* thread) { FullObjectSlot(&thread->context_)); + v->VisitRootPointer(Root::kStackRoots, nullptr, + FullObjectSlot(&thread->last_entered_context_)); + v->VisitRootPointer(Root::kStackRoots, nullptr, + FullObjectSlot(&promise_context_tag_)); for (v8::TryCatch* block = thread->try_catch_handler_; block != nullptr; block = block->next_) { -@@ -6395,6 +6397,7 @@ bool Isolate::Init(SnapshotData* startup_snapshot_data, +@@ -6423,6 +6425,7 @@ bool Isolate::Init(SnapshotData* startup_snapshot_data, shared_heap_object_cache_.push_back(ReadOnlyRoots(this).undefined_value()); } @@ -267,7 +267,7 @@ index 80e39c7c3260ef9d005f45e5ddba302fb490e18e..a8c10b94c664b2fc68312a0bd3089aff InitializeThreadLocal(); // Profiler has to be created after ThreadLocal is initialized -@@ -8553,5 +8556,40 @@ void Isolate::PrintNumberStringCacheStats(const char* comment, +@@ -8581,5 +8584,40 @@ void Isolate::PrintNumberStringCacheStats(const char* comment, PrintF("\n"); } @@ -309,10 +309,10 @@ index 80e39c7c3260ef9d005f45e5ddba302fb490e18e..a8c10b94c664b2fc68312a0bd3089aff } // namespace internal } // namespace v8 diff --git a/src/execution/isolate.h b/src/execution/isolate.h -index 7ff2adf5e3ebcdf97699a4abc2969bcd3882dd79..10bde98a47b9ad44597f51e1dc31b075f36e7b96 100644 +index 536cea9188fca4fee429f90afaaa0459f1b86d47..8f0e28838af8102835a90f77fc1ee31d6ebd1080 100644 --- a/src/execution/isolate.h +++ b/src/execution/isolate.h -@@ -2485,6 +2485,15 @@ class V8_EXPORT_PRIVATE Isolate final : private HiddenFactory { +@@ -2500,6 +2500,15 @@ class V8_EXPORT_PRIVATE Isolate final : private HiddenFactory { v8::ExceptionContext callback_kind); void SetExceptionPropagationCallback(ExceptionPropagationCallback callback); @@ -328,10 +328,10 @@ index 7ff2adf5e3ebcdf97699a4abc2969bcd3882dd79..10bde98a47b9ad44597f51e1dc31b075 #ifdef V8_ENABLE_WASM_SIMD256_REVEC void set_wasm_revec_verifier_for_test( compiler::turboshaft::WasmRevecVerifier* verifier) { -@@ -3020,6 +3029,12 @@ class V8_EXPORT_PRIVATE Isolate final : private HiddenFactory { - +@@ -3044,6 +3053,12 @@ class V8_EXPORT_PRIVATE Isolate final : private HiddenFactory { bool is_frozen_ = false; + friend class HandleScopeImplementer; + Tagged promise_context_tag_; + PromiseCrossContextCallback promise_cross_context_callback_; + bool in_promise_cross_context_callback_ = false; @@ -341,7 +341,7 @@ index 7ff2adf5e3ebcdf97699a4abc2969bcd3882dd79..10bde98a47b9ad44597f51e1dc31b075 friend class GlobalSafepoint; friend class heap::HeapTester; friend class IsolateForPointerCompression; -@@ -3027,6 +3042,7 @@ class V8_EXPORT_PRIVATE Isolate final : private HiddenFactory { +@@ -3051,6 +3066,7 @@ class V8_EXPORT_PRIVATE Isolate final : private HiddenFactory { friend class IsolateGroup; friend class TestSerializer; friend class SharedHeapNoClientsTest; @@ -350,10 +350,10 @@ index 7ff2adf5e3ebcdf97699a4abc2969bcd3882dd79..10bde98a47b9ad44597f51e1dc31b075 // The current entered Isolate and its thread data. Do not access these diff --git a/src/heap/factory.cc b/src/heap/factory.cc -index fbe5f7c7c69b3f0ace50ed7dfb51a173abca0ff3..63d15d64de7a933353dedfbc9aa79f6d60daeb24 100644 +index be56abbc9d7fa700319c58856356e9a111880ea7..56573889afd5888dc6a178e21bc7905e8e183890 100644 --- a/src/heap/factory.cc +++ b/src/heap/factory.cc -@@ -5096,6 +5096,12 @@ Handle Factory::NewJSPromiseWithoutHook() { +@@ -5116,6 +5116,12 @@ Handle Factory::NewJSPromiseWithoutHook() { DisallowGarbageCollection no_gc; Tagged raw = *promise; raw->set_reactions_or_result(Smi::zero(), SKIP_WRITE_BARRIER); @@ -366,11 +366,11 @@ index fbe5f7c7c69b3f0ace50ed7dfb51a173abca0ff3..63d15d64de7a933353dedfbc9aa79f6d raw->set_flags(0); // TODO(v8) remove once embedder data slots are always zero-initialized. InitEmbedderFields(*promise, Smi::zero()); -diff --git a/src/maglev/maglev-graph-builder.cc b/src/maglev/maglev-graph-builder.cc -index c4facfe62aa2fd8a0535e268e59a8dfb7eb34935..81db14a4e0b5f020f59f12d3126aa4ccf95c3098 100644 ---- a/src/maglev/maglev-graph-builder.cc -+++ b/src/maglev/maglev-graph-builder.cc -@@ -15061,9 +15061,10 @@ VirtualObject* MaglevGraphBuilder::CreateJSPromiseObject() { +diff --git a/src/maglev/maglev-reducer-inl.h b/src/maglev/maglev-reducer-inl.h +index 9d45995cb89536e455c486ee065f9b841a31cfd1..8fa3a3ef3a0edbaec3089143ea556d604c8b0486 100644 +--- a/src/maglev/maglev-reducer-inl.h ++++ b/src/maglev/maglev-reducer-inl.h +@@ -5203,9 +5203,10 @@ VirtualObject* MaglevReducer::CreateJSPromiseObject() { vobj->set(offsetof(JSObject, elements_), GetRootConstant(RootIndex::kEmptyFixedArray)); vobj->set(offsetof(JSPromise, reactions_or_result_), GetSmiConstant(0)); @@ -383,10 +383,10 @@ index c4facfe62aa2fd8a0535e268e59a8dfb7eb34935..81db14a4e0b5f020f59f12d3126aa4cc offset < static_cast(sizeof(JSPromise)) + v8::Promise::kEmbedderFieldCount * kEmbedderDataSlotSize; diff --git a/src/objects/js-promise-inl.h b/src/objects/js-promise-inl.h -index a15f6889ddc9168caa0b7367d094d0ffe516a5f7..4e484bbc909c70f23d89baa3f1858a93366e330a 100644 +index 464b9c0712dea1c096d7993582a7399ce5ab47f7..5e07e81cfeba06c46d8400c3bfb17b535cf13b15 100644 --- a/src/objects/js-promise-inl.h +++ b/src/objects/js-promise-inl.h -@@ -27,6 +27,12 @@ void JSPromise::set_reactions_or_result( +@@ -29,6 +29,12 @@ void JSPromise::set_reactions_or_result( reactions_or_result_.store(this, value, mode); } @@ -400,10 +400,10 @@ index a15f6889ddc9168caa0b7367d094d0ffe516a5f7..4e484bbc909c70f23d89baa3f1858a93 void JSPromise::set_flags(int value) { diff --git a/src/objects/js-promise.h b/src/objects/js-promise.h -index 9e3f79fa2d316c059e84c1468948fe4b02423e9c..2bc8ef88f9f0c3423a1f86bcf7b5e5d6ed2d77bf 100644 +index b76ea162d9f7b7e5a7a828d189181b0d52d31349..1071da0394a9604dd5b77866817a7802dcf76ecf 100644 --- a/src/objects/js-promise.h +++ b/src/objects/js-promise.h -@@ -39,6 +39,10 @@ V8_OBJECT class JSPromise : public JSObjectWithEmbedderSlots { +@@ -37,6 +37,10 @@ V8_OBJECT class JSPromise : public JSObjectWithEmbedderSlots { Tagged> value, WriteBarrierMode mode = UPDATE_WRITE_BARRIER); @@ -414,7 +414,7 @@ index 9e3f79fa2d316c059e84c1468948fe4b02423e9c..2bc8ef88f9f0c3423a1f86bcf7b5e5d6 inline int flags() const; inline void set_flags(int value); -@@ -111,9 +115,16 @@ V8_OBJECT class JSPromise : public JSObjectWithEmbedderSlots { +@@ -113,9 +117,16 @@ V8_OBJECT class JSPromise : public JSObjectWithEmbedderSlots { // Smi 0 terminated list of PromiseReaction objects in case the JSPromise // was not settled yet, otherwise the result. TaggedMember> reactions_or_result_; @@ -431,8 +431,8 @@ index 9e3f79fa2d316c059e84c1468948fe4b02423e9c..2bc8ef88f9f0c3423a1f86bcf7b5e5d6 private: // https://tc39.es/ecma262/#sec-triggerpromisereactions static Handle TriggerPromiseReactions(Isolate* isolate, -@@ -122,6 +133,9 @@ V8_OBJECT class JSPromise : public JSObjectWithEmbedderSlots { - PromiseReaction::Type type); +@@ -127,6 +138,9 @@ V8_OBJECT class JSPromise : public JSObjectWithEmbedderSlots { + V8_OBJECT class JSPromiseConstructor : public JSFunctionWithPrototype { } V8_OBJECT_END; +inline constexpr int JSPromise::kContextTagOffset = @@ -442,10 +442,10 @@ index 9e3f79fa2d316c059e84c1468948fe4b02423e9c..2bc8ef88f9f0c3423a1f86bcf7b5e5d6 } // namespace v8 diff --git a/src/objects/js-promise.tq b/src/objects/js-promise.tq -index 6ea2059e932519c065cfd40d878ad5e43b9afb8a..ff19da88e1aa4fbd8d7a28dae8e954e8d4a6e99f 100644 +index 2e498caeb9a4dea6f4ef361ffe6547769e749874..227d693af230ef14dbc3f90f955701717e710fa2 100644 --- a/src/objects/js-promise.tq +++ b/src/objects/js-promise.tq -@@ -34,6 +34,7 @@ extern class JSPromise extends JSObjectWithEmbedderSlots { +@@ -35,6 +35,7 @@ extern class JSPromise extends JSObjectWithEmbedderSlots { // Smi 0 terminated list of PromiseReaction objects in case the JSPromise was // not settled yet, otherwise the result. reactions_or_result: Zero|PromiseReaction|JSAny; @@ -454,10 +454,10 @@ index 6ea2059e932519c065cfd40d878ad5e43b9afb8a..ff19da88e1aa4fbd8d7a28dae8e954e8 } diff --git a/src/profiler/heap-snapshot-generator.cc b/src/profiler/heap-snapshot-generator.cc -index 6834bd35f93c919196368e4b4dc731769c082f7b..c0c87c69cfceed53b052159ea7921e7af51dcbe3 100644 +index d4a51613c2a383527fd80d382f75fe871bd9c610..1d876d10456a595039bff8c569bd270b6ff7759f 100644 --- a/src/profiler/heap-snapshot-generator.cc +++ b/src/profiler/heap-snapshot-generator.cc -@@ -2236,6 +2236,8 @@ void V8HeapExplorer::ExtractJSPromiseReferences(HeapEntry* entry, +@@ -2319,6 +2319,8 @@ void V8HeapExplorer::ExtractJSPromiseReferences(HeapEntry* entry, SetInternalReference(entry, "reactions_or_result", promise->reactions_or_result(), offsetof(JSPromise, reactions_or_result_)); @@ -467,10 +467,10 @@ index 6834bd35f93c919196368e4b4dc731769c082f7b..c0c87c69cfceed53b052159ea7921e7a void V8HeapExplorer::ExtractJSGeneratorObjectReferences( diff --git a/src/runtime/runtime-promise.cc b/src/runtime/runtime-promise.cc -index eeb85328beeb07d885968efd1de9e2302105d3e1..d75b8e7901043e637604bea60585dd3633a85014 100644 +index 169d48bd4567686a94cc2dbca54daab3e73ad658..618dbb2f0d6e946c34f0271ec25b143cc0d7338f 100644 --- a/src/runtime/runtime-promise.cc +++ b/src/runtime/runtime-promise.cc -@@ -260,5 +260,41 @@ RUNTIME_FUNCTION(Runtime_ConstructSuppressedError) { +@@ -236,5 +236,41 @@ RUNTIME_FUNCTION(Runtime_ConstructSuppressedError) { return *result; } @@ -513,10 +513,10 @@ index eeb85328beeb07d885968efd1de9e2302105d3e1..d75b8e7901043e637604bea60585dd36 } // namespace internal } // namespace v8 diff --git a/src/runtime/runtime.h b/src/runtime/runtime.h -index 6e78c60fcb25c314bd65623d9642f06dd340fff7..a8ca0f8fa3a0c29d411860a8c67d781727adaf6d 100644 +index 2d7628dd0f55399d92aab173beabb6fbe134f7b5..c1bc7761429de83818fe4b1266c9061da95f8330 100644 --- a/src/runtime/runtime.h +++ b/src/runtime/runtime.h -@@ -434,20 +434,22 @@ constexpr bool CanTriggerGC(T... properties) { +@@ -434,18 +434,20 @@ constexpr bool CanTriggerGC(T... properties) { F(StrictNotEqual, 2, 1) \ F(ReferenceEqual, 2, 1) @@ -529,8 +529,6 @@ index 6e78c60fcb25c314bd65623d9642f06dd340fff7..a8ca0f8fa3a0c29d411860a8c67d7817 - F(PromiseRevokeReject, 1, 1) \ - F(RejectPromise, 3, 1) \ - F(ResolvePromise, 2, 1) \ -- F(PromiseRejectAfterResolved, 2, 1) \ -- F(PromiseResolveAfterResolved, 2, 1) \ - F(ConstructSuppressedError, 3, 1) \ - F(ConstructAggregateErrorHelper, 4, 1) \ - F(ConstructInternalAggregateErrorHelper, -1 /* <= 5*/, 1) @@ -543,8 +541,6 @@ index 6e78c60fcb25c314bd65623d9642f06dd340fff7..a8ca0f8fa3a0c29d411860a8c67d7817 + F(PromiseRevokeReject, 1, 1) \ + F(RejectPromise, 3, 1) \ + F(ResolvePromise, 2, 1) \ -+ F(PromiseRejectAfterResolved, 2, 1) \ -+ F(PromiseResolveAfterResolved, 2, 1) \ + F(ConstructSuppressedError, 3, 1) \ + F(ConstructAggregateErrorHelper, 4, 1) \ + F(ConstructInternalAggregateErrorHelper, -1 /* <= 5*/, 1) \ diff --git a/patches/v8/0007-Randomize-the-initial-ExecutionContextId-used-by-the.patch b/patches/v8/0007-Randomize-the-initial-ExecutionContextId-used-by-the.patch index 9107e58b61f..d1a89114374 100644 --- a/patches/v8/0007-Randomize-the-initial-ExecutionContextId-used-by-the.patch +++ b/patches/v8/0007-Randomize-the-initial-ExecutionContextId-used-by-the.patch @@ -10,7 +10,7 @@ live workers (https://chat.google.com/room/AAAAnS2bXT4/GX5-pa8O0ts). Signed-off-by: James M Snell diff --git a/src/inspector/v8-inspector-impl.cc b/src/inspector/v8-inspector-impl.cc -index 697d92265a613efc1e18b0b85477d0db1d15ce80..dd787bfbaaa8787b5cfea834bd9f24bd1cd7d098 100644 +index 4fd23c1fd33cfb02b91d0f75084cde7419b66a69..dc179be19c6fed309dd4d8979dc55f1c7a1f3794 100644 --- a/src/inspector/v8-inspector-impl.cc +++ b/src/inspector/v8-inspector-impl.cc @@ -68,7 +68,7 @@ V8InspectorImpl::V8InspectorImpl(v8::Isolate* isolate, diff --git a/patches/v8/0009-Add-ValueSerializer-SetTreatFunctionsAsHostObjects.patch b/patches/v8/0009-Add-ValueSerializer-SetTreatFunctionsAsHostObjects.patch index cabe7965831..bfc08e4c563 100644 --- a/patches/v8/0009-Add-ValueSerializer-SetTreatFunctionsAsHostObjects.patch +++ b/patches/v8/0009-Add-ValueSerializer-SetTreatFunctionsAsHostObjects.patch @@ -30,10 +30,10 @@ index 596be18adeb3a5a81794aaa44b1d347dec6c0c7d..141f138e08de849e3e02b3b2b346e643 * Write raw data in various common formats to the buffer. * Note that integer types are written in base-128 varint format, not with a diff --git a/src/api/api.cc b/src/api/api.cc -index eda12d8f7d1468fc57d53ec5c4f0268d2f28c0ec..cac637e8f245e9c3d1cacd44cb5c3782b68ddf0e 100644 +index e97e02becb356808b6f5026bb9fa65e278dab155..ca176118c9299870c366a4d58e54f0e2ed691117 100644 --- a/src/api/api.cc +++ b/src/api/api.cc -@@ -3583,6 +3583,10 @@ void ValueSerializer::SetTreatArrayBufferViewsAsHostObjects(bool mode) { +@@ -3613,6 +3613,10 @@ void ValueSerializer::SetTreatArrayBufferViewsAsHostObjects(bool mode) { private_->serializer.SetTreatArrayBufferViewsAsHostObjects(mode); } @@ -45,10 +45,10 @@ index eda12d8f7d1468fc57d53ec5c4f0268d2f28c0ec..cac637e8f245e9c3d1cacd44cb5c3782 Local value) { auto i_isolate = i::Isolate::Current(); diff --git a/src/objects/value-serializer.cc b/src/objects/value-serializer.cc -index 804bd5264043790ba70ed19e2dfc06c4ef8c2e3d..fd54ce9a832b2bcc25a0a538449d0f85df1372d4 100644 +index 1063a750c3b3ccb8f10127d77237880a6f8a15cf..608501699c7868ce07d0b7c93c68513fe1e6b299 100644 --- a/src/objects/value-serializer.cc +++ b/src/objects/value-serializer.cc -@@ -340,6 +340,10 @@ void ValueSerializer::SetTreatArrayBufferViewsAsHostObjects(bool mode) { +@@ -339,6 +339,10 @@ void ValueSerializer::SetTreatArrayBufferViewsAsHostObjects(bool mode) { treat_array_buffer_views_as_host_objects_ = mode; } @@ -59,7 +59,7 @@ index 804bd5264043790ba70ed19e2dfc06c4ef8c2e3d..fd54ce9a832b2bcc25a0a538449d0f85 void ValueSerializer::WriteTag(SerializationTag tag) { uint8_t raw_tag = static_cast(tag); WriteRawBytes(&raw_tag, sizeof(raw_tag)); -@@ -610,13 +614,17 @@ Maybe ValueSerializer::WriteJSReceiver( +@@ -609,13 +613,17 @@ Maybe ValueSerializer::WriteJSReceiver( // Eliminate callable and exotic objects, which should not be serialized. InstanceType instance_type = receiver->map()->instance_type(); diff --git a/patches/v8/0010-Modify-where-to-look-for-fp16-dependency.-This-depen.patch b/patches/v8/0010-Modify-where-to-look-for-fp16-dependency.-This-depen.patch index 5ab9f2f3fd9..dc487d1d0c8 100644 --- a/patches/v8/0010-Modify-where-to-look-for-fp16-dependency.-This-depen.patch +++ b/patches/v8/0010-Modify-where-to-look-for-fp16-dependency.-This-depen.patch @@ -8,10 +8,10 @@ Subject: Modify where to look for fp16 dependency. This dependency is normally Signed-off-by: James M Snell diff --git a/BUILD.bazel b/BUILD.bazel -index e46b1dcfd207f5eb50f6d7e4493d58e2f92030b1..4bc28906f59a85c535c35756e684d11450d9aa3f 100644 +index 37a39771739b475e2a367d4f4b3b0b68aacfe6ec..1bf532f07dd3e6f45d57df1a2ee744a62e0156a3 100644 --- a/BUILD.bazel +++ b/BUILD.bazel -@@ -4153,17 +4153,23 @@ v8_library( +@@ -4180,17 +4180,23 @@ v8_library( ], ) diff --git a/patches/v8/0011-Revert-heap-Add-masm-specific-unwinding-annotations-.patch b/patches/v8/0011-Revert-heap-Add-masm-specific-unwinding-annotations-.patch index e73467bd2ab..31a515184a4 100644 --- a/patches/v8/0011-Revert-heap-Add-masm-specific-unwinding-annotations-.patch +++ b/patches/v8/0011-Revert-heap-Add-masm-specific-unwinding-annotations-.patch @@ -14,10 +14,10 @@ of getting our V8 upgrade unblocked. Signed-off-by: James M Snell diff --git a/BUILD.gn b/BUILD.gn -index a19ec5212547a907fd9c094dc030e806110d176c..b4d28a2e2734e9215948da4dda208768a29af507 100644 +index 03938a26b4d5b3fe5d6cee0ea411fedfd92df661..2a1f30b602a8e35fdaf7ff533af981fe6c6ffa7d 100644 --- a/BUILD.gn +++ b/BUILD.gn -@@ -4669,8 +4669,8 @@ v8_header_set("v8_internal_headers") { +@@ -4692,8 +4692,8 @@ v8_header_set("v8_internal_headers") { "src/tasks/operations-barrier.h", "src/tasks/task-utils.h", "src/torque/runtime-macro-shims.h", @@ -27,7 +27,7 @@ index a19ec5212547a907fd9c094dc030e806110d176c..b4d28a2e2734e9215948da4dda208768 "src/tracing/trace-id.h", "src/tracing/traced-value.h", "src/tracing/tracing-category-observer.h", -@@ -7604,12 +7604,7 @@ v8_source_set("v8_heap_base") { +@@ -7639,12 +7639,7 @@ v8_source_set("v8_heap_base") { ] if (current_cpu == "x64") { diff --git a/patches/v8/0012-Update-illegal-invocation-error-message-in-v8.patch b/patches/v8/0012-Update-illegal-invocation-error-message-in-v8.patch index 5997765c6c1..3440adada68 100644 --- a/patches/v8/0012-Update-illegal-invocation-error-message-in-v8.patch +++ b/patches/v8/0012-Update-illegal-invocation-error-message-in-v8.patch @@ -23,10 +23,10 @@ index 4480bd4a9a9ac090def94116d8da123ff3acedea..718bf26860b7b292f380196b520f0fa4 "Immutable prototype object '%' cannot have their prototype set") \ T(ImportAttributesDuplicateKey, "Import attribute has duplicate key '%'") \ diff --git a/test/cctest/test-api.cc b/test/cctest/test-api.cc -index 43aa13f784e80acfa52bf08663993f89abae87f9..100cc11404190f9c60857ec2e63d1d987abdcbe2 100644 +index 0e5aefebed637304086356d66a92de5994946d11..30aee6a4c4d2369c3b7d886551976bc1dc98e3e0 100644 --- a/test/cctest/test-api.cc +++ b/test/cctest/test-api.cc -@@ -225,6 +225,17 @@ THREADED_TEST(IsolateOfContext) { +@@ -227,6 +227,17 @@ THREADED_TEST(IsolateOfContext) { CHECK(isolate->IsCurrent()); } @@ -44,7 +44,7 @@ index 43aa13f784e80acfa52bf08663993f89abae87f9..100cc11404190f9c60857ec2e63d1d98 static void TestSignatureLooped(const char* operation, Local receiver, v8::Isolate* isolate) { auto source = v8::base::OwnedVector::NewForOverwrite(200); -@@ -242,12 +253,7 @@ static void TestSignatureLooped(const char* operation, Local receiver, +@@ -244,12 +255,7 @@ static void TestSignatureLooped(const char* operation, Local receiver, if (!expected_to_throw) { CHECK_EQ(10, signature_callback_count); } else { @@ -58,7 +58,7 @@ index 43aa13f784e80acfa52bf08663993f89abae87f9..100cc11404190f9c60857ec2e63d1d98 } signature_expected_receiver_global.Reset(); } -@@ -274,12 +280,7 @@ static void TestSignatureOptimized(const char* operation, Local receiver, +@@ -276,12 +282,7 @@ static void TestSignatureOptimized(const char* operation, Local receiver, if (!expected_to_throw) { CHECK_EQ(3, signature_callback_count); } else { diff --git a/patches/v8/0013-Implement-cross-request-context-promise-resolve-hand.patch b/patches/v8/0013-Implement-cross-request-context-promise-resolve-hand.patch index 0808323675f..6202239582e 100644 --- a/patches/v8/0013-Implement-cross-request-context-promise-resolve-hand.patch +++ b/patches/v8/0013-Implement-cross-request-context-promise-resolve-hand.patch @@ -36,10 +36,10 @@ index 2feae234f77b60de3220804edc6d2ccd067f5670..cb04861438b5f16abefb2375d1284be3 #endif // INCLUDE_V8_ISOLATE_CALLBACKS_H_ diff --git a/include/v8-isolate.h b/include/v8-isolate.h -index 4f1690bbba7d3c32085932cb536f1acd1c62bc2e..90ec2bf7625e25017a633b845be86bda1c2d7880 100644 +index f1484dc626b6c21d57e97695eb576706c0c725f3..7d25bc6c25faac2b36441a63f186085e416eaccb 100644 --- a/include/v8-isolate.h +++ b/include/v8-isolate.h -@@ -1880,6 +1880,8 @@ class V8_EXPORT Isolate { +@@ -1882,6 +1882,8 @@ class V8_EXPORT Isolate { class PromiseContextScope; void SetPromiseCrossContextCallback(PromiseCrossContextCallback callback); @@ -49,10 +49,10 @@ index 4f1690bbba7d3c32085932cb536f1acd1c62bc2e..90ec2bf7625e25017a633b845be86bda Isolate() = delete; ~Isolate() = delete; diff --git a/src/api/api.cc b/src/api/api.cc -index cac637e8f245e9c3d1cacd44cb5c3782b68ddf0e..0e7e6cb2ed80c6b6ea0ec2b55705e904521efa5f 100644 +index ca176118c9299870c366a4d58e54f0e2ed691117..420cce4e0eb93356c87d616632cb2ed80f51a76f 100644 --- a/src/api/api.cc +++ b/src/api/api.cc -@@ -12722,13 +12722,19 @@ void Isolate::SetPromiseCrossContextCallback( +@@ -12689,13 +12689,19 @@ void Isolate::SetPromiseCrossContextCallback( isolate->set_promise_cross_context_callback(callback); } @@ -74,10 +74,10 @@ index cac637e8f245e9c3d1cacd44cb5c3782b68ddf0e..0e7e6cb2ed80c6b6ea0ec2b55705e904 Isolate::PromiseContextScope::~PromiseContextScope() { diff --git a/src/builtins/promise-abstract-operations.tq b/src/builtins/promise-abstract-operations.tq -index 59b8d8d5e243cf46a8093c76613ae2ce420e22e8..fd7418198103f0cf31b47155794db048012cd110 100644 +index 5eda0cd4e0cd4d7d8ad4cda6c26e0b6645ac7cba..c161714cd4d53ee4a8cc9be7ee2626e8b1a564d8 100644 --- a/src/builtins/promise-abstract-operations.tq +++ b/src/builtins/promise-abstract-operations.tq -@@ -23,6 +23,9 @@ extern transitioning runtime PromiseRejectEventFromStack( +@@ -17,6 +17,9 @@ extern transitioning runtime PromiseRejectEventFromStack( extern transitioning runtime PromiseContextCheck( implicit context: Context)(JSPromise): JSPromise; @@ -87,7 +87,7 @@ index 59b8d8d5e243cf46a8093c76613ae2ce420e22e8..fd7418198103f0cf31b47155794db048 } // https://tc39.es/ecma262/#sec-promise-abstract-operations -@@ -246,13 +249,16 @@ extern macro PromiseBuiltinsAssembler:: +@@ -240,13 +243,16 @@ extern macro PromiseBuiltinsAssembler:: transitioning builtin RejectPromise( implicit context: Context)(promise: JSPromise, reason: JSAny, debugEvent: Boolean): JSAny { @@ -120,10 +120,10 @@ index 202180adbbae91a689a667c40d20b4b1b9cb6edd..5e618fcc7521d6c9ba15d83cca949099 deferred { return runtime::ResolvePromise(promise, resolution); diff --git a/src/execution/isolate-inl.h b/src/execution/isolate-inl.h -index 3fe3b5e3607d02466dbb972d2dcaf32cad473ca0..a9844cff4a0f08b86cdd150a82ef9d5272ae83b4 100644 +index 89bfb809f427851fcddf26e3bbf644c2495a9af0..9e42a18d9fa86c45f0c9774f78368bc0b0671ee2 100644 --- a/src/execution/isolate-inl.h +++ b/src/execution/isolate-inl.h -@@ -124,18 +124,20 @@ bool Isolate::is_execution_terminating() { +@@ -125,18 +125,20 @@ bool Isolate::is_execution_terminating() { i::ReadOnlyRoots(this).termination_exception(); } @@ -149,7 +149,7 @@ index 3fe3b5e3607d02466dbb972d2dcaf32cad473ca0..a9844cff4a0f08b86cdd150a82ef9d52 } void Isolate::set_promise_cross_context_callback( -@@ -143,6 +145,15 @@ void Isolate::set_promise_cross_context_callback( +@@ -144,6 +146,15 @@ void Isolate::set_promise_cross_context_callback( promise_cross_context_callback_ = callback; } @@ -166,19 +166,19 @@ index 3fe3b5e3607d02466dbb972d2dcaf32cad473ca0..a9844cff4a0f08b86cdd150a82ef9d52 Tagged Isolate::VerifyBuiltinsResult(Tagged result) { if (is_execution_terminating() && !v8_flags.strict_termination_checks) { diff --git a/src/execution/isolate.cc b/src/execution/isolate.cc -index a8c10b94c664b2fc68312a0bd3089affaee261ec..43fba14a9b7b2eb2a487ae441b8e0c209bd4ca37 100644 +index a75b94aa2a39d5d0e5dbad86891a2d3461d2873f..444a6f1b4146720437bcec164e2771ffe1f02091 100644 --- a/src/execution/isolate.cc +++ b/src/execution/isolate.cc -@@ -682,8 +682,6 @@ void Isolate::Iterate(RootVisitor* v, ThreadLocalTop* thread) { - FullObjectSlot(&thread->pending_message_)); - v->VisitRootPointer(Root::kStackRoots, nullptr, +@@ -678,8 +678,6 @@ void Isolate::Iterate(RootVisitor* v, ThreadLocalTop* thread) { FullObjectSlot(&thread->context_)); + v->VisitRootPointer(Root::kStackRoots, nullptr, + FullObjectSlot(&thread->last_entered_context_)); - v->VisitRootPointer(Root::kStackRoots, nullptr, - FullObjectSlot(&promise_context_tag_)); for (v8::TryCatch* block = thread->try_catch_handler_; block != nullptr; block = block->next_) { -@@ -8591,5 +8589,23 @@ MaybeHandle Isolate::RunPromiseCrossContextCallback( +@@ -8619,5 +8617,23 @@ MaybeHandle Isolate::RunPromiseCrossContextCallback( return v8::Utils::OpenHandle(*result); } @@ -203,18 +203,18 @@ index a8c10b94c664b2fc68312a0bd3089affaee261ec..43fba14a9b7b2eb2a487ae441b8e0c20 } // namespace internal } // namespace v8 diff --git a/src/execution/isolate.h b/src/execution/isolate.h -index 10bde98a47b9ad44597f51e1dc31b075f36e7b96..7d7a21036bc49b90f3455b2ca72571d9d8796678 100644 +index 8f0e28838af8102835a90f77fc1ee31d6ebd1080..b6ce92538e19472d6096d3fb10628e615164dd9f 100644 --- a/src/execution/isolate.h +++ b/src/execution/isolate.h -@@ -45,6 +45,7 @@ +@@ -47,6 +47,7 @@ #include "src/objects/contexts.h" #include "src/objects/debug-objects.h" #include "src/objects/js-objects.h" +#include "src/objects/promise.h" #include "src/objects/tagged.h" #include "src/runtime/runtime.h" - #include "src/sandbox/code-pointer-table.h" -@@ -2485,15 +2486,24 @@ class V8_EXPORT_PRIVATE Isolate final : private HiddenFactory { + #include "src/sandbox/external-pointer-table.h" +@@ -2500,15 +2501,24 @@ class V8_EXPORT_PRIVATE Isolate final : private HiddenFactory { v8::ExceptionContext callback_kind); void SetExceptionPropagationCallback(ExceptionPropagationCallback callback); @@ -241,10 +241,10 @@ index 10bde98a47b9ad44597f51e1dc31b075f36e7b96..7d7a21036bc49b90f3455b2ca72571d9 #ifdef V8_ENABLE_WASM_SIMD256_REVEC void set_wasm_revec_verifier_for_test( compiler::turboshaft::WasmRevecVerifier* verifier) { -@@ -3029,9 +3039,11 @@ class V8_EXPORT_PRIVATE Isolate final : private HiddenFactory { - +@@ -3053,9 +3063,11 @@ class V8_EXPORT_PRIVATE Isolate final : private HiddenFactory { bool is_frozen_ = false; + friend class HandleScopeImplementer; - Tagged promise_context_tag_; - PromiseCrossContextCallback promise_cross_context_callback_; + PromiseCrossContextCallback promise_cross_context_callback_ = nullptr; @@ -256,10 +256,10 @@ index 10bde98a47b9ad44597f51e1dc31b075f36e7b96..7d7a21036bc49b90f3455b2ca72571d9 class PromiseCrossContextCallbackScope; diff --git a/src/heap/factory.cc b/src/heap/factory.cc -index 63d15d64de7a933353dedfbc9aa79f6d60daeb24..4bebad30ca932e25574217104b182e8d5b10769a 100644 +index 56573889afd5888dc6a178e21bc7905e8e183890..5c4f0f4b94ff120a7fd7c303f07aa55a3bb54fdc 100644 --- a/src/heap/factory.cc +++ b/src/heap/factory.cc -@@ -5094,18 +5094,17 @@ Handle Factory::NewJSPromiseWithoutHook() { +@@ -5114,18 +5114,17 @@ Handle Factory::NewJSPromiseWithoutHook() { Handle promise = Cast(NewJSObject(isolate()->promise_function())); DisallowGarbageCollection no_gc; @@ -284,10 +284,10 @@ index 63d15d64de7a933353dedfbc9aa79f6d60daeb24..4bebad30ca932e25574217104b182e8d } diff --git a/src/objects/js-promise.h b/src/objects/js-promise.h -index 2bc8ef88f9f0c3423a1f86bcf7b5e5d6ed2d77bf..aaa280f10e663cf28671cd5699cefb96b7ef1f64 100644 +index 1071da0394a9604dd5b77866817a7802dcf76ecf..12d8ec7ec8c862e5e182ebef59f76639d2f68a51 100644 --- a/src/objects/js-promise.h +++ b/src/objects/js-promise.h -@@ -111,6 +111,13 @@ V8_OBJECT class JSPromise : public JSObjectWithEmbedderSlots { +@@ -113,6 +113,13 @@ V8_OBJECT class JSPromise : public JSObjectWithEmbedderSlots { static_assert(v8::Promise::kFulfilled == 1); static_assert(v8::Promise::kRejected == 2); @@ -302,10 +302,10 @@ index 2bc8ef88f9f0c3423a1f86bcf7b5e5d6ed2d77bf..aaa280f10e663cf28671cd5699cefb96 // Smi 0 terminated list of PromiseReaction objects in case the JSPromise // was not settled yet, otherwise the result. diff --git a/src/objects/objects.cc b/src/objects/objects.cc -index fc4c43b02f0df02a13f8fe01ab43fdb014fd4abb..3484878e6d11e30adce8c1bc8e155fcaa63ab81d 100644 +index 0563f0d73a26e10c4ac5ccd704c68e54b1f3c638..a2357ac4ebc4776b44688eeab5025d47485df25b 100644 --- a/src/objects/objects.cc +++ b/src/objects/objects.cc -@@ -4678,6 +4678,28 @@ Handle JSPromise::Fulfill(DirectHandle promise, +@@ -4700,6 +4700,28 @@ Handle JSPromise::Fulfill(DirectHandle promise, } #endif @@ -334,7 +334,7 @@ index fc4c43b02f0df02a13f8fe01ab43fdb014fd4abb..3484878e6d11e30adce8c1bc8e155fca // 1. Assert: The value of promise.[[PromiseState]] is "pending". CHECK_EQ(Promise::kPending, promise->status()); -@@ -4730,6 +4752,25 @@ Handle JSPromise::Reject(DirectHandle promise, +@@ -4752,6 +4774,25 @@ Handle JSPromise::Reject(DirectHandle promise, isolate->RunAllPromiseHooks(PromiseHookType::kResolve, promise, isolate->factory()->undefined_value()); @@ -360,7 +360,7 @@ index fc4c43b02f0df02a13f8fe01ab43fdb014fd4abb..3484878e6d11e30adce8c1bc8e155fca // 1. Assert: The value of promise.[[PromiseState]] is "pending". CHECK_EQ(Promise::kPending, promise->status()); -@@ -4858,6 +4899,33 @@ MaybeHandle JSPromise::Resolve(DirectHandle promise, +@@ -4880,6 +4921,33 @@ MaybeHandle JSPromise::Resolve(DirectHandle promise, } // static @@ -395,10 +395,10 @@ index fc4c43b02f0df02a13f8fe01ab43fdb014fd4abb..3484878e6d11e30adce8c1bc8e155fca Isolate* isolate, DirectHandle reactions, DirectHandle argument, PromiseReaction::Type type) { diff --git a/src/roots/roots.h b/src/roots/roots.h -index 47485ae9bcfe79d29168ca86d23a6d1a2016bebe..e8574e653096fe489fe1af808c1c9e6be6fe7aa1 100644 +index bb32bfe04e90b37b64338adf33807cd7d46ec7c9..8a0af47abb29420e5a896ba710326a7738150211 100644 --- a/src/roots/roots.h +++ b/src/roots/roots.h -@@ -451,7 +451,8 @@ class RootVisitor; +@@ -453,7 +453,8 @@ class RootVisitor; V(FunctionTemplateInfo, error_stack_getter_fun_template, \ ErrorStackGetterSharedFun) \ V(FunctionTemplateInfo, error_stack_setter_fun_template, \ @@ -409,10 +409,10 @@ index 47485ae9bcfe79d29168ca86d23a6d1a2016bebe..e8574e653096fe489fe1af808c1c9e6b // Entries in this list are limited to Smis and are not visited during GC. #define SMI_ROOT_LIST(V) \ diff --git a/src/runtime/runtime-promise.cc b/src/runtime/runtime-promise.cc -index d75b8e7901043e637604bea60585dd3633a85014..4baee17597e102fab03e8662f8fff2c2eae59198 100644 +index 618dbb2f0d6e946c34f0271ec25b143cc0d7338f..87e76df57f58096f82da95c5af72ae87947c0eff 100644 --- a/src/runtime/runtime-promise.cc +++ b/src/runtime/runtime-promise.cc -@@ -177,8 +177,10 @@ RUNTIME_FUNCTION(Runtime_RejectPromise) { +@@ -153,8 +153,10 @@ RUNTIME_FUNCTION(Runtime_RejectPromise) { DirectHandle promise = args.at(0); DirectHandle reason = args.at(1); DirectHandle debug_event = args.at(2); @@ -425,7 +425,7 @@ index d75b8e7901043e637604bea60585dd3633a85014..4baee17597e102fab03e8662f8fff2c2 } RUNTIME_FUNCTION(Runtime_ResolvePromise) { -@@ -266,8 +268,8 @@ RUNTIME_FUNCTION(Runtime_PromiseContextInit) { +@@ -242,8 +244,8 @@ RUNTIME_FUNCTION(Runtime_PromiseContextInit) { if (!isolate->has_promise_context_tag()) { args.at(0)->set_context_tag(Smi::zero()); } else { @@ -436,7 +436,7 @@ index d75b8e7901043e637604bea60585dd3633a85014..4baee17597e102fab03e8662f8fff2c2 } return ReadOnlyRoots(isolate).undefined_value(); } -@@ -281,8 +283,9 @@ RUNTIME_FUNCTION(Runtime_PromiseContextCheck) { +@@ -257,8 +259,9 @@ RUNTIME_FUNCTION(Runtime_PromiseContextCheck) { // If promise.context_tag() is strict equal to isolate.promise_context_tag(), // or if the promise being checked does not have a context tag, we'll just // return promise directly. @@ -448,7 +448,7 @@ index d75b8e7901043e637604bea60585dd3633a85014..4baee17597e102fab03e8662f8fff2c2 return *promise; } -@@ -296,5 +299,23 @@ RUNTIME_FUNCTION(Runtime_PromiseContextCheck) { +@@ -272,5 +275,23 @@ RUNTIME_FUNCTION(Runtime_PromiseContextCheck) { return *result; } @@ -473,10 +473,10 @@ index d75b8e7901043e637604bea60585dd3633a85014..4baee17597e102fab03e8662f8fff2c2 } // namespace internal } // namespace v8 diff --git a/src/runtime/runtime.h b/src/runtime/runtime.h -index a8ca0f8fa3a0c29d411860a8c67d781727adaf6d..7fa891c500c7bc8ac19790b8706686e555705d64 100644 +index c1bc7761429de83818fe4b1266c9061da95f8330..769a0b10bf1b13a2fdb4fa9475f83812b1783b50 100644 --- a/src/runtime/runtime.h +++ b/src/runtime/runtime.h -@@ -449,7 +449,8 @@ constexpr bool CanTriggerGC(T... properties) { +@@ -447,7 +447,8 @@ constexpr bool CanTriggerGC(T... properties) { F(ConstructAggregateErrorHelper, 4, 1) \ F(ConstructInternalAggregateErrorHelper, -1 /* <= 5*/, 1) \ F(PromiseContextInit, 1, 1) \ diff --git a/patches/v8/0014-Add-another-slot-in-the-isolate-for-embedder.patch b/patches/v8/0014-Add-another-slot-in-the-isolate-for-embedder.patch index 3ae28c82157..328306962f0 100644 --- a/patches/v8/0014-Add-another-slot-in-the-isolate-for-embedder.patch +++ b/patches/v8/0014-Add-another-slot-in-the-isolate-for-embedder.patch @@ -6,10 +6,10 @@ Subject: Add another slot in the isolate for embedder Signed-off-by: James M Snell diff --git a/include/v8-internal.h b/include/v8-internal.h -index cf0110b6f596b205cca6af45a3c2345d78a97e2c..93c4329db6905ba0e23bf7f5dd7cc07e92e2a2cb 100644 +index 9427fc2b56e00bd2f09253128c3e42288ccfa50c..65a134f41b8fb2adbee9681edf43dc25990b298c 100644 --- a/include/v8-internal.h +++ b/include/v8-internal.h -@@ -1055,7 +1055,7 @@ class Internals { +@@ -1006,7 +1006,7 @@ class Internals { // AccessorInfo::data and InterceptorInfo::data field. static const int kCallbackInfoDataOffset = 1 * kApiTaggedSize; diff --git a/patches/v8/0015-Add-ValueSerializer-SetTreatProxiesAsHostObjects.patch b/patches/v8/0015-Add-ValueSerializer-SetTreatProxiesAsHostObjects.patch index ce02859f1af..7c185b65804 100644 --- a/patches/v8/0015-Add-ValueSerializer-SetTreatProxiesAsHostObjects.patch +++ b/patches/v8/0015-Add-ValueSerializer-SetTreatProxiesAsHostObjects.patch @@ -30,10 +30,10 @@ index 141f138e08de849e3e02b3b2b346e643b9e40c70..bdcb2831c55e21c6d511f56dfc79a507 * Write raw data in various common formats to the buffer. * Note that integer types are written in base-128 varint format, not with a diff --git a/src/api/api.cc b/src/api/api.cc -index 0e7e6cb2ed80c6b6ea0ec2b55705e904521efa5f..a2a77fb67133562f9e5c29395d8fbc80eea0d408 100644 +index 420cce4e0eb93356c87d616632cb2ed80f51a76f..f89a85739d66cfcb3b4a6692228bb4acacb36dbc 100644 --- a/src/api/api.cc +++ b/src/api/api.cc -@@ -3587,6 +3587,10 @@ void ValueSerializer::SetTreatFunctionsAsHostObjects(bool mode) { +@@ -3617,6 +3617,10 @@ void ValueSerializer::SetTreatFunctionsAsHostObjects(bool mode) { private_->serializer.SetTreatFunctionsAsHostObjects(mode); } @@ -45,10 +45,10 @@ index 0e7e6cb2ed80c6b6ea0ec2b55705e904521efa5f..a2a77fb67133562f9e5c29395d8fbc80 Local value) { auto i_isolate = i::Isolate::Current(); diff --git a/src/objects/value-serializer.cc b/src/objects/value-serializer.cc -index fd54ce9a832b2bcc25a0a538449d0f85df1372d4..17ce9d28f78fc204cab8aadc543e64746331dac0 100644 +index 608501699c7868ce07d0b7c93c68513fe1e6b299..173d5056541e633ea37d9579f26629695ee3ba55 100644 --- a/src/objects/value-serializer.cc +++ b/src/objects/value-serializer.cc -@@ -344,6 +344,10 @@ void ValueSerializer::SetTreatFunctionsAsHostObjects(bool mode) { +@@ -343,6 +343,10 @@ void ValueSerializer::SetTreatFunctionsAsHostObjects(bool mode) { treat_functions_as_host_objects_ = mode; } @@ -59,7 +59,7 @@ index fd54ce9a832b2bcc25a0a538449d0f85df1372d4..17ce9d28f78fc204cab8aadc543e6474 void ValueSerializer::WriteTag(SerializationTag tag) { uint8_t raw_tag = static_cast(tag); WriteRawBytes(&raw_tag, sizeof(raw_tag)); -@@ -616,7 +620,12 @@ Maybe ValueSerializer::WriteJSReceiver( +@@ -615,7 +619,12 @@ Maybe ValueSerializer::WriteJSReceiver( InstanceType instance_type = receiver->map()->instance_type(); if (IsCallable(*receiver)) { if (treat_functions_as_host_objects_) { @@ -73,7 +73,7 @@ index fd54ce9a832b2bcc25a0a538449d0f85df1372d4..17ce9d28f78fc204cab8aadc543e6474 } return ThrowDataCloneError(MessageTemplate::kDataCloneError, receiver); } else if (IsSpecialReceiverInstanceType(instance_type) && -@@ -1283,7 +1292,7 @@ Maybe ValueSerializer::WriteSharedObject( +@@ -1282,7 +1291,7 @@ Maybe ValueSerializer::WriteSharedObject( return ThrowIfOutOfMemory(); } diff --git a/patches/v8/0017-Enable-V8-shared-linkage.patch b/patches/v8/0017-Enable-V8-shared-linkage.patch index 7118244d250..87b5fb51fb7 100644 --- a/patches/v8/0017-Enable-V8-shared-linkage.patch +++ b/patches/v8/0017-Enable-V8-shared-linkage.patch @@ -6,10 +6,10 @@ Subject: Enable V8 shared linkage Signed-off-by: James M Snell diff --git a/BUILD.bazel b/BUILD.bazel -index 4bc28906f59a85c535c35756e684d11450d9aa3f..2bfeca0a2fdaf58ee25584006eaf8b48ef4f0122 100644 +index 1bf532f07dd3e6f45d57df1a2ee744a62e0156a3..6970a5d09ce7574d654161583b6cec023620cdaa 100644 --- a/BUILD.bazel +++ b/BUILD.bazel -@@ -1516,6 +1516,7 @@ filegroup( +@@ -1527,6 +1527,7 @@ filegroup( "src/builtins/constants-table-builder.cc", "src/builtins/constants-table-builder.h", "src/builtins/data-view-ops.h", @@ -17,7 +17,7 @@ index 4bc28906f59a85c535c35756e684d11450d9aa3f..2bfeca0a2fdaf58ee25584006eaf8b48 "src/builtins/profile-data-reader.h", "src/builtins/superspread.h", "src/codegen/aligned-slot-allocator.cc", -@@ -1701,7 +1702,6 @@ filegroup( +@@ -1712,7 +1713,6 @@ filegroup( "src/execution/futex-emulation.h", "src/execution/interrupts-scope.cc", "src/execution/interrupts-scope.h", @@ -25,7 +25,7 @@ index 4bc28906f59a85c535c35756e684d11450d9aa3f..2bfeca0a2fdaf58ee25584006eaf8b48 "src/execution/isolate.h", "src/execution/isolate-data.h", "src/execution/isolate-data-fields.h", -@@ -3332,7 +3332,6 @@ filegroup( +@@ -3353,7 +3353,6 @@ filegroup( filegroup( name = "v8_compiler_files", srcs = [ @@ -33,7 +33,7 @@ index 4bc28906f59a85c535c35756e684d11450d9aa3f..2bfeca0a2fdaf58ee25584006eaf8b48 "src/compiler/access-builder.cc", "src/compiler/access-builder.h", "src/compiler/access-info.cc", -@@ -3943,8 +3942,6 @@ filegroup( +@@ -3970,8 +3969,6 @@ filegroup( "src/builtins/growable-fixed-array-gen.cc", "src/builtins/growable-fixed-array-gen.h", "src/builtins/number-builtins-reducer-inl.h", @@ -42,7 +42,7 @@ index 4bc28906f59a85c535c35756e684d11450d9aa3f..2bfeca0a2fdaf58ee25584006eaf8b48 "src/builtins/setup-builtins-internal.cc", "src/builtins/torque-csa-header-includes.h", "src/codegen/turboshaft-builtins-assembler-inl.h", -@@ -4216,6 +4213,7 @@ filegroup( +@@ -4243,6 +4240,7 @@ filegroup( "src/snapshot/snapshot-empty.cc", "src/snapshot/static-roots-gen.cc", "src/snapshot/static-roots-gen.h", @@ -50,7 +50,7 @@ index 4bc28906f59a85c535c35756e684d11450d9aa3f..2bfeca0a2fdaf58ee25584006eaf8b48 ], ) -@@ -4326,6 +4324,10 @@ filegroup( +@@ -4353,6 +4351,10 @@ filegroup( name = "noicu/snapshot_files", srcs = [ "src/init/setup-isolate-deserialize.cc", @@ -61,7 +61,7 @@ index 4bc28906f59a85c535c35756e684d11450d9aa3f..2bfeca0a2fdaf58ee25584006eaf8b48 ] + select({ "@v8//bazel/config:v8_target_arm": [ "google3/snapshots/arm/noicu/embedded.S", -@@ -4343,6 +4345,7 @@ filegroup( +@@ -4370,6 +4372,7 @@ filegroup( name = "icu/snapshot_files", srcs = [ "src/init/setup-isolate-deserialize.cc", @@ -70,10 +70,10 @@ index 4bc28906f59a85c535c35756e684d11450d9aa3f..2bfeca0a2fdaf58ee25584006eaf8b48 "@v8//bazel/config:v8_target_arm": [ "google3/snapshots/arm/icu/embedded.S", diff --git a/bazel/defs.bzl b/bazel/defs.bzl -index 1a790af6467107e2536690abf52b5230d0d78ce8..664af26f427318cfa37de18bac1cdf420c1d294c 100644 +index c395faec16e9264a066355c17daf46003937b182..16b4222296b39a17cd4002a8b87948318eed5a0a 100644 --- a/bazel/defs.bzl +++ b/bazel/defs.bzl -@@ -304,7 +304,6 @@ def v8_library( +@@ -313,7 +313,6 @@ def v8_library( copts = copts + default.copts, linkopts = linkopts + default.linkopts, alwayslink = 1, @@ -81,7 +81,7 @@ index 1a790af6467107e2536690abf52b5230d0d78ce8..664af26f427318cfa37de18bac1cdf42 **kwargs ) -@@ -323,7 +322,6 @@ def v8_library( +@@ -332,7 +331,6 @@ def v8_library( copts = copts + default.copts + ENABLE_I18N_SUPPORT_DEFINES, linkopts = linkopts + default.linkopts, alwayslink = 1, @@ -89,7 +89,7 @@ index 1a790af6467107e2536690abf52b5230d0d78ce8..664af26f427318cfa37de18bac1cdf42 **kwargs ) -@@ -343,7 +341,6 @@ def v8_library( +@@ -352,7 +350,6 @@ def v8_library( copts = copts + default.copts, linkopts = linkopts + default.linkopts, alwayslink = 1, diff --git a/patches/v8/0018-Modify-where-to-look-for-fast_float-and-simdutf.patch b/patches/v8/0018-Modify-where-to-look-for-fast_float-and-simdutf.patch index 8088feec3bc..a327d0fc761 100644 --- a/patches/v8/0018-Modify-where-to-look-for-fast_float-and-simdutf.patch +++ b/patches/v8/0018-Modify-where-to-look-for-fast_float-and-simdutf.patch @@ -12,10 +12,10 @@ include changes are needed. Signed-off-by: James M Snell diff --git a/BUILD.bazel b/BUILD.bazel -index 2bfeca0a2fdaf58ee25584006eaf8b48ef4f0122..3cd1323be830f8ff174fca64b03bf816030559d1 100644 +index 6970a5d09ce7574d654161583b6cec023620cdaa..8f9091a93b039c9ea08ceb6ab5b92eb726acb084 100644 --- a/BUILD.bazel +++ b/BUILD.bazel -@@ -4635,17 +4635,19 @@ cc_library( +@@ -4666,17 +4666,19 @@ cc_library( ], ) @@ -46,7 +46,7 @@ index 2bfeca0a2fdaf58ee25584006eaf8b48ef4f0122..3cd1323be830f8ff174fca64b03bf816 v8_library( name = "v8_libshared", -@@ -4676,15 +4678,15 @@ v8_library( +@@ -4707,15 +4709,15 @@ v8_library( ], deps = [ ":lib_dragonbox", diff --git a/patches/v8/0019-Remove-unneded-latomic-linker-flag.patch b/patches/v8/0019-Remove-unneded-latomic-linker-flag.patch index 85db5a40c9a..d0f7ba3297a 100644 --- a/patches/v8/0019-Remove-unneded-latomic-linker-flag.patch +++ b/patches/v8/0019-Remove-unneded-latomic-linker-flag.patch @@ -6,10 +6,10 @@ Subject: Remove unneded -latomic linker flag Signed-off-by: James M Snell diff --git a/bazel/defs.bzl b/bazel/defs.bzl -index 664af26f427318cfa37de18bac1cdf420c1d294c..cfdc3e38ff98e7ff956a05f1b2afba95282c4285 100644 +index 16b4222296b39a17cd4002a8b87948318eed5a0a..540594124b0e518965caf1e55b211843a8b3283a 100644 --- a/bazel/defs.bzl +++ b/bazel/defs.bzl -@@ -211,7 +211,7 @@ def _default_args(): +@@ -220,7 +220,7 @@ def _default_args(): "Shell32.lib", ], "@v8//bazel/config:is_macos": ["-pthread"], diff --git a/patches/v8/0020-Add-methods-to-get-heap-and-external-memory-sizes-di.patch b/patches/v8/0020-Add-methods-to-get-heap-and-external-memory-sizes-di.patch index c4705d99607..7765d0fb860 100644 --- a/patches/v8/0020-Add-methods-to-get-heap-and-external-memory-sizes-di.patch +++ b/patches/v8/0020-Add-methods-to-get-heap-and-external-memory-sizes-di.patch @@ -8,10 +8,10 @@ Subject: Add methods to get heap and external memory sizes directly. Signed-off-by: James M Snell diff --git a/include/v8-isolate.h b/include/v8-isolate.h -index 90ec2bf7625e25017a633b845be86bda1c2d7880..79022fc13d745cd016be2aa07f551418c4a30085 100644 +index 7d25bc6c25faac2b36441a63f186085e416eaccb..66d7dec9d5a5decb6dd3f7d2e377ecebd8d533ed 100644 --- a/include/v8-isolate.h +++ b/include/v8-isolate.h -@@ -1085,6 +1085,16 @@ class V8_EXPORT Isolate { +@@ -1088,6 +1088,16 @@ class V8_EXPORT Isolate { V8_DEPRECATE_SOON("Use ExternalMemoryAccounter instead.") int64_t AdjustAmountOfExternalAllocatedMemory(int64_t change_in_bytes); @@ -29,10 +29,10 @@ index 90ec2bf7625e25017a633b845be86bda1c2d7880..79022fc13d745cd016be2aa07f551418 * Returns heap profiler for this isolate. Will return NULL until the isolate * is initialized. diff --git a/src/api/api.cc b/src/api/api.cc -index a2a77fb67133562f9e5c29395d8fbc80eea0d408..13192517ba0b0c6d17748a1a45a30b94a7baea15 100644 +index f89a85739d66cfcb3b4a6692228bb4acacb36dbc..fc27f5939e758b1d9515f38b0969d6bf11f8a82c 100644 --- a/src/api/api.cc +++ b/src/api/api.cc -@@ -10466,6 +10466,14 @@ void Isolate::GetHeapStatistics(HeapStatistics* heap_statistics) { +@@ -10560,6 +10560,14 @@ void Isolate::GetHeapStatistics(HeapStatistics* heap_statistics) { #endif // V8_ENABLE_WEBASSEMBLY } diff --git a/patches/v8/0021-Port-concurrent-mksnapshot-support.patch b/patches/v8/0021-Port-concurrent-mksnapshot-support.patch index 1b3d285e749..18d4fd0cb24 100644 --- a/patches/v8/0021-Port-concurrent-mksnapshot-support.patch +++ b/patches/v8/0021-Port-concurrent-mksnapshot-support.patch @@ -6,10 +6,10 @@ Subject: Port concurrent mksnapshot support Change-Id: I57c8158ff5d624e5379e6b072f27ac7a40419522 diff --git a/BUILD.bazel b/BUILD.bazel -index 3cd1323be830f8ff174fca64b03bf816030559d1..60c6d5f762e9e325ccb8dbe9518b47a251a7e8e3 100644 +index 8f9091a93b039c9ea08ceb6ab5b92eb726acb084..4521044b5ae2a2c36d0b141f1149cc632a6f2cf3 100644 --- a/BUILD.bazel +++ b/BUILD.bazel -@@ -120,6 +120,11 @@ v8_flag(name = "v8_enable_hugepage") +@@ -126,6 +126,11 @@ v8_flag(name = "v8_enable_hugepage") v8_flag(name = "v8_enable_fast_mksnapshot") @@ -21,7 +21,7 @@ index 3cd1323be830f8ff174fca64b03bf816030559d1..60c6d5f762e9e325ccb8dbe9518b47a2 v8_flag(name = "v8_enable_future") # NOTE: Transitions are not recommended in library targets: -@@ -4556,6 +4561,13 @@ v8_mksnapshot( +@@ -4585,6 +4590,13 @@ v8_mksnapshot( "--no-turbo-verify-allocation", ], "//conditions:default": [], diff --git a/patches/v8/0022-Port-V8_USE_ZLIB-support.patch b/patches/v8/0022-Port-V8_USE_ZLIB-support.patch index f2fe3927040..0c576829c7e 100644 --- a/patches/v8/0022-Port-V8_USE_ZLIB-support.patch +++ b/patches/v8/0022-Port-V8_USE_ZLIB-support.patch @@ -6,10 +6,10 @@ Subject: Port V8_USE_ZLIB support Change-Id: Icfedf3e90522f1ff5037517a39a5f0e3d44abace diff --git a/BUILD.bazel b/BUILD.bazel -index 60c6d5f762e9e325ccb8dbe9518b47a251a7e8e3..23665c89fd04db517ccf0ab8dccb5346cd5cac45 100644 +index 4521044b5ae2a2c36d0b141f1149cc632a6f2cf3..bd86f080c6e96327aec1addaa11cddd7eb947dd4 100644 --- a/BUILD.bazel +++ b/BUILD.bazel -@@ -162,6 +162,11 @@ v8_flag(name = "v8_enable_verify_predictable") +@@ -168,6 +168,11 @@ v8_flag(name = "v8_enable_verify_predictable") v8_flag(name = "v8_enable_test_features") @@ -21,7 +21,7 @@ index 60c6d5f762e9e325ccb8dbe9518b47a251a7e8e3..23665c89fd04db517ccf0ab8dccb5346 v8_flag(name = "v8_wasm_random_fuzzers") v8_flag( -@@ -533,6 +538,7 @@ v8_config( +@@ -540,6 +545,7 @@ v8_config( "v8_enable_vtunejit": "ENABLE_VTUNE_JIT_INTERFACE", "v8_enable_undefined_double": "V8_ENABLE_UNDEFINED_DOUBLE", "v8_use_host_cpu_arm_features": "V8_USE_HOST_CPU_ARM_FEATURES", @@ -29,7 +29,7 @@ index 60c6d5f762e9e325ccb8dbe9518b47a251a7e8e3..23665c89fd04db517ccf0ab8dccb5346 }, defines = [ "GOOGLE3", -@@ -4699,6 +4705,8 @@ v8_library( +@@ -4730,6 +4736,8 @@ v8_library( "@highway//:hwy", "@fast_float", "@simdutf", @@ -39,11 +39,11 @@ index 60c6d5f762e9e325ccb8dbe9518b47a251a7e8e3..23665c89fd04db517ccf0ab8dccb5346 ) diff --git a/src/deoptimizer/frame-translation-builder.cc b/src/deoptimizer/frame-translation-builder.cc -index 497f11e5cd2c20b33b86a71615fb12e48c32048f..b2bbe6aca39bec6299551be0932eda89034775e8 100644 +index d8d430ea39e6e1ede1125b6230d9ecd2ec275535..3433a528b3adaf1dc2a7310222311cf623ed7f25 100644 --- a/src/deoptimizer/frame-translation-builder.cc +++ b/src/deoptimizer/frame-translation-builder.cc -@@ -11,7 +11,7 @@ - #include "src/objects/fixed-array-inl.h" +@@ -13,7 +13,7 @@ + #include "src/objects/fixed-primitive-array-inl.h" #ifdef V8_USE_ZLIB -#include "third_party/zlib/google/compression_utils_portable.h" diff --git a/patches/v8/0023-Modify-where-to-look-for-dragonbox.patch b/patches/v8/0023-Modify-where-to-look-for-dragonbox.patch index 49addb7bd19..86d092a196b 100644 --- a/patches/v8/0023-Modify-where-to-look-for-dragonbox.patch +++ b/patches/v8/0023-Modify-where-to-look-for-dragonbox.patch @@ -5,10 +5,10 @@ Subject: Modify where to look for dragonbox diff --git a/BUILD.bazel b/BUILD.bazel -index 23665c89fd04db517ccf0ab8dccb5346cd5cac45..63682dc1766c6036fd8f10ff717589a85bb25c23 100644 +index bd86f080c6e96327aec1addaa11cddd7eb947dd4..8fd8f17debe018b1c17e07b6a0d42ea3f912e8c9 100644 --- a/BUILD.bazel +++ b/BUILD.bazel -@@ -4151,14 +4151,9 @@ filegroup( +@@ -4178,14 +4178,9 @@ filegroup( ) v8_library( diff --git a/patches/v8/0024-Disable-slow-handle-check.patch b/patches/v8/0024-Disable-slow-handle-check.patch index 817be4ca9d7..e4763c19aef 100644 --- a/patches/v8/0024-Disable-slow-handle-check.patch +++ b/patches/v8/0024-Disable-slow-handle-check.patch @@ -6,7 +6,7 @@ Subject: Disable slow handle check Signed-off-by: James M Snell diff --git a/src/handles/handles.h b/src/handles/handles.h -index 9367344523f562d00a6bf129fa6ed053fa3d2ce1..bc84ff7f491f36625e72291e128e332b76eefa26 100644 +index 1a2f93c37552c489314f7265f6b7d416047d372f..c1c05fbdd597d8f7cd65c937694a3712ae0736ea 100644 --- a/src/handles/handles.h +++ b/src/handles/handles.h @@ -636,11 +636,7 @@ IndirectHandle indirect_handle(DirectHandle handle, diff --git a/patches/v8/0026-Implement-additional-Exception-construction-methods.patch b/patches/v8/0026-Implement-additional-Exception-construction-methods.patch index 48ebed2dcb8..7c23fbf05b2 100644 --- a/patches/v8/0026-Implement-additional-Exception-construction-methods.patch +++ b/patches/v8/0026-Implement-additional-Exception-construction-methods.patch @@ -25,10 +25,10 @@ index f240d9a609e92b4a3055256996ad69d8fc14ac49..f8546f34d207e4e2e6fd1c5d8b87b83b /** * Creates an error message for the given exception. diff --git a/src/api/api.cc b/src/api/api.cc -index 13192517ba0b0c6d17748a1a45a30b94a7baea15..723181b6eb39d2deabaa65fd0cc8109483f32640 100644 +index fc27f5939e758b1d9515f38b0969d6bf11f8a82c..4b9c6a9be83d9bb809a16bf026c45191b3493b7b 100644 --- a/src/api/api.cc +++ b/src/api/api.cc -@@ -11357,6 +11357,10 @@ DEFINE_ERROR(WasmCompileError, wasm_compile_error) +@@ -11456,6 +11456,10 @@ DEFINE_ERROR(WasmCompileError, wasm_compile_error) DEFINE_ERROR(WasmLinkError, wasm_link_error) DEFINE_ERROR(WasmRuntimeError, wasm_runtime_error) DEFINE_ERROR(WasmSuspendError, wasm_suspend_error) @@ -40,7 +40,7 @@ index 13192517ba0b0c6d17748a1a45a30b94a7baea15..723181b6eb39d2deabaa65fd0cc81094 #undef DEFINE_ERROR diff --git a/src/logging/runtime-call-stats.h b/src/logging/runtime-call-stats.h -index b21bc8318dc4675c1acb6b17fdcdde32da71cf96..f2cddea8b99c6d59751d7f88419c65783759839b 100644 +index 85ee617e3d9e429824acff33239b99872572fa56..a5bbe4b1849e310b2a9d97d3d158a2073e1e4786 100644 --- a/src/logging/runtime-call-stats.h +++ b/src/logging/runtime-call-stats.h @@ -219,7 +219,11 @@ namespace v8::internal { diff --git a/patches/v8/0028-bind-icu-to-googlesource.patch b/patches/v8/0028-bind-icu-to-googlesource.patch index 37d9b71eb79..8409a0d57e9 100644 --- a/patches/v8/0028-bind-icu-to-googlesource.patch +++ b/patches/v8/0028-bind-icu-to-googlesource.patch @@ -5,10 +5,10 @@ Subject: bind icu to googlesource diff --git a/BUILD.bazel b/BUILD.bazel -index 63682dc1766c6036fd8f10ff717589a85bb25c23..3d7a0d12c7e980ede305470b2212d3d2cb08eef1 100644 +index 8fd8f17debe018b1c17e07b6a0d42ea3f912e8c9..f537b7f661b855f60f7b00f32f741e517e64fb86 100644 --- a/BUILD.bazel +++ b/BUILD.bazel -@@ -4676,7 +4676,7 @@ v8_library( +@@ -4707,7 +4707,7 @@ v8_library( copts = ["-Wno-implicit-fallthrough"], icu_deps = [ ":icu/generated_torque_definitions_headers", @@ -17,7 +17,7 @@ index 63682dc1766c6036fd8f10ff717589a85bb25c23..3d7a0d12c7e980ede305470b2212d3d2 ], icu_srcs = [ ":generated_regexp_special_case", -@@ -4799,7 +4799,7 @@ v8_binary( +@@ -4830,7 +4830,7 @@ v8_binary( ], deps = [ ":v8_libbase", diff --git a/patches/v8/0029-Add-v8-String-IsFlat-API.patch b/patches/v8/0029-Add-v8-String-IsFlat-API.patch index 0efc84efdf0..dc961564253 100644 --- a/patches/v8/0029-Add-v8-String-IsFlat-API.patch +++ b/patches/v8/0029-Add-v8-String-IsFlat-API.patch @@ -24,10 +24,10 @@ index 8466df80175de9362dd785d83fa527a20a3be1a6..c28282462218fb5e7c66face4402e39d enum { kNone = 0, diff --git a/src/api/api.cc b/src/api/api.cc -index 723181b6eb39d2deabaa65fd0cc8109483f32640..af43652cab06174938e10cf50ea03ab6eb51f047 100644 +index 4b9c6a9be83d9bb809a16bf026c45191b3493b7b..ab4f3a21b9c36b9874c8ac61e9862593b6c9fc89 100644 --- a/src/api/api.cc +++ b/src/api/api.cc -@@ -5825,6 +5825,10 @@ bool String::IsOneByte() const { +@@ -5885,6 +5885,10 @@ bool String::IsOneByte() const { return Utils::OpenDirectHandle(this)->IsOneByteRepresentation(); } diff --git a/patches/v8/0030-Expose-AdjustAmountOfExternalAllocatedMemoryImpl-as-.patch b/patches/v8/0030-Expose-AdjustAmountOfExternalAllocatedMemoryImpl-as-.patch index 3f214ce981a..3b2b7c0b8a7 100644 --- a/patches/v8/0030-Expose-AdjustAmountOfExternalAllocatedMemoryImpl-as-.patch +++ b/patches/v8/0030-Expose-AdjustAmountOfExternalAllocatedMemoryImpl-as-.patch @@ -14,10 +14,10 @@ method. This patch simply makes it public for embedder use. Signed-off-by: Aditya Tewari diff --git a/include/v8-isolate.h b/include/v8-isolate.h -index 79022fc13d745cd016be2aa07f551418c4a30085..01e48a3399a734e7e5841303c2ced1afabd3db65 100644 +index 66d7dec9d5a5decb6dd3f7d2e377ecebd8d533ed..e31a532dcb71caf75b68dfc9b119152ae8d2eb23 100644 --- a/include/v8-isolate.h +++ b/include/v8-isolate.h -@@ -1095,6 +1095,14 @@ class V8_EXPORT Isolate { +@@ -1098,6 +1098,14 @@ class V8_EXPORT Isolate { */ size_t GetHeapSize(); @@ -32,7 +32,7 @@ index 79022fc13d745cd016be2aa07f551418c4a30085..01e48a3399a734e7e5841303c2ced1af /** * Returns heap profiler for this isolate. Will return NULL until the isolate * is initialized. -@@ -1911,7 +1919,6 @@ class V8_EXPORT Isolate { +@@ -1913,7 +1921,6 @@ class V8_EXPORT Isolate { internal::ValueHelper::InternalRepresentationType GetDataFromSnapshotOnce( size_t index); diff --git a/patches/v8/0031-Add-verify_write_barriers-flag-in-V8-s-bazel-config.patch b/patches/v8/0031-Add-verify_write_barriers-flag-in-V8-s-bazel-config.patch index a909140e353..bc217374c75 100644 --- a/patches/v8/0031-Add-verify_write_barriers-flag-in-V8-s-bazel-config.patch +++ b/patches/v8/0031-Add-verify_write_barriers-flag-in-V8-s-bazel-config.patch @@ -5,10 +5,10 @@ Subject: Add verify_write_barriers flag in V8's bazel config diff --git a/BUILD.bazel b/BUILD.bazel -index 3d7a0d12c7e980ede305470b2212d3d2cb08eef1..20f28d46fb9d9ce396455eca12409f2a8cf15c5b 100644 +index f537b7f661b855f60f7b00f32f741e517e64fb86..7f40324dfedb267cbb4c0bb3529b79ad2460324e 100644 --- a/BUILD.bazel +++ b/BUILD.bazel -@@ -552,6 +552,7 @@ v8_config( +@@ -559,6 +559,7 @@ v8_config( "@v8//bazel/config:is_debug": [ "DEBUG", "V8_ENABLE_CHECKS", diff --git a/patches/v8/0032-Change-lamba-signature-to-get-around-windows-build-f.patch b/patches/v8/0032-Change-lamba-signature-to-get-around-windows-build-f.patch index 5e060d4c684..4f0dc6f53fd 100644 --- a/patches/v8/0032-Change-lamba-signature-to-get-around-windows-build-f.patch +++ b/patches/v8/0032-Change-lamba-signature-to-get-around-windows-build-f.patch @@ -5,10 +5,10 @@ Subject: Change lamba signature to get around windows build failure diff --git a/src/objects/backing-store.cc b/src/objects/backing-store.cc -index 5297cd116b571bb4c679c4f44a3c58781dcc712d..f9e815e3a47fe8caa1bc874d06e57d7c8b3f999f 100644 +index c98ab72ad532350404f1fac0bedf3c4e83a1ab03..e43b8ea2e693b9981b18a87191ebee5b903d4872 100644 --- a/src/objects/backing-store.cc +++ b/src/objects/backing-store.cc -@@ -321,7 +321,7 @@ std::unique_ptr BackingStore::TryAllocateAndPartiallyCommitMemory( +@@ -322,7 +322,7 @@ std::unique_ptr BackingStore::TryAllocateAndPartiallyCommitMemory( // For accounting purposes, whether a GC was necessary. bool did_retry = false; diff --git a/patches/v8/0033-Return-false-on-Object.hasOwnProperty-with-intercept.patch b/patches/v8/0033-Return-false-on-Object.hasOwnProperty-with-intercept.patch index 82b87aea84e..0d552b4d6da 100644 --- a/patches/v8/0033-Return-false-on-Object.hasOwnProperty-with-intercept.patch +++ b/patches/v8/0033-Return-false-on-Object.hasOwnProperty-with-intercept.patch @@ -5,10 +5,10 @@ Subject: Return false on Object.hasOwnProperty with interceptors diff --git a/src/objects/js-objects.cc b/src/objects/js-objects.cc -index 7813b7777b39b2135acd74f8e98a74e6c2ec0c64..adea65e73e615a616c032bf188f476f4e1204177 100644 +index a3b0b130d7b037c7f91c22c9aaabc82d6d724f2f..9f1be52a113a3ad19eb0b5f178cc301bcb5e333b 100644 --- a/src/objects/js-objects.cc +++ b/src/objects/js-objects.cc -@@ -158,6 +158,9 @@ Maybe JSReceiver::HasOwnProperty(Isolate* isolate, +@@ -162,6 +162,9 @@ Maybe JSReceiver::HasOwnProperty(Isolate* isolate, if (IsJSObject(*object)) { // Shortcut. PropertyKey key(isolate, name); LookupIterator it(isolate, object, key, LookupIterator::OWN); diff --git a/patches/v8/0034-Remove-V8-MODULE.bazel-llvm-toolchain-and-libcxx-rep.patch b/patches/v8/0034-Remove-V8-MODULE.bazel-llvm-toolchain-and-libcxx-rep.patch index 6f5f3f85b3b..989784f90ba 100644 --- a/patches/v8/0034-Remove-V8-MODULE.bazel-llvm-toolchain-and-libcxx-rep.patch +++ b/patches/v8/0034-Remove-V8-MODULE.bazel-llvm-toolchain-and-libcxx-rep.patch @@ -7,7 +7,7 @@ These reference third_party/ sources that are not present in the GitHub tarball. Workerd provides its own toolchain, so these are not needed. diff --git a/MODULE.bazel b/MODULE.bazel -index b8bf8bd29c7cd5834b20cc2f762f45befd093e1b..804316256e142358ddf8a4eb6d75393d1c8ae9ca 100644 +index 00ad66496a5e418d4a3f419d39cb54301fa0d64d..f66c1566f509a0ce8e3f3bba94906f05840aa4c6 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -22,170 +22,6 @@ pip.parse( diff --git a/patches/v8/0035-Remove-libcxx-dep-from-defs.bzl-not-resolvable-via-h.patch b/patches/v8/0035-Remove-libcxx-dep-from-defs.bzl-not-resolvable-via-h.patch index a4c59692c9e..ef52619fbd3 100644 --- a/patches/v8/0035-Remove-libcxx-dep-from-defs.bzl-not-resolvable-via-h.patch +++ b/patches/v8/0035-Remove-libcxx-dep-from-defs.bzl-not-resolvable-via-h.patch @@ -11,10 +11,10 @@ own toolchain that includes libc++. Author: Workerd Maintainers diff --git a/bazel/defs.bzl b/bazel/defs.bzl -index cfdc3e38ff98e7ff956a05f1b2afba95282c4285..7e29f81545f97b10b2064f5fae2179646ed0f80f 100644 +index 540594124b0e518965caf1e55b211843a8b3283a..32978e22e3c51cdc5bd4fe0bc196cf8b852bd437 100644 --- a/bazel/defs.bzl +++ b/bazel/defs.bzl -@@ -97,7 +97,7 @@ v8_config = rule( +@@ -106,7 +106,7 @@ v8_config = rule( def _default_args(): return struct( diff --git a/patches/v8/0037-Fix-ExtendedMap-layout-on-Windows.patch b/patches/v8/0036-Fix-ExtendedMap-layout-on-Windows.patch similarity index 89% rename from patches/v8/0037-Fix-ExtendedMap-layout-on-Windows.patch rename to patches/v8/0036-Fix-ExtendedMap-layout-on-Windows.patch index be691481141..e593e25ad0d 100644 --- a/patches/v8/0037-Fix-ExtendedMap-layout-on-Windows.patch +++ b/patches/v8/0036-Fix-ExtendedMap-layout-on-Windows.patch @@ -38,10 +38,10 @@ index 7a0838bb84d985467fd495e5b5fd10764bf82bf0..25f1d670094ec2f43739c992314fc474 indexed_interceptor: InterceptorInfo; } diff --git a/src/objects/map.h b/src/objects/map.h -index 6b2c8e6acf7c1aa4f428af0ffce7cbb0da95c71d..0750c476a422034da005a2f65fd31f2c8aadfb38 100644 +index b907c3ddc112618f40480e06b314213587b90e1c..9eb229400c8d29e84e1b599c14ffa6224aaa7b40 100644 --- a/src/objects/map.h +++ b/src/objects/map.h -@@ -1285,7 +1285,8 @@ V8_ABSTRACT_OBJECT class ExtendedMap : public Map { +@@ -1290,7 +1290,8 @@ V8_ABSTRACT_OBJECT class ExtendedMap : public Map { static const int kStartOfStrongExtendedFieldsOffset; std::atomic bit_field_ex_; @@ -52,10 +52,10 @@ index 6b2c8e6acf7c1aa4f428af0ffce7cbb0da95c71d..0750c476a422034da005a2f65fd31f2c constexpr int ExtendedMapSizeForKind(ExtendedMapKind kind); diff --git a/src/objects/map.tq b/src/objects/map.tq -index a32cb5bdc32ecfe48a0b885b900acefcf763767c..e34b51ae8b01e6683fd7d599001b1baf81f8c95b 100644 +index cb85e7788e6a38116fc690341ca51f6288869247..687c95966d88c31b48ff3cb045c452762b2af237 100644 --- a/src/objects/map.tq +++ b/src/objects/map.tq -@@ -113,7 +113,9 @@ bitfield struct ExtendedMapBitFields extends uint8 { +@@ -116,7 +116,9 @@ bitfield struct ExtendedMapBitFields extends uint8 { @doNotGenerateCast extern class ExtendedMap extends Map { bit_field_ex: ExtendedMapBitFields; diff --git a/patches/v8/0036-Fix-non-portable-std-atomic_flag-construction-in-run.patch b/patches/v8/0036-Fix-non-portable-std-atomic_flag-construction-in-run.patch deleted file mode 100644 index c9b231e7ee4..00000000000 --- a/patches/v8/0036-Fix-non-portable-std-atomic_flag-construction-in-run.patch +++ /dev/null @@ -1,24 +0,0 @@ -From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 -From: Dan Carney -Date: Tue, 5 May 2026 16:08:11 +0000 -Subject: Fix non-portable std::atomic_flag construction in runtime-test.cc - -std::atomic_flag has only a default constructor (per C++20+); MSVC's -standard library correctly rejects 'std::atomic_flag flag{false}'. -Initial state of a default-constructed atomic_flag is unspecified -before C++20, but cleared from C++20 onward, which matches the -intent of the original code (which used the equivalent of false). - -diff --git a/src/runtime/runtime-test.cc b/src/runtime/runtime-test.cc -index 49c40c126b1f00a1380f09dc147fc14a9fce2f13..33884dfcf2ab7da6bfde12c073116ac38c2518b6 100644 ---- a/src/runtime/runtime-test.cc -+++ b/src/runtime/runtime-test.cc -@@ -1193,7 +1193,7 @@ RUNTIME_FUNCTION(Runtime_SetAllocationTimeout) { - CONVERT_INT32_ARG_FUZZ_SAFE(timeout, 1); - isolate->heap()->set_allocation_timeout(timeout); - #else // !V8_ENABLE_ALLOCATION_TIMEOUT -- static std::atomic_flag printed_warning{false}; -+ static std::atomic_flag printed_warning; - if (!printed_warning.test_and_set()) { - base::OS::PrintError( - "Warning: %%SetAllocationTimeout has no effect in this build. Set the " diff --git a/patches/v8/0038-Fix-CFunction-MemorySpan-declarations-on-Windows.patch b/patches/v8/0037-Fix-CFunction-MemorySpan-declarations-on-Windows.patch similarity index 87% rename from patches/v8/0038-Fix-CFunction-MemorySpan-declarations-on-Windows.patch rename to patches/v8/0037-Fix-CFunction-MemorySpan-declarations-on-Windows.patch index ecd2954b627..738a6f75fc4 100644 --- a/patches/v8/0038-Fix-CFunction-MemorySpan-declarations-on-Windows.patch +++ b/patches/v8/0037-Fix-CFunction-MemorySpan-declarations-on-Windows.patch @@ -11,17 +11,17 @@ which leaves the type incomplete and fails under clang-cl with MSVC's STL. Include v8-fast-api-calls.h before these declarations so CFunction is complete. diff --git a/include/v8-template.h b/include/v8-template.h -index 5a890e70cac28a91b54888339aace254b2e29d18..c2d72da08c1b1dd1e93bb9df395456ad87537be2 100644 +index f270c74681ad3eeb35500697dbccef24b1949f93..103bbf2a1f626978f9de718cd1e14ad67f387005 100644 --- a/include/v8-template.h +++ b/include/v8-template.h -@@ -10,6 +10,7 @@ +@@ -11,6 +11,7 @@ #include "v8-data.h" // NOLINT(build/include_directory) #include "v8-exception.h" // NOLINT(build/include_directory) +#include "v8-fast-api-calls.h" // NOLINT(build/include_directory) #include "v8-function-callback.h" // NOLINT(build/include_directory) #include "v8-local-handle.h" // NOLINT(build/include_directory) - #include "v8-memory-span.h" // NOLINT(build/include_directory) + #include "v8-object.h" // NOLINT(build/include_directory) @@ -18,7 +19,6 @@ namespace v8 { diff --git a/patches/v8/0039-Properly-depend-on-llvm-libc.patch b/patches/v8/0038-Properly-depend-on-llvm-libc.patch similarity index 84% rename from patches/v8/0039-Properly-depend-on-llvm-libc.patch rename to patches/v8/0038-Properly-depend-on-llvm-libc.patch index 1245f02602b..27fdad6fa2c 100644 --- a/patches/v8/0039-Properly-depend-on-llvm-libc.patch +++ b/patches/v8/0038-Properly-depend-on-llvm-libc.patch @@ -6,10 +6,10 @@ Subject: Properly depend on llvm-libc Change-Id: I9858b147d027e67704c6ca84e0c42fab39a5c97a diff --git a/BUILD.bazel b/BUILD.bazel -index 20f28d46fb9d9ce396455eca12409f2a8cf15c5b..983c7e2e8a0d6605dcb420adfafaccde9b1c05ce 100644 +index 7f40324dfedb267cbb4c0bb3529b79ad2460324e..3a4f48806cd0018a70cfa9cda0c192c2e55ffaa1 100644 --- a/BUILD.bazel +++ b/BUILD.bazel -@@ -4600,13 +4600,6 @@ cc_library( +@@ -4629,13 +4629,6 @@ cc_library( strip_include_prefix = "noicu", ) @@ -23,7 +23,7 @@ index 20f28d46fb9d9ce396455eca12409f2a8cf15c5b..983c7e2e8a0d6605dcb420adfafaccde v8_library( name = "v8_libbase", srcs = [ -@@ -4615,7 +4608,7 @@ v8_library( +@@ -4644,7 +4637,7 @@ v8_library( ], copts = ["-Wno-implicit-fallthrough"], deps = [ diff --git a/src/workerd/api/global-scope.c++ b/src/workerd/api/global-scope.c++ index ec8589f46f6..98918666546 100644 --- a/src/workerd/api/global-scope.c++ +++ b/src/workerd/api/global-scope.c++ @@ -37,6 +37,8 @@ #include #include +#include + #include namespace workerd::api { @@ -1019,7 +1021,7 @@ void ServiceWorkerGlobalScope::queueMicrotask(jsg::Lock& js, jsg::FunctionEnqueueMicrotask(fn); + js.v8Context()->GetMicrotaskQueue()->EnqueueMicrotask(js.v8Isolate, fn); } jsg::JsValue ServiceWorkerGlobalScope::structuredClone( diff --git a/src/workerd/api/unsafe.c++ b/src/workerd/api/unsafe.c++ index 10e295ec311..96fa9e24cdb 100644 --- a/src/workerd/api/unsafe.c++ +++ b/src/workerd/api/unsafe.c++ @@ -6,6 +6,8 @@ #include #include +#include + namespace workerd::api { namespace { @@ -205,7 +207,7 @@ jsg::JsValue UnsafeEval::newWasmModule(jsg::Lock& js, kj::Array src) { KJ_DEFER(js.setAllowEval(false)); auto maybeWasmModule = v8::WasmModuleObject::Compile( - js.v8Isolate, v8::MemorySpan(src.begin(), src.size())); + js.v8Isolate, std::span(src.begin(), src.size())); return jsg::JsValue(jsg::check(maybeWasmModule)); } diff --git a/src/workerd/io/worker-fs.c++ b/src/workerd/io/worker-fs.c++ index 97f80392d9f..9bce914fdf1 100644 --- a/src/workerd/io/worker-fs.c++ +++ b/src/workerd/io/worker-fs.c++ @@ -5,6 +5,8 @@ #include #include +#include + #include namespace workerd { @@ -1998,7 +2000,7 @@ class StdioFile final: public File { } }); }); - js.v8Isolate->EnqueueMicrotask(callback); + js.v8Context()->GetMicrotaskQueue()->EnqueueMicrotask(js.v8Isolate, callback); } }; diff --git a/src/workerd/jsg/jsg.h b/src/workerd/jsg/jsg.h index f96a240e93b..312120c2693 100644 --- a/src/workerd/jsg/jsg.h +++ b/src/workerd/jsg/jsg.h @@ -31,6 +31,8 @@ #include #include +#include + using kj::byte; using kj::uint; @@ -720,7 +722,7 @@ concept HasStructTypeScriptDefine = requires { T::_JSG_STRUCT_TS_DEFINE_DO_NOT_U JSG_FOR_EACH(JSG_STRUCT_FIELD_COL, , __VA_ARGS__); \ auto namesPtr = names.asPtr().asConst(); \ return v8::DictionaryTemplate::New( \ - isolate, v8::MemorySpan(namesPtr.begin(), namesPtr.size())); \ + isolate, std::span(namesPtr.begin(), namesPtr.size())); \ } \ template \ static void registerMembersInternal(Registry& registry, Config arg) { \ diff --git a/src/workerd/jsg/modules-new.c++ b/src/workerd/jsg/modules-new.c++ index 22f08dd1ab5..6dfc7d0570e 100644 --- a/src/workerd/jsg/modules-new.c++ +++ b/src/workerd/jsg/modules-new.c++ @@ -13,6 +13,8 @@ #include #include +#include + namespace workerd::jsg::modules { namespace { @@ -277,7 +279,7 @@ class SyntheticModule final: public Module { exports[n++] = js.strIntern(exp); } return v8::Module::CreateSyntheticModule(js.v8Isolate, js.str(id().getHref()), - v8::MemorySpan>(exports.data(), exports.size()), + std::span>(exports.data(), exports.size()), evaluationSteps); } diff --git a/src/workerd/jsg/modules.c++ b/src/workerd/jsg/modules.c++ index 98470511c86..ede9aafd886 100644 --- a/src/workerd/jsg/modules.c++ +++ b/src/workerd/jsg/modules.c++ @@ -6,6 +6,7 @@ #include "setup.h" #include "util.h" +#include #include #include @@ -400,7 +401,7 @@ v8::Local createSyntheticModule( } } return v8::Module::CreateSyntheticModule(js.v8Isolate, v8StrIntern(js.v8Isolate, name), - v8::MemorySpan>(exportNames.data(), exportNames.size()), + std::span>(exportNames.data(), exportNames.size()), &evaluateSyntheticModuleCallback); } } // namespace @@ -443,7 +444,7 @@ v8::Local compileWasmModule( auto compilationObserver = observer.onWasmCompilationStart(js.v8Isolate, code.size()); return jsg::check(v8::WasmModuleObject::Compile( - js.v8Isolate, v8::MemorySpan(code.begin(), code.size()))); + js.v8Isolate, std::span(code.begin(), code.size()))); } // ====================================================================================== diff --git a/src/workerd/jsg/wrappable.h b/src/workerd/jsg/wrappable.h index 849f596b554..607b9e5cb67 100644 --- a/src/workerd/jsg/wrappable.h +++ b/src/workerd/jsg/wrappable.h @@ -155,7 +155,10 @@ class Wrappable: public kj::Refcounted { INTERNAL_FIELD_COUNT, }; - static constexpr v8::CppHeapPointerTag WRAPPABLE_TAG = v8::CppHeapPointerTag::kDefaultTag; + // kFirstObjectWrappableTag is the first embedder-assignable wrappable tag. It is valid in both + // sandbox configurations (workerd uses a single tag consistently for all of its objects). + static constexpr v8::CppHeapPointerTag WRAPPABLE_TAG = + v8::CppHeapPointerTag::kFirstObjectWrappableTag; // The value pointed to by the internal field field `WRAPPABLE_TAG_FIELD_INDEX`. // From 3223f34f42c74f1bdb297d6a2421e3bda13912d5 Mon Sep 17 00:00:00 2001 From: Felix Hanau Date: Mon, 13 Jul 2026 23:47:36 -0400 Subject: [PATCH 081/101] [o11y] Omit fetch headers when stringifying FetchEventInfo outside of tests These are not relevant when trying to understand missing exception logs/stacks --- src/workerd/io/trace.c++ | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/workerd/io/trace.c++ b/src/workerd/io/trace.c++ index 5fdb7e7496c..b44f1a5fc6b 100644 --- a/src/workerd/io/trace.c++ +++ b/src/workerd/io/trace.c++ @@ -436,9 +436,15 @@ FetchEventInfo FetchEventInfo::clone() const { } kj::String FetchEventInfo::toString() const { - return kj::str("FetchEventInfo: ", - kj::delimited( - kj::arr(kj::str(method), kj::str(url), kj::str(cfJson), kj::str(headers)), ", "_kjc)); + // Only stringify headers in predictable mode, these should not be logged in prod + if (isPredictableModeForTest()) { + return kj::str("FetchEventInfo: ", + kj::delimited( + kj::arr(kj::str(method), kj::str(url), kj::str(cfJson), kj::str(headers)), ", "_kjc)); + } else { + return kj::str("FetchEventInfo: ", + kj::delimited(kj::arr(kj::str(method), kj::str(url), kj::str(cfJson)), ", "_kjc)); + } } FetchEventInfo::Header::Header(kj::String name, kj::String value) From 6d285c26981aa6f8342778202f04e97482eac7d5 Mon Sep 17 00:00:00 2001 From: Felix Hanau Date: Mon, 13 Jul 2026 16:58:21 -0400 Subject: [PATCH 082/101] [build] Update to Bazel 9.2.0, build maintenance - Globally disable "diabolic" python behavior so we don't have to do it for each target - Disable rules_cc toolchain on macOS - Remove obsolete cc_compatibility_proxy import - Fix warning about ada-url not being in rust target build_deps --- .bazelrc | 8 ++++++++ .bazelversion | 2 +- MODULE.bazel | 5 +---- src/pyodide/BUILD.bazel | 3 --- src/rust/api/BUILD.bazel | 12 +++++++----- 5 files changed, 17 insertions(+), 13 deletions(-) diff --git a/.bazelrc b/.bazelrc index 765fb0c477e..205e3acae1a 100644 --- a/.bazelrc +++ b/.bazelrc @@ -54,6 +54,9 @@ build:unix --features=external_include_paths --host_features=external_include_pa # Disable deprecated cfg = "host" Bazel rule setting. Blocked on perfetto. # common --incompatible_disable_starlark_host_transitions +# Globally disable the "diabolic behavior" of implicit __init__.py creation (https://github.com/bazel-contrib/rules_python/issues/2945) +common --incompatible_default_to_explicit_init_py + # Our dependencies (ICU, zlib, etc.) produce a lot of these warnings, so we disable them. build --per_file_copt='external@-Wno-error' build --per_file_copt='external@-Wno-suggest-override' @@ -282,6 +285,11 @@ build:unix --cxxopt='-Wsuggest-override' build:linux --config=unix build:macos --config=unix +# Disable the deprecated rules_cc macOS toolchain – no need to configure this since we always use +# the apple_support one, this will be removed in the future. See https://github.com/bazelbuild/rules_cc/issues/754 +# for context. +build:macos --repo_env=BAZEL_USE_LEGACY_MACOS_TOOLCHAIN=0 + # Support macOS 13 as the minimum version. There should be at least a warning when backward # compatibility is broken as -Wunguarded-availability-new is enabled by default. Only enable for # target configuration as host configuration tools are only used during the build process. diff --git a/.bazelversion b/.bazelversion index 44931da2660..deeb3d66ef0 100644 --- a/.bazelversion +++ b/.bazelversion @@ -1 +1 @@ -9.1.1 +9.2.0 diff --git a/MODULE.bazel b/MODULE.bazel index 4b750849ad6..f6860655898 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -49,9 +49,6 @@ include("//build/deps:v8.MODULE.bazel") include("//build/deps:oci.MODULE.bazel") include("//build/deps/formatters:rustfmt.MODULE.bazel") -# compatibility proxy is needed when using current rules_cc with Bazel 8 -compat = use_extension("@rules_cc//cc:extensions.bzl", "compatibility_proxy") -use_repo(compat, "cc_compatibility_proxy") - +# Make local_config_cc available, used for retrieving Windows clang-cl toolchain cc_configure = use_extension("@rules_cc//cc:extensions.bzl", "cc_configure_extension") use_repo(cc_configure, "local_config_cc") diff --git a/src/pyodide/BUILD.bazel b/src/pyodide/BUILD.bazel index 8847f737936..4f0fa134c20 100644 --- a/src/pyodide/BUILD.bazel +++ b/src/pyodide/BUILD.bazel @@ -16,7 +16,6 @@ wd_cc_capnp_library( py_binary( name = "pack_python_packages", srcs = ["pack_python_packages.py"], - legacy_create_init = False, visibility = ["//visibility:public"], ) @@ -82,7 +81,6 @@ py_test( "internal/introspection.py", "internal/test_introspection.py", ] + glob(["internal/workers-api/src/workers/*"]), - legacy_create_init = False, main = "internal/test_introspection.py", ) @@ -90,7 +88,6 @@ py_test( name = "unit-test-frozen-sdk", srcs = ["internal/test_frozen_sdk.py"], data = glob(["internal/workers-api/**"]), - legacy_create_init = False, main = "internal/test_frozen_sdk.py", target_compatible_with = ["@platforms//os:linux"], ) diff --git a/src/rust/api/BUILD.bazel b/src/rust/api/BUILD.bazel index 7a4826ce23a..ab0c927b011 100644 --- a/src/rust/api/BUILD.bazel +++ b/src/rust/api/BUILD.bazel @@ -4,17 +4,19 @@ wd_rust_crate( name = "api", cxx_bridge_deps = ["//src/rust/jsg"], cxx_bridge_srcs = ["lib.rs"], + link_deps = [ + # ada-url is built with its `bundled` feature disabled (see + # //deps/rust:Cargo.toml), so it does not compile its own copy of the + # C++ ada library. Provide the symbols via workerd's existing C++ + # @ada-url instead, avoiding a duplicate copy of ada in the binary. + "@ada-url", + ], proc_macro_deps = [ "//src/rust/jsg-macros", ], test_deps = ["//src/rust/jsg-test"], visibility = ["//visibility:public"], deps = [ - # ada-url is built with its `bundled` feature disabled (see - # //deps/rust:Cargo.toml), so it does not compile its own copy of the - # C++ ada library. Provide the symbols via workerd's existing C++ - # @ada-url instead, avoiding a duplicate copy of ada in the binary. - "@ada-url", "//src/rust/jsg", "@crates_vendor//:ada-url", "@crates_vendor//:thiserror", From 6ec059d71de914c5dd1c21393e76846726a1fb63 Mon Sep 17 00:00:00 2001 From: Nicholas Paun Date: Mon, 13 Jul 2026 17:02:05 +0000 Subject: [PATCH 083/101] Link wd_rust_binary against our rust_runtime target --- build/wd_rust_binary.bzl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build/wd_rust_binary.bzl b/build/wd_rust_binary.bzl index 395fe6970bd..7cf8bdb3667 100644 --- a/build/wd_rust_binary.bzl +++ b/build/wd_rust_binary.bzl @@ -55,7 +55,7 @@ def wd_rust_binary( deps = deps, # wd_rust_binary is not used for the workerd production binary so far – apply default # optimization instead of linkopts_tool - link_deps = link_deps + ["//build/deps:linkopts_default"], + link_deps = link_deps + ["//build/deps:linkopts_default", "@@//deps:rust_runtime"], visibility = visibility, data = data, experimental_use_cc_common_link = 1, @@ -81,7 +81,7 @@ def wd_rust_binary( "//conditions:default": [], }), experimental_use_cc_common_link = 1, - link_deps = ["//build/deps:linkopts_default"], + link_deps = ["//build/deps:linkopts_default", "@@//deps:rust_runtime"], size = test_size, tags = ["no-coverage"], ) From 4241b8a04285fedb4c4c6e4e5f756320bd5823a9 Mon Sep 17 00:00:00 2001 From: Felix Hanau Date: Sun, 21 Jun 2026 22:21:25 -0400 Subject: [PATCH 084/101] [o11y] Additional hardening for outcome reporting - Don't report plain exception outcome before checking exception details first. This may fix some cases of exception outcomes being reported when there should be a different outcome. - Make sure to reportFailure() so that exception stack trace/description get captured in internal trace - Add missing return tail events for restore events --- src/workerd/api/global-scope.c++ | 1 + src/workerd/api/hibernatable-web-socket.c++ | 1 + src/workerd/api/queue.c++ | 10 ++++++++-- src/workerd/api/restore.c++ | 9 +++++++++ 4 files changed, 19 insertions(+), 2 deletions(-) diff --git a/src/workerd/api/global-scope.c++ b/src/workerd/api/global-scope.c++ index 98918666546..7ba5d54aead 100644 --- a/src/workerd/api/global-scope.c++ +++ b/src/workerd/api/global-scope.c++ @@ -731,6 +731,7 @@ kj::Promise ServiceWorkerGlobalScope::runAlarm(kj: actorId = kj::str(s); } } + context.getMetrics().reportFailure(e); auto isUserGeneratedError = isAlarmFailureUserError( e.getDescription(), e.getDetail(jsg::EXCEPTION_IS_USER_ERROR) != kj::none); auto shouldRetryCountsAgainstLimits = alarmRetryCountsAgainstLimit({ diff --git a/src/workerd/api/hibernatable-web-socket.c++ b/src/workerd/api/hibernatable-web-socket.c++ index 0749c816e5b..aafe85fdf1d 100644 --- a/src/workerd/api/hibernatable-web-socket.c++ +++ b/src/workerd/api/hibernatable-web-socket.c++ @@ -121,6 +121,7 @@ kj::Promise HibernatableWebSocketCustomEve !jsg::isTunneledException(desc) && !jsg::isDoNotLogException(desc)) { LOG_EXCEPTION("HibernatableWebSocketCustomEvent"_kj, e); } + incomingRequest->getMetrics().reportFailure(e); outcome = EventOutcome::EXCEPTION; } diff --git a/src/workerd/api/queue.c++ b/src/workerd/api/queue.c++ index f56b10b740a..df82c3bd09b 100644 --- a/src/workerd/api/queue.c++ +++ b/src/workerd/api/queue.c++ @@ -728,8 +728,9 @@ kj::Promise QueueCustomEvent::run( // all waitUntil'ed promises. auto outcome = co_await runProm .then([]() mutable -> kj::Promise { return EventOutcome::OK; }) - .catch_([](kj::Exception&& e) { + .catch_([weakIoctx = context.getWeakRef()](kj::Exception&& e) { // If any exceptions were thrown, mark the outcome accordingly. + weakIoctx->runIfAlive([&e](IoContext& context) { context.getMetrics().reportFailure(e); }); return EventOutcome::EXCEPTION; }) .exclusiveJoin(timeoutPromise.then([] { @@ -739,8 +740,13 @@ kj::Promise QueueCustomEvent::run( // Also handle anything that might cause the worker to get aborted. // This is a change from the outcome we returned on abort before the compat flag, but better // matches the behavior of fetch() handlers and the semantics of what's actually happening. + // abortFulfiller should only ever be rejected instead of being fulfilled, return an + // internalError outcome if it does happen + return EventOutcome::INTERNAL_ERROR; + }, [weakIoctx = context.getWeakRef()](kj::Exception&& e) { + weakIoctx->runIfAlive([&e](IoContext& context) { context.getMetrics().reportFailure(e); }); return EventOutcome::EXCEPTION; - }, [](kj::Exception&&) { return EventOutcome::EXCEPTION; })); + })); if (outcome == EventOutcome::OK && queueEventHolder->isServiceWorkerHandler) { // HACK: For service-worker syntax, we effectively ignore the compatibility flag and wait diff --git a/src/workerd/api/restore.c++ b/src/workerd/api/restore.c++ index aead746973d..6734a0c47d5 100644 --- a/src/workerd/api/restore.c++ +++ b/src/workerd/api/restore.c++ @@ -8,6 +8,7 @@ #include "http.h" #include "worker-rpc.h" +#include #include namespace workerd::api { @@ -321,10 +322,14 @@ kj::Promise RestoreServiceCustomEvent::run // Keep the restore event's IoContext alive as long as the restored service channel exists. co_await donePromise.exclusiveJoin(ioctx.onAbort()); + KJ_IF_SOME(t, ioctx.getWorkerTracer()) { + t.setReturn(ioctx.now()); + } co_return WorkerInterface::CustomEvent::Result{.outcome = EventOutcome::OK}; } KJ_CATCH(exception) { + incomingRequest->getMetrics().reportFailure(exception); channelFulfiller->reject(kj::mv(exception)); co_return WorkerInterface::CustomEvent::Result{.outcome = EventOutcome::EXCEPTION}; @@ -440,10 +445,14 @@ kj::Promise RestoreRpcStubCustomEvent::run // `donePromise` resolves once there are no longer any capabilities pointing between the client // and server as part of this session. co_await donePromise.exclusiveJoin(ioctx.onAbort()); + KJ_IF_SOME(t, ioctx.getWorkerTracer()) { + t.setReturn(ioctx.now()); + } co_return WorkerInterface::CustomEvent::Result{.outcome = EventOutcome::OK}; } KJ_CATCH(exception) { + incomingRequest->getMetrics().reportFailure(exception); capFulfiller->reject(kj::mv(exception)); co_return WorkerInterface::CustomEvent::Result{.outcome = EventOutcome::EXCEPTION}; From d3c25951516cb1a44215f06d4d66375867ea1117 Mon Sep 17 00:00:00 2001 From: James M Snell Date: Tue, 14 Jul 2026 09:02:33 -0700 Subject: [PATCH 085/101] Enable nodejs_compat by default. It's time. --- src/workerd/io/compatibility-date.capnp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/workerd/io/compatibility-date.capnp b/src/workerd/io/compatibility-date.capnp index de851318b80..54ad872652a 100644 --- a/src/workerd/io/compatibility-date.capnp +++ b/src/workerd/io/compatibility-date.capnp @@ -240,7 +240,8 @@ struct CompatibilityFlags @0x8f8c1b68151b6cef { nodeJsCompat @21 :Bool $compatEnableFlag("nodejs_compat") - $compatDisableFlag("no_nodejs_compat"); + $compatDisableFlag("no_nodejs_compat") + $compatEnableDate("2026-08-04"); # Enables nodejs compat imports in the application. obsolete22 @22 :Bool From 2ca28431853fbd435fe26ae8282f61c3b45ab540 Mon Sep 17 00:00:00 2001 From: James M Snell Date: Tue, 14 Jul 2026 09:42:32 -0700 Subject: [PATCH 086/101] Fixup tests with nodejs_compat as default --- src/workerd/api/node/tests/crypto_cipher-test.wd-test | 2 +- src/workerd/api/node/tests/node-compat-v2-test.js | 5 ----- src/workerd/api/tests/als-only-test.wd-test | 2 +- src/workerd/io/compatibility-date.capnp | 3 ++- src/workerd/tests/performance-test.js | 9 --------- 5 files changed, 4 insertions(+), 17 deletions(-) diff --git a/src/workerd/api/node/tests/crypto_cipher-test.wd-test b/src/workerd/api/node/tests/crypto_cipher-test.wd-test index b94715bdafb..f2c159ff165 100644 --- a/src/workerd/api/node/tests/crypto_cipher-test.wd-test +++ b/src/workerd/api/node/tests/crypto_cipher-test.wd-test @@ -7,7 +7,7 @@ const unitTests :Workerd.Config = ( modules = [ (name = "worker", esModule = embed "crypto_cipher-test.js") ], - compatibilityFlags = ["nodejs_compat_v2", "experimental"], + compatibilityFlags = ["nodejs_compat_v2", "add_nodejs_compat_eol"], bindings = [ ( name = "rsa_private.pem", text = embed "fixtures/rsa_private.pem" ), ( name = "rsa_public.pem", text = embed "fixtures/rsa_public.pem" ), diff --git a/src/workerd/api/node/tests/node-compat-v2-test.js b/src/workerd/api/node/tests/node-compat-v2-test.js index ab675e1e3d8..c433a8c1828 100644 --- a/src/workerd/api/node/tests/node-compat-v2-test.js +++ b/src/workerd/api/node/tests/node-compat-v2-test.js @@ -39,11 +39,6 @@ export const nodeJsGetBuiltins = { // But process.getBuiltinModule should always return the built-in module. const builtInPath = process.getBuiltinModule('node:path'); - const builtInVm = process.getBuiltinModule('node:vm'); - - // The built-in node:vm module is not available in workers, so this should - // return undefined. - assert.strictEqual(builtInVm, undefined); // These are from the worker bundle.... assert.strictEqual(fs, 1); diff --git a/src/workerd/api/tests/als-only-test.wd-test b/src/workerd/api/tests/als-only-test.wd-test index d90bf7d7dcb..89cd00e982b 100644 --- a/src/workerd/api/tests/als-only-test.wd-test +++ b/src/workerd/api/tests/als-only-test.wd-test @@ -7,7 +7,7 @@ const unitTests :Workerd.Config = ( modules = [ (name = "worker", esModule = embed "als-only-test.js") ], - compatibilityFlags = ["nodejs_als"] + compatibilityFlags = ["nodejs_als", "no_nodejs_compat", "no_nodejs_compat_v2"] ) ), ], diff --git a/src/workerd/io/compatibility-date.capnp b/src/workerd/io/compatibility-date.capnp index 54ad872652a..794998cdd37 100644 --- a/src/workerd/io/compatibility-date.capnp +++ b/src/workerd/io/compatibility-date.capnp @@ -466,7 +466,8 @@ struct CompatibilityFlags @0x8f8c1b68151b6cef { nodeJsCompatV2 @50 :Bool $compatEnableFlag("nodejs_compat_v2") $compatDisableFlag("no_nodejs_compat_v2") - $impliedByAfterDate(name = "nodeJsCompat", date = "2024-09-23"); + $impliedByAfterDate(name = "nodeJsCompat", date = "2024-09-23") + $compatEnableDate("2026-08-04"); # Implies nodeJSCompat with the following additional modifications: # * Node.js Compat built-ins may be imported/required with or without the node: prefix # * Node.js Compat the globals Buffer and process are available everywhere diff --git a/src/workerd/tests/performance-test.js b/src/workerd/tests/performance-test.js index 6214e9c99e6..8893a6d6d25 100644 --- a/src/workerd/tests/performance-test.js +++ b/src/workerd/tests/performance-test.js @@ -14,15 +14,6 @@ if (globalThis.performance.now() !== 0.0) { throw new Error('performance.now() is not 0.0'); } -if ('addEventListener' in globalThis.performance) { - throw new Error('performance.addEventListener should not be defined'); -} - -// Performance class should not be available. -if (typeof globalThis.Performance !== 'undefined') { - throw new Error('Performance should not be defined'); -} - export const test = { async test(_ctrl, _env, _ctx) { const start = performance.now(); From e960fba17b5838d9f3fe8691d3a555142293fee3 Mon Sep 17 00:00:00 2001 From: Mike Aizatsky Date: Tue, 14 Jul 2026 10:24:51 -0700 Subject: [PATCH 087/101] iocontext run tidy --- .clang-tidy | 43 +++++++++++- BUILD.bazel | 1 + src/workerd/api/capnp.c++ | 4 +- src/workerd/api/worker-rpc.c++ | 2 +- src/workerd/io/io-context.c++ | 2 +- src/workerd/io/io-context.h | 4 +- src/workerd/io/worker.c++ | 2 +- src/workerd/tests/test-fixture.h | 10 ++- tools/clang-tidy/BUILD.bazel | 24 +++++++ ...ntext-run-manual-capture-negative-test.c++ | 53 +++++++++++++++ ...ntext-run-manual-capture-positive-test.c++ | 39 +++++++++++ .../iocontext-run-manual-capture-test.sh | 66 +++++++++++++++++++ 12 files changed, 236 insertions(+), 14 deletions(-) create mode 100644 tools/clang-tidy/iocontext-run-manual-capture-negative-test.c++ create mode 100644 tools/clang-tidy/iocontext-run-manual-capture-positive-test.c++ create mode 100755 tools/clang-tidy/iocontext-run-manual-capture-test.sh diff --git a/.clang-tidy b/.clang-tidy index 0af40047339..2db291cd203 100644 --- a/.clang-tidy +++ b/.clang-tidy @@ -67,7 +67,8 @@ Checks: > readability-use-concise-preprocessor-directives, jsg-visit-for-gc, workerd-consume, - workerd-unsafe-continuation-capture + workerd-unsafe-continuation-capture, + custom-iocontext-run-manual-capture # TODO: Fix and enable # bugprone-derived-method-shadowing-base-method @@ -93,3 +94,43 @@ CheckOptions: value: "jsg/jsg.h|jsg/dom-exception.h" - key: cppcoreguidelines-missing-std-forward.ForwardFunction value: "kj::fwd" + +#### +# Custom checks. +# +# Query-based custom checks (see capnp-cpp's `.clang-tidy` for more examples). +# These require clang-tidy's `--experimental-custom-checks` flag, which is passed +# by `build/tools/clang_tidy/clang_tidy.bzl`. +# +# References: +# - https://clang.llvm.org/extra/clang-tidy/QueryBasedCustomChecks.html +# - https://clang.llvm.org/docs/LibASTMatchersReference.html +CustomChecks: + - Name: iocontext-run-manual-capture + # `IoContext::run()` (and friends) accept a callback of either + # `(Worker::Lock&)` or `(Worker::Lock&, IoContext&)`. When the callback needs + # the IoContext, the two-argument form should be used so that the IoContext is + # passed in as a parameter, rather than capturing it manually in the lambda + # capture list (which is easy to get wrong with respect to lifetimes). + # + # This matches a call to `IoContext::run(...)` whose first argument is a lambda + # that captures a variable whose type is `IoContext` (by value, reference, or + # pointer), e.g. `ctx.run([&ctx](Worker::Lock& lock) { ... })`. + Query: | + match cxxMemberCallExpr( + callee(cxxMethodDecl( + hasName("run"), + ofClass(cxxRecordDecl(hasName("::workerd::IoContext"))) + )), + hasArgument(0, ignoringParenImpCasts(lambdaExpr( + hasAnyCapture(lambdaCapture(capturesVar(varDecl(hasType(qualType(anyOf( + hasDeclaration(cxxRecordDecl(hasName("::workerd::IoContext"))), + references(cxxRecordDecl(hasName("::workerd::IoContext"))), + pointsTo(cxxRecordDecl(hasName("::workerd::IoContext"))) + ))))))) + ).bind("lambda"))) + ).bind("call") + Diagnostic: + - BindName: lambda + Message: "IoContext::run() lambda captures the IoContext manually; use the two-argument (Worker::Lock&, IoContext&) lambda form instead" + Level: Warning diff --git a/BUILD.bazel b/BUILD.bazel index 0fdbedfc240..6acb6a77353 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -43,6 +43,7 @@ genrule( outs = ["merged.clang-tidy"], cmd = "$(location //build/tools/clang_tidy:merge_clang_tidy_configs) $@ $(location :.clang-tidy) $(location @capnp-cpp//:.clang-tidy)", tools = ["//build/tools/clang_tidy:merge_clang_tidy_configs"], + visibility = ["//visibility:public"], ) # Plugin to generate .js files diff --git a/src/workerd/api/capnp.c++ b/src/workerd/api/capnp.c++ index 75107bb4c4b..ad2197a092f 100644 --- a/src/workerd/api/capnp.c++ +++ b/src/workerd/api/capnp.c++ @@ -568,8 +568,8 @@ kj::Promise CapnpServer::call(capnp::InterfaceSchema::Method method, kj::Promise result = nullptr; bool live = ioContext->runIfAlive([&](IoContext& rc) { - result = - rc.run([this, method, rpcContext, &rc](Worker::Lock& lock) mutable -> kj::Promise { + result = rc.run( + [this, method, rpcContext](Worker::Lock& lock, IoContext& rc) mutable -> kj::Promise { jsg::Lock& js = lock; auto handle = object.getHandle(js); auto methodName = method.getProto().getName(); diff --git a/src/workerd/api/worker-rpc.c++ b/src/workerd/api/worker-rpc.c++ index 1f3f7082f2d..151d9e9aa76 100644 --- a/src/workerd/api/worker-rpc.c++ +++ b/src/workerd/api/worker-rpc.c++ @@ -1023,7 +1023,7 @@ class JsRpcTargetBase: public rpc::JsRpcTarget::Server { enterIsolateAndCall([this, &ctx](CallContext callContext) { // Note: No need to topUpActor() since this is the start of a top-level request, so the // actor will already have been topped up by IncomingRequest::delivered(). - return ctx.run([this, &ctx, callContext](Worker::Lock& lock) mutable { + return ctx.run([this, callContext](Worker::Lock& lock, IoContext& ctx) mutable { return callImpl(lock, ctx, callContext); }); }), diff --git a/src/workerd/io/io-context.c++ b/src/workerd/io/io-context.c++ index 2cec0b1105f..83082ba387c 100644 --- a/src/workerd/io/io-context.c++ +++ b/src/workerd/io/io-context.c++ @@ -829,7 +829,7 @@ void IoContext::TimeoutManagerImpl::setTimeoutImpl(IoContext& context, Iterator // the timer, so we don't want to addTask() it, which awaitIo() does implicitly. auto promise = paf.promise.then([this, &context, it, cs = context.getCriticalSection()]() mutable { - return context.run([this, &context, it](Worker::Lock& lock) mutable { + return context.run([this, it](Worker::Lock& lock, IoContext& context) mutable { auto& state = it->second; auto stateGuard = kj::defer([&] { diff --git a/src/workerd/io/io-context.h b/src/workerd/io/io-context.h index d6dd5961981..c2c9c722286 100644 --- a/src/workerd/io/io-context.h +++ b/src/workerd/io/io-context.h @@ -1659,8 +1659,8 @@ auto IoContext::makeReentryCallbackImpl(Func func, kj::Own attachment) { } return ctx.canceler.wrap(ctx.run( - [&ctx, &ioFunc, ... params = kj::fwd(params)]( - Worker::Lock& lock) mutable { + [&ioFunc, ... params = kj::fwd(params)]( + Worker::Lock& lock, IoContext& ctx) mutable { using ResultType = kj::Decay(params)...))>; auto& func = *ioFunc; diff --git a/src/workerd/io/worker.c++ b/src/workerd/io/worker.c++ index 4d5d0d2d60b..b9d779d22a1 100644 --- a/src/workerd/io/worker.c++ +++ b/src/workerd/io/worker.c++ @@ -3992,7 +3992,7 @@ kj::Promise Worker::Actor::ensureConstructedImpl(IoContext& context, Actor containerRunning = status.getRunning(); } - co_await context.run([this, &context, &info, containerRunning](Worker::Lock& lock) { + co_await context.run([this, &info, containerRunning](Worker::Lock& lock, IoContext& context) { jsg::Lock& js = lock; kj::Maybe> storage; diff --git a/src/workerd/tests/test-fixture.h b/src/workerd/tests/test-fixture.h index cfd5e120dd5..aaa0ddbc9fe 100644 --- a/src/workerd/tests/test-fixture.h +++ b/src/workerd/tests/test-fixture.h @@ -90,9 +90,8 @@ struct TestFixture { waitScope = &KJ_REQUIRE_NONNULL(io).waitScope; } - auto& context = request->getContext(); - return context - .run([&](Worker::Lock& lock) { + return request->getContext() + .run([&](Worker::Lock& lock, IoContext& context) { // auto features = workerBundle.getFeatureFlags(); auto& js = jsg::Lock::from(lock.getIsolate()); Environment env = {{.isolate = lock.getIsolate()}, context, lock, js}; @@ -135,9 +134,8 @@ struct TestFixture { // IncomingRequest. template void enterContext(IoContext::IncomingRequest& request, Callback&& callback) { - auto& context = request.getContext(); - context - .run([&](Worker::Lock& lock) { + request.getContext() + .run([&](Worker::Lock& lock, IoContext& context) { auto& js = jsg::Lock::from(lock.getIsolate()); Environment env = {{.isolate = lock.getIsolate()}, context, lock, js}; callback(env); diff --git a/tools/clang-tidy/BUILD.bazel b/tools/clang-tidy/BUILD.bazel index f28371941f7..37b18bce51c 100644 --- a/tools/clang-tidy/BUILD.bazel +++ b/tools/clang-tidy/BUILD.bazel @@ -32,6 +32,9 @@ exports_files([ "consume-negative-test.c++", "consume-positive-test.c++", "consume-test.sh", + "iocontext-run-manual-capture-negative-test.c++", + "iocontext-run-manual-capture-positive-test.c++", + "iocontext-run-manual-capture-test.sh", "workerd-lint.c++", "visit-for-gc.c++", "visit-for-gc.h", @@ -121,3 +124,24 @@ sh_test( "//conditions:default": ["@platforms//:incompatible"], }), ) + +# Tests the `custom-iocontext-run-manual-capture` query-based check, which is +# defined in the workerd `.clang-tidy` config rather than in the plugin. The +# test drives clang-tidy against the real merged config, so it does not need the +# plugin loaded. +sh_test( + name = "iocontext-run-manual-capture-test", + srcs = ["iocontext-run-manual-capture-test.sh"], + data = [ + ":iocontext-run-manual-capture-negative-test.c++", + ":iocontext-run-manual-capture-positive-test.c++", + "//:merged_clang_tidy_config", + "//tools:clang-tidy", + ], + tags = ["no-asan"], + target_compatible_with = select({ + "@platforms//os:linux": [], + "@platforms//os:macos": [], + "//conditions:default": ["@platforms//:incompatible"], + }), +) diff --git a/tools/clang-tidy/iocontext-run-manual-capture-negative-test.c++ b/tools/clang-tidy/iocontext-run-manual-capture-negative-test.c++ new file mode 100644 index 00000000000..d705dd54b1b --- /dev/null +++ b/tools/clang-tidy/iocontext-run-manual-capture-negative-test.c++ @@ -0,0 +1,53 @@ +// Copyright (c) 2017-2026 Cloudflare, Inc. +// Licensed under the Apache 2.0 license found in the LICENSE file or at: +// https://opensource.org/licenses/Apache-2.0 + +// Negative cases for the `custom-iocontext-run-manual-capture` query-based +// clang-tidy check defined in the workerd `.clang-tidy` config. None of the +// calls below should be flagged: they either use the two-argument lambda form +// or do not capture the IoContext at all. + +namespace workerd { + +struct Worker { + struct Lock {}; +}; + +// Minimal stand-in for the real IoContext. The check only cares about the +// fully-qualified name `::workerd::IoContext` and a method named `run`. +class IoContext { + public: + template + void run(Func&& func) {} +}; + +// A different class that also has a `run` method taking a lambda; calling it +// must not be flagged even if the lambda captures an IoContext, because the +// method does not belong to IoContext. +class NotAnIoContext { + public: + template + void run(Func&& func) {} +}; + +void twoArgumentForm(IoContext& ctx) { + // GOOD: IoContext is received as a parameter, not captured. + ctx.run([](Worker::Lock& lock, IoContext& ioContext) { (void)ioContext; }); +} + +void noCapture(IoContext& ctx) { + // GOOD: nothing captured. + ctx.run([](Worker::Lock& lock) {}); +} + +void unrelatedCapture(IoContext& ctx, int value) { + // GOOD: captures an unrelated variable, not the IoContext. + ctx.run([value](Worker::Lock& lock) { (void)value; }); +} + +void runOnOtherClass(NotAnIoContext& other, IoContext& ctx) { + // GOOD: `run` is not IoContext::run, so capturing the IoContext is fine here. + other.run([&ctx](Worker::Lock& lock) { (void)ctx; }); +} + +} // namespace workerd diff --git a/tools/clang-tidy/iocontext-run-manual-capture-positive-test.c++ b/tools/clang-tidy/iocontext-run-manual-capture-positive-test.c++ new file mode 100644 index 00000000000..b89be329452 --- /dev/null +++ b/tools/clang-tidy/iocontext-run-manual-capture-positive-test.c++ @@ -0,0 +1,39 @@ +// Copyright (c) 2017-2026 Cloudflare, Inc. +// Licensed under the Apache 2.0 license found in the LICENSE file or at: +// https://opensource.org/licenses/Apache-2.0 + +// Positive cases for the `custom-iocontext-run-manual-capture` query-based +// clang-tidy check defined in the workerd `.clang-tidy` config. Every call to +// IoContext::run() below passes a lambda that captures the IoContext manually, +// which the check should flag. + +namespace workerd { + +struct Worker { + struct Lock {}; +}; + +// Minimal stand-in for the real IoContext. The check only cares about the +// fully-qualified name `::workerd::IoContext` and a method named `run`. +class IoContext { + public: + template + void run(Func&& func) {} +}; + +void captureByReference(IoContext& ctx) { + // Captures `ctx` (an IoContext&) explicitly by reference. + ctx.run([&ctx](Worker::Lock& lock) { (void)ctx; }); +} + +void captureByPointer(IoContext* ctx) { + // Captures `ctx` (an IoContext*) explicitly by value. + ctx->run([ctx](Worker::Lock& lock) { (void)ctx; }); +} + +void captureByDefaultReference(IoContext& ctx) { + // Default-capture-by-reference implicitly captures `ctx`. + ctx.run([&](Worker::Lock& lock) { (void)ctx; }); +} + +} // namespace workerd diff --git a/tools/clang-tidy/iocontext-run-manual-capture-test.sh b/tools/clang-tidy/iocontext-run-manual-capture-test.sh new file mode 100755 index 00000000000..aebfcf84806 --- /dev/null +++ b/tools/clang-tidy/iocontext-run-manual-capture-test.sh @@ -0,0 +1,66 @@ +#!/usr/bin/env bash + +# Copyright (c) 2017-2026 Cloudflare, Inc. +# Licensed under the Apache 2.0 license found in the LICENSE file or at: +# https://opensource.org/licenses/Apache-2.0 + +# Tests the `custom-iocontext-run-manual-capture` query-based clang-tidy check. +# Unlike the plugin checks (e.g. workerd-consume), this check is defined entirely +# in the workerd `.clang-tidy` config as a CustomChecks entry, so the test drives +# clang-tidy against the real merged config with --experimental-custom-checks. + +set -euo pipefail + +readonly ROOT="${TEST_SRCDIR}/${TEST_WORKSPACE}" +readonly CLANG_TIDY="${ROOT}/tools/clang_tidy" +readonly CONFIG="${ROOT}/merged.clang-tidy" +readonly POSITIVE="${ROOT}/tools/clang-tidy/iocontext-run-manual-capture-positive-test.c++" +readonly NEGATIVE="${ROOT}/tools/clang-tidy/iocontext-run-manual-capture-negative-test.c++" +readonly CHECK="custom-iocontext-run-manual-capture" +readonly CHECKS="-*,${CHECK}" +readonly MESSAGE="lambda captures the IoContext manually" + +run_check() { + local file="$1" + "${CLANG_TIDY}" \ + --experimental-custom-checks \ + --config-file="${CONFIG}" \ + --checks="${CHECKS}" \ + --warnings-as-errors='*' \ + "${file}" -- -xc++ -std=c++23 2>&1 +} + +# Positive cases: the check must fire (non-zero exit) and emit its diagnostic. +set +e +positive_output=$(run_check "${POSITIVE}") +positive_status=$? +set -e + +if [[ ${positive_status} -eq 0 ]]; then + printf '%s\n' "Expected ${CHECK} to flag manual IoContext captures." >&2 + printf '%s\n' "${positive_output}" >&2 + exit 1 +fi + +if [[ "${positive_output}" != *"${MESSAGE}"* ]]; then + printf '%s\n' "Expected ${CHECK} diagnostic message." >&2 + printf '%s\n' "${positive_output}" >&2 + exit 1 +fi + +# Each of the three positive cases should be flagged. +positive_count=$(printf '%s\n' "${positive_output}" | grep -c "${CHECK}") +if [[ ${positive_count} -ne 3 ]]; then + printf '%s\n' "Expected 3 ${CHECK} diagnostics, got ${positive_count}." >&2 + printf '%s\n' "${positive_output}" >&2 + exit 1 +fi + +# Negative cases: the check must not fire on any of them. +negative_output=$(run_check "${NEGATIVE}") + +if [[ "${negative_output}" == *"${CHECK}"* ]]; then + printf '%s\n' "Expected ${CHECK} to accept two-arg form and unrelated captures." >&2 + printf '%s\n' "${negative_output}" >&2 + exit 1 +fi From b4f8fc9b0a1a0e7dc4d501813e538abc31e7ac02 Mon Sep 17 00:00:00 2001 From: Ibrahim Khajanchi Date: Tue, 14 Jul 2026 18:22:42 -0400 Subject: [PATCH 088/101] use more idiomatic switch with which --- src/workerd/io/trace.c++ | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/workerd/io/trace.c++ b/src/workerd/io/trace.c++ index 002e012d149..39bc3657028 100644 --- a/src/workerd/io/trace.c++ +++ b/src/workerd/io/trace.c++ @@ -855,10 +855,14 @@ Log::Log(rpc::Trace::Log::Reader reader) auto slots = kj::heapArray>(listReader.size()); for (auto i: kj::zeroTo(listReader.size())) { auto slotReader = listReader[i]; - if (slotReader.which() == rpc::Trace::ErrorInfoSlot::INFO) { - slots[i] = ErrorInfo(slotReader.getInfo()); - } - // else: stays kj::none + switch (slotReader.which()) { + case rpc::Trace::ErrorInfoSlot::INFO: + slots[i] = tracing::ErrorInfo(slotReader.getInfo()); + break; + case rpc::Trace::ErrorInfoSlot::NONE: + // slot stays kj::none + break; + } } errorInfo = kj::mv(slots); } From 0b10be77345c41b41b36d58d2e63af7603af6f71 Mon Sep 17 00:00:00 2001 From: Dan Lapid Date: Tue, 14 Jul 2026 19:12:28 +0000 Subject: [PATCH 089/101] Extra ffi utilities for headers --- src/rust/kj/ffi.h | 25 ++++++++++++ src/rust/kj/http.rs | 18 ++++++++ src/rust/kj/tests/ffi-test.c++ | 75 ++++++++++++++++++++++++++++++++++ src/rust/kj/tests/lib.rs | 19 +++++++++ 4 files changed, 137 insertions(+) diff --git a/src/rust/kj/ffi.h b/src/rust/kj/ffi.h index 0260ebb1b37..0c64a6c6218 100644 --- a/src/rust/kj/ffi.h +++ b/src/rust/kj/ffi.h @@ -46,6 +46,10 @@ inline kj::Own clone_shallow(const HttpHeaders& headers) { return kj::heap(headers.cloneShallow()); } +inline void clear_headers(HttpHeaders& headers) { + headers.clear(); +} + inline kj::HttpHeaderId toHeaderId(BuiltinIndicesEnum id) { switch (id) { case kj::HttpHeaders::BuiltinIndicesEnum::CONNECTION: @@ -102,6 +106,27 @@ inline kj::Maybe<::rust::Slice> get_header_by_id( return header.map([](auto header) { return header.asBytes().template as(); }); } +// Case-insensitive lookup of a header by name. Note that kj::HttpHeaders::forEach only visits +// headers that have been explicitly set, so a header registered in the HttpHeaderTable but never +// set will not be found (returns none, matching get_header/get_header_by_id). +inline kj::Maybe<::rust::Slice> get_header_by_name( + const HttpHeaders& headers, ::rust::Str requestedName) { + auto requested = kj::ArrayPtr(requestedName.data(), requestedName.size()); + auto lower = [](char c) { return c >= 'A' && c <= 'Z' ? c + ('a' - 'A') : c; }; + kj::Maybe result; + headers.forEach([&](kj::StringPtr name, kj::StringPtr value) { + // kj::HttpHeaders::get() (used by get_header/get_header_by_id) returns the first set value, so + // keep the first match here too and ignore any subsequent duplicate header names. + if (result != kj::none) return; + if (name.size() != requested.size()) return; + for (size_t i = 0; i < name.size(); ++i) { + if (lower(name[i]) != lower(requested[i])) return; + } + result = value; + }); + return result.map([](auto header) { return header.asBytes().template as(); }); +} + // --- kj::HttpService ffi using AsyncInputStream = kj::AsyncInputStream; using AsyncIoStream = kj::AsyncIoStream; diff --git a/src/rust/kj/http.rs b/src/rust/kj/http.rs index 3c4764d1868..b62ad89e448 100644 --- a/src/rust/kj/http.rs +++ b/src/rust/kj/http.rs @@ -98,6 +98,7 @@ pub mod ffi { type HttpHeaders; fn new_http_headers(table: &HttpHeaderTable) -> KjOwn; fn clone_shallow(this_: &HttpHeaders) -> KjOwn; + fn clear_headers(this_: Pin<&mut HttpHeaders>); fn set_header(this_: Pin<&mut HttpHeaders>, id: BuiltinIndicesEnum, value: &str); unsafe fn get_header<'a>( this_: &'a HttpHeaders, @@ -107,6 +108,7 @@ pub mod ffi { this_: &'a HttpHeaders, id: &HttpHeaderId, ) -> KjMaybe<&'a [u8]>; + unsafe fn get_header_by_name<'a>(this_: &'a HttpHeaders, name: &str) -> KjMaybe<&'a [u8]>; } // --- kj::HttpService ffi @@ -272,6 +274,16 @@ impl HeadersRef<'_> { } impl<'a> HeadersRef<'a> { + /// Look up a header value by name, matching case-insensitively. + /// + /// If the same header name appears more than once, the first set value is returned (matching + /// [`get`](Self::get) / [`get_by_id`](Self::get_by_id)). Only headers that have been explicitly + /// set are considered; a header registered in the header table but never set returns `None`. + pub fn get_by_name(&self, name: &str) -> Option<&'a [u8]> { + // SAFETY: the returned value borrows header storage owned for `'a`. + unsafe { ffi::get_header_by_name(self.0, name).into() } + } + /// The underlying `kj::HttpHeaders`, for passing to FFI functions that take a `const&`. #[must_use] pub fn as_ffi(self) -> &'a ffi::HttpHeaders { @@ -307,6 +319,12 @@ impl<'a> Headers<'a> { ffi::set_header(self.own.as_mut(), id, value); } + /// Remove all headers, leaving the collection empty. This is a destructive operation; after + /// calling it, [`get`](HeadersRef::get) for any previously-set header returns `None`. + pub fn clear(&mut self) { + ffi::clear_headers(self.own.as_mut()); + } + pub fn as_ref(&'a self) -> HeadersRef<'a> { HeadersRef(self.own.as_ref()) } diff --git a/src/rust/kj/tests/ffi-test.c++ b/src/rust/kj/tests/ffi-test.c++ index 4c69c34356e..e4218071c45 100644 --- a/src/rust/kj/tests/ffi-test.c++ +++ b/src/rust/kj/tests/ffi-test.c++ @@ -101,6 +101,81 @@ KJ_TEST("get header by id round-trip through Rust") { } } +KJ_TEST("get header by name round-trip through Rust") { + kj::HttpHeaderTable::Builder builder; + auto customId = builder.add("X-Custom-Header"); + auto table = builder.build(); + + kj::HttpHeaders headers(*table); + headers.setPtr(customId, "hello-from-cpp"); + headers.setPtr(kj::HttpHeaderId::HOST, "example.com"); + + // Case-insensitive match on a custom header. + { + auto maybe = kj::rust::tests::get_header_value_via_name(headers, "x-custom-header"); + KJ_IF_SOME(value, maybe) { + auto strValue = kj::StringPtr(reinterpret_cast(value.data()), value.size()); + KJ_EXPECT(strValue == "hello-from-cpp", strValue); + } else { + KJ_FAIL_EXPECT("expected Some for custom header, got None"); + } + } + + // Case-insensitive match on a builtin header. + { + auto maybe = kj::rust::tests::get_header_value_via_name(headers, "HOST"); + KJ_IF_SOME(value, maybe) { + auto strValue = kj::StringPtr(reinterpret_cast(value.data()), value.size()); + KJ_EXPECT(strValue == "example.com", strValue); + } else { + KJ_FAIL_EXPECT("expected Some for HOST header, got None"); + } + } + + // Absent header returns None. + { + auto maybe = kj::rust::tests::get_header_value_via_name(headers, "X-Absent-Header"); + KJ_EXPECT(maybe == kj::none, "expected None for absent header"); + } +} + +KJ_TEST("get header by name returns first value for duplicate names") { + kj::HttpHeaderTable table; + kj::HttpHeaders headers(table); + + // Add the same (unindexed) header name twice; forEach visits both, and get_header_by_name must + // return the first value to match kj::HttpHeaders::get() semantics. + headers.addPtrPtr("X-Dup", "first"); + headers.addPtrPtr("X-Dup", "second"); + + auto maybe = kj::rust::tests::get_header_value_via_name(headers, "x-dup"); + KJ_IF_SOME(value, maybe) { + auto strValue = kj::StringPtr(reinterpret_cast(value.data()), value.size()); + KJ_EXPECT(strValue == "first", strValue); + } else { + KJ_FAIL_EXPECT("expected Some for duplicate header, got None"); + } +} + +KJ_TEST("clear headers round-trip through Rust") { + kj::HttpHeaderTable::Builder builder; + auto customId = builder.add("X-Custom-Header"); + auto table = builder.build(); + + kj::HttpHeaders headers(*table); + headers.setPtr(customId, "hello-from-cpp"); + headers.setPtr(kj::HttpHeaderId::HOST, "example.com"); + + // Sanity check: the header is present before clearing. + KJ_EXPECT(kj::rust::tests::get_header_value_via_id(headers, customId) != kj::none); + + kj::rust::tests::clear_headers_via_rust(headers); + + // After clearing, previously-set headers are gone. + KJ_EXPECT(kj::rust::tests::get_header_value_via_id(headers, customId) == kj::none); + KJ_EXPECT(kj::rust::tests::get_header_value_via_id(headers, kj::HttpHeaderId::HOST) == kj::none); +} + KJ_TEST("assert header ids present via ArrayPtr round-trip through Rust") { kj::HttpHeaderTable::Builder builder; auto custom1 = builder.add("X-First"); diff --git a/src/rust/kj/tests/lib.rs b/src/rust/kj/tests/lib.rs index a7b06a9c2ac..fcb48eb922a 100644 --- a/src/rust/kj/tests/lib.rs +++ b/src/rust/kj/tests/lib.rs @@ -43,6 +43,17 @@ pub mod ffi { id: &HttpHeaderId, ) -> KjMaybe<&'a [u8]>; + /// Look up a header value by name via `HeadersRef::get_by_name`, returning the value if + /// present. This exercises the C++ -> Rust -> C++ round-trip for name-based lookup. + unsafe fn get_header_value_via_name<'a>( + headers: &'a HttpHeaders, + name: &str, + ) -> KjMaybe<&'a [u8]>; + + /// Clear all headers via the `clear_headers` FFI shim. This exercises the + /// C++ -> Rust -> C++ round-trip for clearing headers. + fn clear_headers_via_rust(headers: Pin<&mut HttpHeaders>); + /// Receive an array of HttpHeaderIdpointers, convert to &[HttpHeaderIdRef] via /// from_ptr_slice, look up each header, and assert all are present. /// This exercises passing a kj::ArrayPtr into Rust. @@ -105,6 +116,14 @@ fn get_header_value_via_id<'a>( unsafe { kj::http::ffi::get_header_by_id(headers, id) } } +fn get_header_value_via_name<'a>(headers: &'a ffi::HttpHeaders, name: &str) -> KjMaybe<&'a [u8]> { + HeadersRef::from(headers).get_by_name(name).into() +} + +fn clear_headers_via_rust(headers: Pin<&mut ffi::HttpHeaders>) { + kj::http::ffi::clear_headers(headers); +} + /// # Safety /// /// Each pointer in `ids` must be non-null and point to a valid, live `HttpHeaderId`. From d1b417ab937e93691c448b0d514e81896362caed Mon Sep 17 00:00:00 2001 From: Nelson Duarte Date: Wed, 8 Jul 2026 15:17:34 +0100 Subject: [PATCH 090/101] Add item_id and key filters to AiSearch list items types --- types/defines/ai-search.d.ts | 7 +++++++ types/generated-snapshot/experimental/index.d.ts | 7 +++++++ types/generated-snapshot/experimental/index.ts | 7 +++++++ types/generated-snapshot/index.d.ts | 7 +++++++ types/generated-snapshot/index.ts | 7 +++++++ 5 files changed, 35 insertions(+) diff --git a/types/defines/ai-search.d.ts b/types/defines/ai-search.d.ts index bb54fe4de66..0416b22835f 100644 --- a/types/defines/ai-search.d.ts +++ b/types/defines/ai-search.d.ts @@ -414,6 +414,13 @@ export type AiSearchListItemsParams = { source?: string; /** JSON-encoded Vectorize filter for metadata filtering. */ metadata_filter?: string; + /** Filter items by their unique ID. Returns at most one item. */ + item_id?: string; + /** + * Filter items by their exact key (object key / filename). Keys are unique + * per source, so combine with `source` to disambiguate across data sources. + */ + key?: string; }; export type AiSearchListItemsResponse = { diff --git a/types/generated-snapshot/experimental/index.d.ts b/types/generated-snapshot/experimental/index.d.ts index 2d92b423687..fbfab1eed84 100755 --- a/types/generated-snapshot/experimental/index.d.ts +++ b/types/generated-snapshot/experimental/index.d.ts @@ -5431,6 +5431,13 @@ type AiSearchListItemsParams = { source?: string; /** JSON-encoded Vectorize filter for metadata filtering. */ metadata_filter?: string; + /** Filter items by their unique ID. Returns at most one item. */ + item_id?: string; + /** + * Filter items by their exact key (object key / filename). Keys are unique + * per source, so combine with `source` to disambiguate across data sources. + */ + key?: string; }; type AiSearchListItemsResponse = { result: AiSearchItemInfo[]; diff --git a/types/generated-snapshot/experimental/index.ts b/types/generated-snapshot/experimental/index.ts index d981c6084bc..96ff8a442ed 100755 --- a/types/generated-snapshot/experimental/index.ts +++ b/types/generated-snapshot/experimental/index.ts @@ -5442,6 +5442,13 @@ export type AiSearchListItemsParams = { source?: string; /** JSON-encoded Vectorize filter for metadata filtering. */ metadata_filter?: string; + /** Filter items by their unique ID. Returns at most one item. */ + item_id?: string; + /** + * Filter items by their exact key (object key / filename). Keys are unique + * per source, so combine with `source` to disambiguate across data sources. + */ + key?: string; }; export type AiSearchListItemsResponse = { result: AiSearchItemInfo[]; diff --git a/types/generated-snapshot/index.d.ts b/types/generated-snapshot/index.d.ts index d980661ed19..9b7e909ddef 100755 --- a/types/generated-snapshot/index.d.ts +++ b/types/generated-snapshot/index.d.ts @@ -4783,6 +4783,13 @@ type AiSearchListItemsParams = { source?: string; /** JSON-encoded Vectorize filter for metadata filtering. */ metadata_filter?: string; + /** Filter items by their unique ID. Returns at most one item. */ + item_id?: string; + /** + * Filter items by their exact key (object key / filename). Keys are unique + * per source, so combine with `source` to disambiguate across data sources. + */ + key?: string; }; type AiSearchListItemsResponse = { result: AiSearchItemInfo[]; diff --git a/types/generated-snapshot/index.ts b/types/generated-snapshot/index.ts index c3ae35db940..18d6b326c10 100755 --- a/types/generated-snapshot/index.ts +++ b/types/generated-snapshot/index.ts @@ -4794,6 +4794,13 @@ export type AiSearchListItemsParams = { source?: string; /** JSON-encoded Vectorize filter for metadata filtering. */ metadata_filter?: string; + /** Filter items by their unique ID. Returns at most one item. */ + item_id?: string; + /** + * Filter items by their exact key (object key / filename). Keys are unique + * per source, so combine with `source` to disambiguate across data sources. + */ + key?: string; }; export type AiSearchListItemsResponse = { result: AiSearchItemInfo[]; From 9627de2c7b6bc8b72bc736e42419cbdf0d2271cf Mon Sep 17 00:00:00 2001 From: Mike Aizatsky Date: Wed, 15 Jul 2026 08:28:26 -0700 Subject: [PATCH 091/101] update capnproto --- build/deps/gen/deps.MODULE.bazel | 6 +++--- src/workerd/api/streams/readable.c++ | 2 +- src/workerd/api/streams/writable.c++ | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/build/deps/gen/deps.MODULE.bazel b/build/deps/gen/deps.MODULE.bazel index 7dfd58add5d..9245b162937 100644 --- a/build/deps/gen/deps.MODULE.bazel +++ b/build/deps/gen/deps.MODULE.bazel @@ -27,10 +27,10 @@ bazel_dep(name = "brotli", version = "1.2.0.bcr.1") # capnp-cpp http.archive( name = "capnp-cpp", - sha256 = "6dae6cdfc776cadc94fe2a802186f94840329520cb53a9d02306175c06101f92", - strip_prefix = "capnproto-capnproto-6ef408c/c++", + sha256 = "fb221d3727b12fe3165b8dbbdfbf6f3d7073b353e434f1b9de9022618a3ed446", + strip_prefix = "capnproto-capnproto-bd8232b/c++", type = "tgz", - url = "https://github.com/capnproto/capnproto/tarball/6ef408c214210e3d19a1da6f695b23e782c8ffd8", + url = "https://github.com/capnproto/capnproto/tarball/bd8232ba2e09761c0901d1f64d14c83c17832001", ) use_repo(http, "capnp-cpp") diff --git a/src/workerd/api/streams/readable.c++ b/src/workerd/api/streams/readable.c++ index 9d0729a1149..2b29f1dd95b 100644 --- a/src/workerd/api/streams/readable.c++ +++ b/src/workerd/api/streams/readable.c++ @@ -390,7 +390,7 @@ ReadableStream::ReadableStream(IoContext& ioContext, kj::Own controller) : ioContext(tryGetIoContext()), controller(kj::mv(controller)) { - getController().setOwnerRef(addWeakToThis()); + getController().setOwnerRef(PtrTarget::addWeakToThis()); } void ReadableStream::visitForGc(jsg::GcVisitor& visitor) { diff --git a/src/workerd/api/streams/writable.c++ b/src/workerd/api/streams/writable.c++ index 663c2fa0209..61642d954d1 100644 --- a/src/workerd/api/streams/writable.c++ +++ b/src/workerd/api/streams/writable.c++ @@ -187,7 +187,7 @@ WritableStream::WritableStream(IoContext& ioContext, WritableStream::WritableStream(kj::Own controller) : ioContext(tryGetIoContext()), controller(kj::mv(controller)) { - getController().setOwnerRef(addWeakToThis()); + getController().setOwnerRef(PtrTarget::addWeakToThis()); } jsg::Ref WritableStream::addRef() { From 619d28ead94008f496f807740bcc6483000d7845 Mon Sep 17 00:00:00 2001 From: Mike Aizatsky Date: Mon, 13 Jul 2026 13:52:16 -0700 Subject: [PATCH 092/101] use WeakRc for weak IoContext --- docs/reference/detail/async-patterns.md | 23 +++++++++------ src/workerd/api/capnp.c++ | 20 ++++++------- src/workerd/api/capnp.h | 2 +- src/workerd/api/node/async-hooks.c++ | 30 +++++++++---------- src/workerd/api/queue.c++ | 8 ++++-- src/workerd/api/worker-loader.c++ | 6 ++-- src/workerd/io/io-context.c++ | 34 ++++++++++++---------- src/workerd/io/io-context.h | 38 ++++++------------------- src/workerd/io/io-own.c++ | 4 +-- src/workerd/io/io-own.h | 32 ++++++++++----------- src/workerd/io/trace-stream.c++ | 14 ++++----- src/workerd/io/tracer.c++ | 18 ++++++------ src/workerd/io/tracer.h | 2 +- 13 files changed, 111 insertions(+), 120 deletions(-) diff --git a/docs/reference/detail/async-patterns.md b/docs/reference/detail/async-patterns.md index 152ee40447e..07591883a7e 100644 --- a/docs/reference/detail/async-patterns.md +++ b/docs/reference/detail/async-patterns.md @@ -57,7 +57,7 @@ Use one of these patterns: | JSG resource | `[self = JSG_THIS]` | | `kj::Refcounted` | `[self = addRefToThis()]` | | IoContext, in JS-lock scope (`Worker::Lock&`/`jsg::Lock&` available) | `auto& context = IoContext::current();` inside the lambda (but see caveat below) | -| IoContext, outside JS-lock scope | `[weakRef = context.getWeakRef()]` + `KJ_ASSERT_NONNULL(weakRef->tryGet())` (alive by invariant) or `weakRef->runIfAlive(...)` (may be gone) | +| IoContext, outside JS-lock scope | `[weakRef = context.getWeakRef()]` + `KJ_ASSERT_NONNULL(weakRef)` (alive by invariant) or `KJ_IF_SOME(ctx, weakRef) { ... }` (may be gone) | | Chain feeds opaque wrapper (`oomCanceler.wrap`, `gate.lockWhile`, `.fork()`) | IILE coroutine — pass `this`/`&context` as a coroutine parameter, not a lambda capture | | Chain is `co_await`ed / `.wait()`ed / joined from a local container | already safe; no change | @@ -75,14 +75,19 @@ possibility. When it can happen, capture the originating context's `getWeakRef()` instead (or use `IoContext::addFunctor`) so the continuation either runs under the correct context or fails loudly. -**IoContext WeakRef.** `IoContext::WeakRef` (returned by `context.getWeakRef()`) -is a refcounted, non-owning handle to an `IoContext`. The WeakRef itself is -safe to hold past the IoContext's destruction; `tryGet()` returns a `Maybe` -that becomes `kj::none` once the context is gone, and `runIfAlive(func)` runs -`func` with the live `IoContext&` and returns whether it did. Use the WeakRef -pattern any time a continuation may outlive its originating context, or when -the continuation runs outside a JS-lock scope where `IoContext::current()` may -not be the right context (or may not exist at all). +**IoContext WeakRc.** `context.getWeakRef()` returns a `kj::WeakRc`, +a non-owning handle to an `IoContext`. The `WeakRc` itself is safe to hold past +the IoContext's destruction. Upgrade it with `KJ_IF_SOME(ctx, weakRef) { ... }` +(or `KJ_ASSERT_NONNULL`/`JSG_REQUIRE_NONNULL`), which binds `ctx` to a transient +strong `kj::Rc` while the block runs and yields `kj::none` once the +context is gone. Test liveness alone with `weakRef != nullptr`, or get a bare +`Maybe` via `weakRef.tryGet()`; make additional weak handles with +`weakRef.addRef()`. The upgraded `Rc` is only safe while it stays local to the +continuation — do not capture or store it, or you reintroduce the refcount cycle +warned about above. Use the WeakRc pattern any time a continuation may outlive +its originating context, or when the continuation runs outside a JS-lock scope +where `IoContext::current()` may not be the right context (or may not exist at +all). For invariants the analyzer fundamentally can't see (capnp `thisCap()`, fiber-blocked stacks, constructor-time `*this`, `CantOutliveIncomingRequest`-style diff --git a/src/workerd/api/capnp.c++ b/src/workerd/api/capnp.c++ index ad2197a092f..88fbb549f5a 100644 --- a/src/workerd/api/capnp.c++ +++ b/src/workerd/api/capnp.c++ @@ -550,16 +550,16 @@ kj::Maybe> CapnpServer::getCloseMethod(jsg::Lock& js) { CapnpServer::~CapnpServer() noexcept(false) { KJ_IF_SOME(c, closeMethod) { - ioContext->runIfAlive([&](IoContext& rc) { - rc.addTask( - rc.run([object = kj::mv(object), closeMethod = kj::mv(c)](Worker::Lock& lock) mutable { + KJ_IF_SOME(rc, ioContext) { + rc->addTask( + rc->run([object = kj::mv(object), closeMethod = kj::mv(c)](Worker::Lock& lock) mutable { auto handle = object.getHandle(lock); auto methodHandle = closeMethod.getHandle(lock); if (methodHandle->IsFunction()) { jsg::check(methodHandle.As()->Call(lock.getContext(), handle, 0, nullptr)); } })); - }); + }; } } @@ -567,9 +567,9 @@ kj::Promise CapnpServer::call(capnp::InterfaceSchema::Method method, capnp::CallContext rpcContext) { kj::Promise result = nullptr; - bool live = ioContext->runIfAlive([&](IoContext& rc) { - result = rc.run( - [this, method, rpcContext](Worker::Lock& lock, IoContext& rc) mutable -> kj::Promise { + KJ_IF_SOME(rc, ioContext) { + result = rc->run([this, method, rpcContext]( + Worker::Lock& lock, IoContext& ctx) mutable -> kj::Promise { jsg::Lock& js = lock; auto handle = object.getHandle(js); auto methodName = method.getProto().getName(); @@ -588,9 +588,9 @@ kj::Promise CapnpServer::call(capnp::InterfaceSchema::Method method, auto result = jsg::check( methodHandle.As()->Call(lock.getContext(), handle, 1, &jsParams)); KJ_IF_SOME(promise, wrapper.tryUnwrapPromise(lock, lock.getContext(), result)) { - return rc.awaitJs(js, + return ctx.awaitJs(js, promise.then( - js, rc.addFunctor([this, rpcContext](jsg::Lock& js, jsg::Value result) mutable { + js, ctx.addFunctor([this, rpcContext](jsg::Lock& js, jsg::Value result) mutable { JsCapnpConverter converter{wrapper}; converter.rpcResultsFromJs(js, rpcContext, result.getHandle(js)); }))); @@ -600,9 +600,7 @@ kj::Promise CapnpServer::call(capnp::InterfaceSchema::Method method, return kj::READY_NOW; } }); - }); - if (live) { return result; } else { return KJ_EXCEPTION(DISCONNECTED, "jsg.Error: Called to event context that is no longer live."); diff --git a/src/workerd/api/capnp.h b/src/workerd/api/capnp.h index a427c00a1c9..7b9dbeb10e3 100644 --- a/src/workerd/api/capnp.h +++ b/src/workerd/api/capnp.h @@ -33,7 +33,7 @@ class CapnpServer final: public capnp::DynamicCapability::Server { capnp::CallContext context) override; private: - kj::Own ioContext; + kj::WeakRc ioContext; jsg::V8Ref object; kj::Maybe> closeMethod; CapnpTypeWrapperBase& wrapper; // only valid if isolate is locked! diff --git a/src/workerd/api/node/async-hooks.c++ b/src/workerd/api/node/async-hooks.c++ index bc0488e8091..df9e0f814b7 100644 --- a/src/workerd/api/node/async-hooks.c++ +++ b/src/workerd/api/node/async-hooks.c++ @@ -17,27 +17,27 @@ namespace { // and check it against the current IoContext where the snapshot function // is invoked. jsg::Function getValidator(jsg::Lock& js) { - kj::Maybe> maybeIoContext; - if (FeatureFlags::get(js).getBindAsyncLocalStorageSnapshot() && IoContext::hasCurrent()) { - // We use a weak reference to the IoContext because the current IoContext - // may be destroyed before the snapshot function is called. - maybeIoContext = IoContext::current().getWeakRef(); + if (!FeatureFlags::get(js).getBindAsyncLocalStorageSnapshot() || !IoContext::hasCurrent()) { + return [](jsg::Lock&) {}; } + // We use a weak reference to the IoContext because the current IoContext + // may be destroyed before the snapshot function is called. + auto context = IoContext::current().getWeakRef(); + static constexpr auto kErrorMessage = "Cannot call this AsyncLocalStorage bound function outside of the " "request in which it was created."_kj; - return [maybeIoContext = kj::mv(maybeIoContext)](jsg::Lock&) { - KJ_IF_SOME(originIoContext, maybeIoContext) { - // We had an IoContext when we created the snapshot function. - // If it is not the current IoContext, or if there is no current - // IoContext, or if the captured IoContext has been destroyed, - // we throw an error. - JSG_REQUIRE(IoContext::hasCurrent() && originIoContext->isValid(), Error, kErrorMessage); - originIoContext->runIfAlive([&](IoContext& otherContext) { - JSG_REQUIRE(&otherContext == &IoContext::current(), Error, kErrorMessage); - }); + return [context = kj::mv(context)](jsg::Lock&) { + // We had an IoContext when we created the snapshot function. + // If it is not the current IoContext, or if there is no current + // IoContext, or if the captured IoContext has been destroyed, + // we throw an error. + KJ_IF_SOME(originIoContext, context) { + JSG_REQUIRE(originIoContext->isCurrent(), Error, kErrorMessage); + } else { + JSG_FAIL_REQUIRE(Error, kErrorMessage); } }; } diff --git a/src/workerd/api/queue.c++ b/src/workerd/api/queue.c++ index df82c3bd09b..daf12910cd9 100644 --- a/src/workerd/api/queue.c++ +++ b/src/workerd/api/queue.c++ @@ -730,7 +730,9 @@ kj::Promise QueueCustomEvent::run( .then([]() mutable -> kj::Promise { return EventOutcome::OK; }) .catch_([weakIoctx = context.getWeakRef()](kj::Exception&& e) { // If any exceptions were thrown, mark the outcome accordingly. - weakIoctx->runIfAlive([&e](IoContext& context) { context.getMetrics().reportFailure(e); }); + KJ_IF_SOME(context, weakIoctx) { + context->getMetrics().reportFailure(e); + } return EventOutcome::EXCEPTION; }) .exclusiveJoin(timeoutPromise.then([] { @@ -744,7 +746,9 @@ kj::Promise QueueCustomEvent::run( // internalError outcome if it does happen return EventOutcome::INTERNAL_ERROR; }, [weakIoctx = context.getWeakRef()](kj::Exception&& e) { - weakIoctx->runIfAlive([&e](IoContext& context) { context.getMetrics().reportFailure(e); }); + KJ_IF_SOME(context, weakIoctx) { + context->getMetrics().reportFailure(e); + } return EventOutcome::EXCEPTION; })); diff --git a/src/workerd/api/worker-loader.c++ b/src/workerd/api/worker-loader.c++ index 70d1e4a9738..ddf48c58d3e 100644 --- a/src/workerd/api/worker-loader.c++ +++ b/src/workerd/api/worker-loader.c++ @@ -83,11 +83,11 @@ jsg::Ref WorkerLoader::get( [weakIoctx = ioctx.getWeakRef(), getCode = kj::mv(getCode), compatDateValidation = compatDateValidation](jsg::Lock& js) mutable { return getCode(js).then(js, - [weakIoctx = kj::addRef(*weakIoctx), compatDateValidation]( + [weakIoctx = weakIoctx.addRef(), compatDateValidation]( jsg::Lock& js, WorkerCode code) -> DynamicWorkerSource { - auto& ioctx = JSG_REQUIRE_NONNULL(weakIoctx->tryGet(), Error, + auto ioctx = JSG_REQUIRE_NONNULL(weakIoctx, Error, "The request which initiated this dynamic worker load has already completed."); - return toDynamicWorkerSource(js, ioctx, compatDateValidation, kj::mv(code)); + return toDynamicWorkerSource(js, *ioctx, compatDateValidation, kj::mv(code)); }); }); diff --git a/src/workerd/io/io-context.c++ b/src/workerd/io/io-context.c++ index 83082ba387c..f9f1000b445 100644 --- a/src/workerd/io/io-context.c++ +++ b/src/workerd/io/io-context.c++ @@ -701,8 +701,8 @@ IoContext::~IoContext() noexcept(false) { pe.maybeContext = kj::none; } - // Kill the sentinel so that no weak references can refer to this IoContext anymore. - selfRef->invalidate(); + // Note: Any outstanding kj::WeakRc references are automatically invalidated as the + // last strong reference is dropped (before this destructor runs), so there's nothing to do here. } IoContext::PendingEvent::~PendingEvent() noexcept(false) { @@ -894,16 +894,20 @@ void IoContext::TimeoutManagerImpl::setTimeoutImpl(IoContext& context, Iterator // to be removed when the promise completes. TimeoutTime timeoutTimesKey{when, timeoutTimesTiebreakerCounter++}; timeoutTimes.insert(timeoutTimesKey, kj::mv(paf.fulfiller)); - auto deferredTimeoutTimeRemoval = kj::defer([this, &context, timeoutTimesKey]() { + auto deferredTimeoutTimeRemoval = + kj::defer([this, weakContext = context.getWeakRef(), timeoutTimesKey]() { // If the promise is being destroyed due to IoContext teardown then IoChannelFactory may // no longer be available, but we can just skip starting a new timer in that case as it'd be - // canceled anyway. Similarly we should skip rescheduling if the context has been aborted since - // there's no way the events can run anyway (and we'll cause trouble if `cancelAll()` is being - // called in ~IoContext_IncomingRequest). - if (context.selfRef->isValid() && context.abortException == kj::none) { - bool isNext = timeoutTimes.begin()->key == timeoutTimesKey; - timeoutTimes.erase(timeoutTimesKey); - if (isNext) resetTimerTask(context.getIoChannelFactory().getTimer()); + // canceled anyway. The weak reference upgrade fails once the IoContext is being torn down. + // Similarly we should skip rescheduling if the context has been aborted since there's no way + // the events can run anyway (and we'll cause trouble if `cancelAll()` is being called in + // ~IoContext_IncomingRequest). + KJ_IF_SOME(context, weakContext) { + if (context->abortException == kj::none) { + bool isNext = timeoutTimes.begin()->key == timeoutTimesKey; + timeoutTimes.erase(timeoutTimesKey); + if (isNext) resetTimerTask(context->getIoChannelFactory().getTimer()); + } } }); @@ -1525,11 +1529,11 @@ kj::Maybe> IoContext::getSelfTokenFa return kj::none; } -auto IoContext::tryGetWeakRefForCurrent() -> kj::Maybe> { +kj::WeakRc IoContext::tryGetWeakRefForCurrent() { KJ_IF_SOME(ioContext, tryCurrent()) { return ioContext.getWeakRef(); } else { - return kj::none; + return nullptr; } } @@ -1617,9 +1621,9 @@ void IoContext::requireCurrentOrThrowJs() { } } -void IoContext::requireCurrentOrThrowJs(WeakRef& weak) { - KJ_IF_SOME(ctx, weak.tryGet()) { - if (ctx.isCurrent()) { +void IoContext::requireCurrentOrThrowJs(kj::WeakRc& weak) { + KJ_IF_SOME(ctx, weak) { + if (ctx->isCurrent()) { return; } } diff --git a/src/workerd/io/io-context.h b/src/workerd/io/io-context.h index c2c9c722286..6120aa8bc03 100644 --- a/src/workerd/io/io-context.h +++ b/src/workerd/io/io-context.h @@ -22,13 +22,13 @@ #include #include #include -#include #include #include #include #include #include +#include #include @@ -472,31 +472,15 @@ class IoContext final: public kj::Refcounted, private kj::TaskSet::ErrorHandler // Like requireCurrent() but throws a JS error if this IoContext is not the current. void requireCurrentOrThrowJs(); - // A WeakRef is a weak reference to a IoContext. Note that because IoContext is not - // itself ref-counted, we cannot follow the usual pattern of a weak reference that potentially - // converts to a strong reference. Instead, intended usage looks like so: - // ``` - // auto& context = IoContext::current(); - // return canOutliveContext().then([contextWeakRef = context.getWeakRef()]() mutable { - // auto hadContext = contextWeakRef.runIfAlive([&](IoContext& context){ - // useContextFinally(context); - // }); - // if (!hadContext) { - // doWhatMustBeDone(); - // } - // }); - // ``` - using WeakRef = workerd::WeakRef; - - kj::Own getWeakRef() { - return kj::addRef(*selfRef); + kj::WeakRc getWeakRef() { + return addWeakToThis(); } // If there is a current IoContext, return its WeakRef. - static kj::Maybe> tryGetWeakRefForCurrent(); + static kj::WeakRc tryGetWeakRefForCurrent(); // Like requireCurrentOrThrowJs() but works on a WeakRef. - static void requireCurrentOrThrowJs(WeakRef& weak); + static void requireCurrentOrThrowJs(kj::WeakRc& weak); // Just throw the error that requireCurrentOrThrowJs() would throw on failure. [[noreturn]] static void throwNotCurrentJsError( @@ -1059,8 +1043,6 @@ class IoContext final: public kj::Refcounted, private kj::TaskSet::ErrorHandler private: ThreadContext& thread; - kj::Own selfRef = kj::refcounted(kj::Badge(), *this); - kj::Maybe> tmpDirStoreScope; kj::Own worker; @@ -1651,14 +1633,14 @@ auto IoContext::makeReentryCallbackImpl(Func func, kj::Own attachment) { return [self = getWeakRef(), cs = getCriticalSection(), attachment = kj::mv(attachment), ioFunc = kj::mv(ioFunc)](auto&&... params) mutable { - auto& ctx = JSG_REQUIRE_NONNULL(self->tryGet(), Error, - "The execution context which hosts this callback is no longer running."); + auto ctx = JSG_REQUIRE_NONNULL( + self, Error, "The execution context which hosts this callback is no longer running."); if constexpr (topUp == TOP_UP) { - ctx.getLimitEnforcer().topUpActor(); + ctx->getLimitEnforcer().topUpActor(); } - return ctx.canceler.wrap(ctx.run( + return ctx->canceler.wrap(ctx->run( [&ioFunc, ... params = kj::fwd(params)]( Worker::Lock& lock, IoContext& ctx) mutable { using ResultType = kj::Decay(params)...))>; @@ -1666,12 +1648,10 @@ auto IoContext::makeReentryCallbackImpl(Func func, kj::Own attachment) { auto& func = *ioFunc; if constexpr (kj::isSameType()) { - (void)ctx; func(lock, kj::fwd(params)...); } else if constexpr (jsg::isPromise()) { return ctx.awaitJs(lock, func(lock, kj::fwd(params)...)); } else { - (void)ctx; return func(lock, kj::fwd(params)...); } }, diff --git a/src/workerd/io/io-own.c++ b/src/workerd/io/io-own.c++ index e4e37965ec9..7643cdc3796 100644 --- a/src/workerd/io/io-own.c++ +++ b/src/workerd/io/io-own.c++ @@ -58,8 +58,8 @@ void DeleteQueue::checkFarGet(const DeleteQueue& deleteQueue, const std::type_in IoContext::current().checkFarGet(deleteQueue, type); } -void DeleteQueue::checkWeakGet(workerd::WeakRef& weak) { - JSG_REQUIRE(weak.isValid(), Error, +void DeleteQueue::checkWeakGet(kj::WeakRc& weak) { + JSG_REQUIRE(weak != nullptr, Error, "Couldn't complete operation because the execution context has ended."); } diff --git a/src/workerd/io/io-own.h b/src/workerd/io/io-own.h index 44982a07b4b..a068c931925 100644 --- a/src/workerd/io/io-own.h +++ b/src/workerd/io/io-own.h @@ -101,12 +101,11 @@ class DeleteQueue: public kj::AtomicRefcounted { IoOwn addObject(kj::Own obj, OwnedObjectList& ownedObjects) const; template - ReverseIoOwn addObjectReverse(kj::Own> weakRef, - kj::Own obj, - OwnedObjectList& ownedObjects) const; + ReverseIoOwn addObjectReverse( + kj::WeakRc weakRef, kj::Own obj, OwnedObjectList& ownedObjects) const; static void checkFarGet(const DeleteQueue& deleteQueue, const std::type_info& type); - static void checkWeakGet(workerd::WeakRef& weak); + static void checkWeakGet(kj::WeakRc& weak); private: template @@ -162,9 +161,8 @@ inline IoOwn DeleteQueue::addObject(kj::Own obj, OwnedObjectList& ownedObj } template -inline ReverseIoOwn DeleteQueue::addObjectReverse(kj::Own> weakRef, - kj::Own obj, - OwnedObjectList& ownedObjects) const { +inline ReverseIoOwn DeleteQueue::addObjectReverse( + kj::WeakRc weakRef, kj::Own obj, OwnedObjectList& ownedObjects) const { return ReverseIoOwn(kj::mv(weakRef), addObjectImpl(kj::mv(obj), ownedObjects)); } @@ -277,7 +275,7 @@ template class ReverseIoOwn { public: ReverseIoOwn(ReverseIoOwn&& other) noexcept; - ReverseIoOwn(decltype(nullptr)): item(nullptr) {} + ReverseIoOwn(decltype(nullptr)): weakRef(nullptr), item(nullptr) {} ~ReverseIoOwn() noexcept(false); KJ_DISALLOW_COPY(ReverseIoOwn); @@ -293,7 +291,7 @@ class ReverseIoOwn { // Returns kj::none if the IoContext has been destroyed or if this is null. // This is a safe alternative to operator->() that won't throw or crash. kj::Maybe tryGet() { - if (item != nullptr && weakRef->isValid()) { + if (item != nullptr && weakRef != nullptr) { return *item->ptr.get(); } return kj::none; @@ -303,10 +301,10 @@ class ReverseIoOwn { friend class IoContext; friend class DeleteQueue; - kj::Own> weakRef; + kj::WeakRc weakRef; SpecificOwnedObject* item; - ReverseIoOwn(kj::Own> weakRef, SpecificOwnedObject* item) + ReverseIoOwn(kj::WeakRc weakRef, SpecificOwnedObject* item) : weakRef(kj::mv(weakRef)), item(item) {} }; @@ -391,14 +389,15 @@ ReverseIoOwn::ReverseIoOwn(ReverseIoOwn&& other) noexcept template ReverseIoOwn::~ReverseIoOwn() noexcept(false) { - if (item != nullptr && weakRef->isValid()) { + if (item != nullptr && weakRef != nullptr) { OwnedObjectList::unlink(*item); } } template ReverseIoOwn& ReverseIoOwn::operator=(ReverseIoOwn&& other) { - if (item != nullptr) { + // Only unlink if the IoContext is still alive; otherwise `item` dangles (see ~ReverseIoOwn()). + if (item != nullptr && weakRef != nullptr) { OwnedObjectList::unlink(*item); } weakRef = kj::mv(other.weakRef); @@ -409,7 +408,8 @@ ReverseIoOwn& ReverseIoOwn::operator=(ReverseIoOwn&& other) { template ReverseIoOwn& ReverseIoOwn::operator=(decltype(nullptr)) { - if (item != nullptr) { + // Only unlink if the IoContext is still alive; otherwise `item` dangles (see ~ReverseIoOwn()). + if (item != nullptr && weakRef != nullptr) { OwnedObjectList::unlink(*item); } weakRef = nullptr; @@ -419,13 +419,13 @@ ReverseIoOwn& ReverseIoOwn::operator=(decltype(nullptr)) { template inline T* ReverseIoOwn::operator->() { - DeleteQueue::checkWeakGet(*weakRef); + DeleteQueue::checkWeakGet(weakRef); return item->ptr; } template inline ReverseIoOwn::operator kj::Own() && { - DeleteQueue::checkWeakGet(*weakRef); + DeleteQueue::checkWeakGet(weakRef); auto result = kj::mv(item->ptr); OwnedObjectList::unlink(*item); item = nullptr; diff --git a/src/workerd/io/trace-stream.c++ b/src/workerd/io/trace-stream.c++ index 864687a6720..b2661f99490 100644 --- a/src/workerd/io/trace-stream.c++ +++ b/src/workerd/io/trace-stream.c++ @@ -526,8 +526,8 @@ jsg::JsValue ToJs(jsg::Lock& js, const Log& log, StringCache& cache) { KJ_IF_SOME(slots, log.errorInfo) { // Emit as a positional JS array: each slot is either { name, message, stack? } // for arguments that were native Errors, or `null` for non-Error arguments. - auto arr = js.arr(slots.asPtr(), - [&cache](jsg::Lock& js, const kj::Maybe& slot) -> jsg::JsValue { + auto arr = js.arr( + slots.asPtr(), [&cache](jsg::Lock& js, const kj::Maybe& slot) -> jsg::JsValue { KJ_IF_SOME(info, slot) { auto errObj = js.obj(); errObj.set(js, NAME_STR, cache.get(js, info.name)); @@ -689,8 +689,8 @@ class TailStreamTarget final: public rpc::TailStreamTarget::Server { // we throw a DISCONNECTED exception: this keeps it out of Sentry (see isInterestingException()) // and lets the source side treat it as the peer simply going away. IoContext& ioContext = ([&]() -> IoContext& { - KJ_IF_SOME(ctx, weakIoContext->tryGet()) { - return ctx; + KJ_IF_SOME(ctx, weakIoContext) { + return *ctx; } kj::throwFatalException(KJ_EXCEPTION(DISCONNECTED, "The destination object for this tail session no longer exists.", doneReceiving)); @@ -730,8 +730,8 @@ class TailStreamTarget final: public rpc::TailStreamTarget::Server { })(); if (ioContext.hasOutputGate()) { - return result.then([weakIoContext = weakIoContext->addRef()]() mutable { - return KJ_REQUIRE_NONNULL(weakIoContext->tryGet()).waitForOutputLocks(); + return result.then([weakIoContext = weakIoContext.addRef()]() mutable { + return KJ_REQUIRE_NONNULL(weakIoContext)->waitForOutputLocks(); }); } else { return kj::mv(result); @@ -996,7 +996,7 @@ class TailStreamTarget final: public rpc::TailStreamTarget::Server { return kj::READY_NOW; } - kj::Own weakIoContext; + kj::WeakRc weakIoContext; kj::Maybe entrypointNamePtr; kj::Maybe versionInfo; Frankenvalue props; diff --git a/src/workerd/io/tracer.c++ b/src/workerd/io/tracer.c++ index 7f79f9804fb..7d97a0f982e 100644 --- a/src/workerd/io/tracer.c++ +++ b/src/workerd/io/tracer.c++ @@ -150,8 +150,8 @@ void WorkerTracer::addLog(const tracing::InvocationSpanContext& context, // Clone errorInfo for the STW path because the batched-tail path below also needs it. auto streamErrorInfo = tracing::cloneLogErrorInfo(errorInfo); writer->report(context, - {tracing::Log(timestamp, logLevel, kj::str(message.first(messageSize)), - kj::mv(streamErrorInfo))}, + {tracing::Log( + timestamp, logLevel, kj::str(message.first(messageSize)), kj::mv(streamErrorInfo))}, timestamp, messageSize + errorInfoSize); } @@ -442,8 +442,9 @@ void WorkerTracer::recordTimestamp(kj::Date timestamp) { kj::Date BaseTracer::getTime() { auto& weakIoCtx = KJ_ASSERT_NONNULL(weakIoContext); kj::Date timestamp = kj::UNIX_EPOCH; - weakIoCtx->runIfAlive([×tamp](IoContext& context) { timestamp = context.now(); }); - if (!weakIoCtx->isValid()) { + KJ_IF_SOME(context, weakIoCtx) { + timestamp = context->now(); + } else { // This can happen if we the IoContext gets destroyed following an exception, but we still need // to report a time for the return event. if (completeTime != kj::UNIX_EPOCH) { @@ -471,9 +472,9 @@ void BaseTracer::adjustSpanTime(tracing::SpanEndData& span, kj::Maybe // present. For the RPC case, the adjustment will already have been done earlier and it's ok // for maybeStartTime to be none as this code won't run based on weakIoContext being none. kj::Date startTime = KJ_ASSERT_NONNULL(maybeStartTime); - weakIoCtx->runIfAlive([this, &span, &startTime](IoContext& context) { - if (context.hasCurrentIncomingRequest()) { - span.endTime = context.now(); + KJ_IF_SOME(context, weakIoCtx) { + if (context->hasCurrentIncomingRequest()) { + span.endTime = context->now(); } else { // We have an IOContext, but there's no current IncomingRequest. Always log a warning here, // this should not be happening. Still report completeTime as a useful timestamp if @@ -491,8 +492,7 @@ void BaseTracer::adjustSpanTime(tracing::SpanEndData& span, kj::Maybe LOG_WARNING_PERIODICALLY("reported span without current request"); } } - }); - if (!weakIoCtx->isValid()) { + } else { // This can happen if we start a customEvent from this event and cancel it after this IoContext // gets destroyed. In that case we no longer have an IoContext available and can't get the // current time, but the outcome timestamp will have already been set. Since the outcome diff --git a/src/workerd/io/tracer.h b/src/workerd/io/tracer.h index 6070deae816..ac1ba3316f2 100644 --- a/src/workerd/io/tracer.h +++ b/src/workerd/io/tracer.h @@ -123,7 +123,7 @@ class BaseTracer: public kj::Refcounted { kj::Date completeTime = kj::UNIX_EPOCH; // Weak reference to the IoContext, used to report span end time if available. - kj::Maybe> weakIoContext; + kj::Maybe> weakIoContext; // When true, the destructor will not log a warning about missing Onset event. // Set via markUnused() when a tracer is intentionally not used (e.g., duplicate alarm requests). From 6d87a62beff390b0d5a2df46dd0d36b9ea803214 Mon Sep 17 00:00:00 2001 From: Mike Aizatsky Date: Wed, 15 Jul 2026 10:28:54 -0700 Subject: [PATCH 093/101] [clang-tidy] improved iocontext capture check I noticed some false negatives, this catches more. --- .clang-tidy | 2 +- src/workerd/api/global-scope.c++ | 6 ++--- src/workerd/api/hibernatable-web-socket.c++ | 4 +-- src/workerd/api/queue.c++ | 4 +-- src/workerd/api/restore.c++ | 14 +++++----- src/workerd/api/trace.c++ | 6 ++--- src/workerd/io/trace-stream.c++ | 7 ++--- src/workerd/io/worker-entrypoint.c++ | 27 +++++++++---------- ...ntext-run-manual-capture-positive-test.c++ | 16 +++++++++++ .../iocontext-run-manual-capture-test.sh | 6 ++--- 10 files changed, 55 insertions(+), 37 deletions(-) diff --git a/.clang-tidy b/.clang-tidy index 2db291cd203..013a685e8cc 100644 --- a/.clang-tidy +++ b/.clang-tidy @@ -122,7 +122,7 @@ CustomChecks: hasName("run"), ofClass(cxxRecordDecl(hasName("::workerd::IoContext"))) )), - hasArgument(0, ignoringParenImpCasts(lambdaExpr( + hasArgument(0, ignoringImplicit(lambdaExpr( hasAnyCapture(lambdaCapture(capturesVar(varDecl(hasType(qualType(anyOf( hasDeclaration(cxxRecordDecl(hasName("::workerd::IoContext"))), references(cxxRecordDecl(hasName("::workerd::IoContext"))), diff --git a/src/workerd/api/global-scope.c++ b/src/workerd/api/global-scope.c++ index 7ba5d54aead..be90b54fef8 100644 --- a/src/workerd/api/global-scope.c++ +++ b/src/workerd/api/global-scope.c++ @@ -620,9 +620,9 @@ kj::Promise ServiceWorkerGlobalScope::runAlarm(kj: auto& alarm = KJ_ASSERT_NONNULL(handler.alarm); return context - .run([exportedHandler, &context, timeout, retryCount, scheduledTime, &alarm, - maybeAsyncContext = jsg::AsyncContextFrame::currentRef(lock)]( - Worker::Lock& lock) mutable -> kj::Promise { + .run([exportedHandler, timeout, retryCount, scheduledTime, &alarm, + maybeAsyncContext = jsg::AsyncContextFrame::currentRef(lock)](Worker::Lock& lock, + IoContext& context) mutable -> kj::Promise { jsg::AsyncContextFrame::Scope asyncScope(lock, maybeAsyncContext); // We want to limit alarm handler walltime to 15 minutes at most. If the timeout promise // completes we want to cancel the alarm handler. If the alarm handler promise completes diff --git a/src/workerd/api/hibernatable-web-socket.c++ b/src/workerd/api/hibernatable-web-socket.c++ index aafe85fdf1d..bfbc5e26c19 100644 --- a/src/workerd/api/hibernatable-web-socket.c++ +++ b/src/workerd/api/hibernatable-web-socket.c++ @@ -83,9 +83,9 @@ kj::Promise HibernatableWebSocketCustomEve try { co_await context.run( - [entrypointName = entrypointName, &context, eventParameters = kj::mv(eventParameters), + [entrypointName = entrypointName, eventParameters = kj::mv(eventParameters), versionInfo = kj::mv(versionInfo), props = kj::mv(props), - isDynamicDispatch](Worker::Lock& lock) mutable { + isDynamicDispatch](Worker::Lock& lock, IoContext& context) mutable { KJ_SWITCH_ONEOF(eventParameters.eventType) { KJ_CASE_ONEOF(text, HibernatableSocketParams::Text) { return lock.getGlobalScope().sendHibernatableWebSocketMessage(context, diff --git a/src/workerd/api/queue.c++ b/src/workerd/api/queue.c++ index daf12910cd9..bc5375c84ea 100644 --- a/src/workerd/api/queue.c++ +++ b/src/workerd/api/queue.c++ @@ -685,10 +685,10 @@ kj::Promise QueueCustomEvent::run( // 2. This is where we call into the worker's queue event handler auto runProm = context.run( - [this, entrypointName = entrypointName, &context, queueEvent = kj::addRef(*queueEventHolder), + [this, entrypointName = entrypointName, queueEvent = kj::addRef(*queueEventHolder), &metrics = incomingRequest->getMetrics(), versionInfo = kj::mv(versionInfo), props = kj::mv(props), - isDynamicDispatch](Worker::Lock& lock) mutable -> kj::Promise { + isDynamicDispatch](Worker::Lock& lock, IoContext& context) mutable -> kj::Promise { jsg::AsyncContextFrame::StorageScope traceScope = context.makeAsyncTraceScope(lock); jsg::AsyncContextFrame::StorageScope userTraceScope = context.makeUserAsyncTraceScope(lock); diff --git a/src/workerd/api/restore.c++ b/src/workerd/api/restore.c++ index 6734a0c47d5..395a345ae4d 100644 --- a/src/workerd/api/restore.c++ +++ b/src/workerd/api/restore.c++ @@ -275,9 +275,10 @@ kj::Promise RestoreServiceCustomEvent::run KJ_DEFER({ incomingRequest->drain(waitUntilTasks, kj::mv(incomingRequest)); }); KJ_TRY { - auto channel = co_await ioctx.run( - [this, entrypointName = entrypointName, &ioctx, versionInfo = kj::mv(versionInfo), - props = kj::mv(props), isDynamicDispatch](Worker::Lock& lock) mutable { + auto channel = + co_await ioctx.run([this, entrypointName = entrypointName, + versionInfo = kj::mv(versionInfo), props = kj::mv(props), + isDynamicDispatch](Worker::Lock& lock, IoContext& ioctx) mutable { // Now that we're inside the IoContext, rehydrate any cap-table entries in the params (e.g. // in the edge runtime, turn dehydrated channel tokens into live channels). See // RestoreRehydrateCallback. @@ -397,9 +398,10 @@ kj::Promise RestoreRpcStubCustomEvent::run }); KJ_TRY { - auto cap = co_await ioctx.run( - [this, entrypointName = entrypointName, &ioctx, versionInfo = kj::mv(versionInfo), - props = kj::mv(props), isDynamicDispatch](Worker::Lock& lock) mutable { + auto cap = + co_await ioctx.run([this, entrypointName = entrypointName, + versionInfo = kj::mv(versionInfo), props = kj::mv(props), + isDynamicDispatch](Worker::Lock& lock, IoContext& ioctx) mutable { // Now that we're inside the IoContext, rehydrate any cap-table entries in the params (e.g. // in the edge runtime, turn dehydrated channel tokens into live channels). See // RestoreRehydrateCallback. diff --git a/src/workerd/api/trace.c++ b/src/workerd/api/trace.c++ index 1677d3290dd..55114fed74e 100644 --- a/src/workerd/api/trace.c++ +++ b/src/workerd/api/trace.c++ @@ -717,9 +717,9 @@ void sendTracesToExportedHandler(kj::Own incomingReq auto entrypointName = mapCopyString(entrypointNamePtr); context.addWaitUntil( context - .run([&context, nonEmptyTraces = kj::mv(nonEmptyTraces), - entrypointName = kj::mv(entrypointName), versionInfo = kj::mv(versionInfo), - props = kj::mv(props), isDynamicDispatch](Worker::Lock& lock) mutable { + .run([nonEmptyTraces = kj::mv(nonEmptyTraces), entrypointName = kj::mv(entrypointName), + versionInfo = kj::mv(versionInfo), props = kj::mv(props), + isDynamicDispatch](Worker::Lock& lock, IoContext& context) mutable { jsg::AsyncContextFrame::StorageScope traceScope = context.makeAsyncTraceScope(lock); jsg::AsyncContextFrame::StorageScope userTraceScope = context.makeUserAsyncTraceScope(lock); diff --git a/src/workerd/io/trace-stream.c++ b/src/workerd/io/trace-stream.c++ index b2661f99490..040e82bbe43 100644 --- a/src/workerd/io/trace-stream.c++ +++ b/src/workerd/io/trace-stream.c++ @@ -703,9 +703,10 @@ class TailStreamTarget final: public rpc::TailStreamTarget::Server { // exception handler. auto sharedResults = kj::rc(reportContext.initResults()); - auto promise = ioContext.run([this, &ioContext, sharedResults = sharedResults.addRef(), - reportContext, ownReportContext = ownReportContext->addRef()]( - Worker::Lock& lock) mutable -> kj::Promise { + auto promise = + ioContext.run([this, sharedResults = sharedResults.addRef(), reportContext, + ownReportContext = ownReportContext->addRef()]( + Worker::Lock& lock, IoContext& ioContext) mutable -> kj::Promise { auto params = reportContext.getParams(); KJ_ASSERT(params.hasEvents(), "Events are required."); auto eventReaders = params.getEvents(); diff --git a/src/workerd/io/worker-entrypoint.c++ b/src/workerd/io/worker-entrypoint.c++ index 47ef90251b9..bc7bc568005 100644 --- a/src/workerd/io/worker-entrypoint.c++ +++ b/src/workerd/io/worker-entrypoint.c++ @@ -439,10 +439,9 @@ kj::Promise WorkerEntrypoint::requestImpl(kj::HttpMethod method, }); KJ_TRY { - api::DeferredProxy deferredProxy = - co_await context.run([this, &context, method, url, &headers, &requestBody, - &wrappedResponse = *wrappedResponse, - entrypointName = entrypointName](Worker::Lock& lock) mutable { + api::DeferredProxy deferredProxy = co_await context.run( + [this, method, url, &headers, &requestBody, &wrappedResponse = *wrappedResponse, + entrypointName = entrypointName](Worker::Lock& lock, IoContext& context) mutable { TRACE_EVENT_END("workerd", PERFETTO_TRACK_FROM_POINTER(&context)); TRACE_EVENT( "workerd", "WorkerEntrypoint::request() run", PERFETTO_FLOW_FROM_POINTER(this)); @@ -674,9 +673,9 @@ kj::Promise WorkerEntrypoint::connect(kj::StringPtr host, return wrapWithCanceler( context - .run([this, &headers, &context, &connection, &response, entrypointName = entrypointName, + .run([this, &headers, &connection, &response, entrypointName = entrypointName, versionInfo = kj::mv(versionInfo), - host = kj::str(host)](Worker::Lock& lock) mutable { + host = kj::str(host)](Worker::Lock& lock, IoContext& context) mutable { jsg::AsyncContextFrame::StorageScope traceScope = context.makeAsyncTraceScope(lock); jsg::AsyncContextFrame::StorageScope userTraceScope = context.makeUserAsyncTraceScope(lock); @@ -803,10 +802,10 @@ kj::Promise WorkerEntrypoint::runScheduled( incomingRequest->delivered(); // Scheduled handlers run entirely in waitUntil() tasks. - context.addWaitUntil( - context.run([scheduledTime, cron, entrypointName = entrypointName, - versionInfo = kj::mv(versionInfo), props = kj::mv(props), &context, - &metrics = incomingRequest->getMetrics()](Worker::Lock& lock) mutable { + context.addWaitUntil(context.run( + [scheduledTime, cron, entrypointName = entrypointName, versionInfo = kj::mv(versionInfo), + props = kj::mv(props), &metrics = incomingRequest->getMetrics()]( + Worker::Lock& lock, IoContext& context) mutable { TRACE_EVENT("workerd", "WorkerEntrypoint::runScheduled() run"); jsg::AsyncContextFrame::StorageScope traceScope = context.makeAsyncTraceScope(lock); jsg::AsyncContextFrame::StorageScope userTraceScope = context.makeUserAsyncTraceScope(lock); @@ -880,8 +879,8 @@ kj::Promise WorkerEntrypoint::runAlarmImpl( try { auto result = co_await context.run([scheduledTime, retryCount, entrypointName = entrypointName, - versionInfo = kj::mv(versionInfo), props = kj::mv(props), - &context](Worker::Lock& lock) mutable { + versionInfo = kj::mv(versionInfo), props = kj::mv(props)]( + Worker::Lock& lock, IoContext& context) mutable { jsg::AsyncContextFrame::StorageScope traceScope = context.makeAsyncTraceScope(lock); jsg::AsyncContextFrame::StorageScope userTraceScope = context.makeUserAsyncTraceScope(lock); @@ -977,8 +976,8 @@ kj::Promise WorkerEntrypoint::test() { context.addWaitUntil( context.run([entrypointName = entrypointName, versionInfo = kj::mv(versionInfo), - props = kj::mv(props), &context, &metrics = incomingRequest->getMetrics()]( - Worker::Lock& lock) mutable -> kj::Promise { + props = kj::mv(props), &metrics = incomingRequest->getMetrics()]( + Worker::Lock& lock, IoContext& context) mutable -> kj::Promise { TRACE_EVENT("workerd", "WorkerEntrypoint::test() run"); jsg::AsyncContextFrame::StorageScope traceScope = context.makeAsyncTraceScope(lock); jsg::AsyncContextFrame::StorageScope userTraceScope = context.makeUserAsyncTraceScope(lock); diff --git a/tools/clang-tidy/iocontext-run-manual-capture-positive-test.c++ b/tools/clang-tidy/iocontext-run-manual-capture-positive-test.c++ index b89be329452..9a7a4db8804 100644 --- a/tools/clang-tidy/iocontext-run-manual-capture-positive-test.c++ +++ b/tools/clang-tidy/iocontext-run-manual-capture-positive-test.c++ @@ -21,6 +21,14 @@ class IoContext { void run(Func&& func) {} }; +// A type with a non-trivial destructor. When captured, it makes the closure's +// own destructor non-trivial, so the lambda argument to run() is wrapped in a +// CXXBindTemporaryExpr in the AST. The matcher must still see through it. +struct OwnsResource { + ~OwnsResource(); + OwnsResource clone(); +}; + void captureByReference(IoContext& ctx) { // Captures `ctx` (an IoContext&) explicitly by reference. ctx.run([&ctx](Worker::Lock& lock) { (void)ctx; }); @@ -36,4 +44,12 @@ void captureByDefaultReference(IoContext& ctx) { ctx.run([&](Worker::Lock& lock) { (void)ctx; }); } +void captureAlongsideNonTrivialClosure(IoContext& ctx, OwnsResource& resource) { + // The init-capture gives the closure a non-trivial destructor, wrapping the + // lambda argument in a CXXBindTemporaryExpr. The check must still flag the + // manual IoContext capture (regression test for ignoringImplicit vs + // ignoringParenImpCasts). + ctx.run([&ctx, held = resource.clone()](Worker::Lock& lock) { (void)ctx; }); +} + } // namespace workerd diff --git a/tools/clang-tidy/iocontext-run-manual-capture-test.sh b/tools/clang-tidy/iocontext-run-manual-capture-test.sh index aebfcf84806..672e396a289 100755 --- a/tools/clang-tidy/iocontext-run-manual-capture-test.sh +++ b/tools/clang-tidy/iocontext-run-manual-capture-test.sh @@ -48,10 +48,10 @@ if [[ "${positive_output}" != *"${MESSAGE}"* ]]; then exit 1 fi -# Each of the three positive cases should be flagged. +# Each of the four positive cases should be flagged. positive_count=$(printf '%s\n' "${positive_output}" | grep -c "${CHECK}") -if [[ ${positive_count} -ne 3 ]]; then - printf '%s\n' "Expected 3 ${CHECK} diagnostics, got ${positive_count}." >&2 +if [[ ${positive_count} -ne 4 ]]; then + printf '%s\n' "Expected 4 ${CHECK} diagnostics, got ${positive_count}." >&2 printf '%s\n' "${positive_output}" >&2 exit 1 fi From b3c97ceb78ee67ab2448a3fe9bb794e6eb298401 Mon Sep 17 00:00:00 2001 From: Mike Aizatsky Date: Wed, 15 Jul 2026 12:54:06 -0700 Subject: [PATCH 094/101] [build] split small io pieces --- src/workerd/api/BUILD.bazel | 16 +++++++-- src/workerd/api/http.h | 2 +- src/workerd/io/BUILD.bazel | 38 +++++++++++++++++++--- src/workerd/io/compatibility-date-test.c++ | 2 ++ src/workerd/io/compatibility-date.c++ | 8 ++--- src/workerd/io/compatibility-date.h | 6 ++-- src/workerd/io/trace.c++ | 20 ++++++------ src/workerd/io/trace.h | 6 ++-- src/workerd/io/validation.h | 31 ++++++++++++++++++ src/workerd/io/worker.h | 17 ++-------- src/workerd/jsg/modules-new.c++ | 3 +- src/workerd/jsg/modules.c++ | 3 +- 12 files changed, 106 insertions(+), 46 deletions(-) create mode 100644 src/workerd/io/validation.h diff --git a/src/workerd/api/BUILD.bazel b/src/workerd/api/BUILD.bazel index b7a60868851..91c459570a1 100644 --- a/src/workerd/api/BUILD.bazel +++ b/src/workerd/api/BUILD.bazel @@ -13,7 +13,6 @@ filegroup( "basics.c++", "blob.c++", "cache.c++", - "cf-property.c++", "container.c++", "events.c++", "eventsource.c++", @@ -65,7 +64,6 @@ filegroup( "basics.h", "blob.h", "cache.h", - "cf-property.h", "container.h", "events.h", "eventsource.h", @@ -180,6 +178,20 @@ wd_cc_library( ], ) +wd_cc_library( + name = "cf-property", + srcs = ["cf-property.c++"], + hdrs = ["cf-property.h"], + implementation_deps = [ + "//src/workerd/io:features", + "//src/workerd/util:own-util", + ], + visibility = ["//visibility:public"], + deps = [ + "//src/workerd/jsg", + ], +) + wd_cc_library( name = "commonjs", srcs = [ diff --git a/src/workerd/api/http.h b/src/workerd/api/http.h index 37a1b3e34aa..398a71e2080 100644 --- a/src/workerd/api/http.h +++ b/src/workerd/api/http.h @@ -6,13 +6,13 @@ #include "basics.h" #include "blob.h" -#include "cf-property.h" #include "form-data.h" #include "headers.h" #include "queue.h" #include "web-socket.h" #include "worker-rpc.h" +#include #include #include #include diff --git a/src/workerd/io/BUILD.bazel b/src/workerd/io/BUILD.bazel index 6875eb2afbd..f9a693b053c 100644 --- a/src/workerd/io/BUILD.bazel +++ b/src/workerd/io/BUILD.bazel @@ -33,8 +33,6 @@ wd_cc_library( # IoContext -> Worker -> ServiceWorkerGlobalScope -> (various api targets) dependency chain. # TODO(cleanup): Fix this. srcs = [ - "compatibility-date.c++", - "external-pusher.c++", "features.c++", "hibernation-manager.c++", "io-channels.c++", @@ -51,8 +49,6 @@ wd_cc_library( ] + ["//src/workerd/api:srcs"], hdrs = [ "access-info.h", - "compatibility-date.h", - "external-pusher.h", "hibernation-manager.h", "io-channels.h", "io-context.h", @@ -89,7 +85,9 @@ wd_cc_library( ":actor-id", ":actor-storage_capnp", ":cdp_capnp", + ":compatibility-date", ":container_capnp", + ":external-pusher", ":features", ":frankenvalue", ":io-gate", @@ -104,6 +102,7 @@ wd_cc_library( ":worker-source", "//src/rust/cxx-integration", "//src/workerd/api:analytics-engine_capnp", + "//src/workerd/api:cf-property", "//src/workerd/api:deferred-proxy", "//src/workerd/api:encoding", "//src/workerd/api:fuzzilli", @@ -130,6 +129,36 @@ wd_cc_library( ], ) +wd_cc_library( + name = "compatibility-date", + srcs = ["compatibility-date.c++"], + hdrs = [ + "compatibility-date.h", + "validation.h", + ], + visibility = ["//visibility:public"], + deps = [ + ":compatibility-date_capnp", + ":maximum-compatibility-date", + ":release-version", + "@capnp-cpp//src/capnp", + "@capnp-cpp//src/kj", + ], +) + +wd_cc_library( + name = "external-pusher", + srcs = ["external-pusher.c++"], + hdrs = ["external-pusher.h"], + visibility = ["//visibility:public"], + deps = [ + ":worker-interface_capnp", + "//src/workerd/jsg", + "@capnp-cpp//src/capnp/compat:http-over-capnp", + "@capnp-cpp//src/kj:kj-async", + ], +) + # Headers only target to avoid having circular dependencies. wd_cc_library( name = "features", @@ -457,6 +486,7 @@ kj_test( kj_test( src = "compatibility-date-test.c++", deps = [ + ":compatibility-date", ":io", "@capnp-cpp//src/capnp:capnpc", ], diff --git a/src/workerd/io/compatibility-date-test.c++ b/src/workerd/io/compatibility-date-test.c++ index 8e27bcb4315..32dcb1d80fe 100644 --- a/src/workerd/io/compatibility-date-test.c++ +++ b/src/workerd/io/compatibility-date-test.c++ @@ -5,10 +5,12 @@ #include "compatibility-date.h" #include +#include #include #include #include +#include #include #include diff --git a/src/workerd/io/compatibility-date.c++ b/src/workerd/io/compatibility-date.c++ index 1a266ec4d43..10dfa180af9 100644 --- a/src/workerd/io/compatibility-date.c++ +++ b/src/workerd/io/compatibility-date.c++ @@ -66,7 +66,7 @@ struct CompatDate { return CompatDate{year, month, day}; } - static CompatDate parse(kj::StringPtr text, Worker::ValidationErrorReporter& errorReporter) { + static CompatDate parse(kj::StringPtr text, ValidationErrorReporter& errorReporter) { static constexpr CompatDate DEFAULT_DATE{2021, 5, 1}; KJ_IF_SOME(v, parse(text)) { return v; @@ -102,7 +102,7 @@ kj::String currentDateStr() { static void compileCompatibilityFlags(kj::StringPtr compatDate, kj::HashSet flagSet, CompatibilityFlags::Builder output, - Worker::ValidationErrorReporter& errorReporter, + ValidationErrorReporter& errorReporter, bool allowExperimentalFeatures, CompatibilityDateValidation dateValidation, kj::ArrayPtr allowedExperimentalFlags) { @@ -273,7 +273,7 @@ static void compileCompatibilityFlags(kj::StringPtr compatDate, void compileCompatibilityFlags(kj::StringPtr compatDate, capnp::List::Reader compatFlags, CompatibilityFlags::Builder output, - Worker::ValidationErrorReporter& errorReporter, + ValidationErrorReporter& errorReporter, bool allowExperimentalFeatures, CompatibilityDateValidation dateValidation, kj::ArrayPtr allowedExperimentalFlags) { @@ -292,7 +292,7 @@ void compileCompatibilityFlags(kj::StringPtr compatDate, void compileCompatibilityFlags(kj::StringPtr compatDate, kj::ArrayPtr compatFlags, CompatibilityFlags::Builder output, - Worker::ValidationErrorReporter& errorReporter, + ValidationErrorReporter& errorReporter, bool allowExperimentalFeatures, CompatibilityDateValidation dateValidation, kj::ArrayPtr allowedExperimentalFlags) { diff --git a/src/workerd/io/compatibility-date.h b/src/workerd/io/compatibility-date.h index 7d405d4178e..a577aa5f94c 100644 --- a/src/workerd/io/compatibility-date.h +++ b/src/workerd/io/compatibility-date.h @@ -5,7 +5,7 @@ #pragma once #include -#include +#include namespace workerd { @@ -34,14 +34,14 @@ enum class CompatibilityDateValidation { void compileCompatibilityFlags(kj::StringPtr compatDate, capnp::List::Reader compatFlags, CompatibilityFlags::Builder output, - Worker::ValidationErrorReporter& errorReporter, + ValidationErrorReporter& errorReporter, bool allowExperimentalFeatures, CompatibilityDateValidation dateValidation, kj::ArrayPtr allowedExperimentalFlags); void compileCompatibilityFlags(kj::StringPtr compatDate, kj::ArrayPtr compatFlags, CompatibilityFlags::Builder output, - Worker::ValidationErrorReporter& errorReporter, + ValidationErrorReporter& errorReporter, bool allowExperimentalFeatures, CompatibilityDateValidation dateValidation, kj::ArrayPtr allowedExperimentalFlags); diff --git a/src/workerd/io/trace.c++ b/src/workerd/io/trace.c++ index 39bc3657028..e79d6a65acb 100644 --- a/src/workerd/io/trace.c++ +++ b/src/workerd/io/trace.c++ @@ -793,8 +793,8 @@ void ErrorInfo::copyTo(rpc::Trace::ErrorInfo::Builder builder) const { } ErrorInfo ErrorInfo::clone() const { - return ErrorInfo(kj::str(name), kj::str(message), - stack.map([](const kj::String& s) { return kj::str(s); })); + return ErrorInfo( + kj::str(name), kj::str(message), stack.map([](const kj::String& s) { return kj::str(s); })); } LogErrorInfo cloneLogErrorInfo(const LogErrorInfo& src) { @@ -855,14 +855,14 @@ Log::Log(rpc::Trace::Log::Reader reader) auto slots = kj::heapArray>(listReader.size()); for (auto i: kj::zeroTo(listReader.size())) { auto slotReader = listReader[i]; - switch (slotReader.which()) { - case rpc::Trace::ErrorInfoSlot::INFO: - slots[i] = tracing::ErrorInfo(slotReader.getInfo()); - break; - case rpc::Trace::ErrorInfoSlot::NONE: - // slot stays kj::none - break; - } + switch (slotReader.which()) { + case rpc::Trace::ErrorInfoSlot::INFO: + slots[i] = tracing::ErrorInfo(slotReader.getInfo()); + break; + case rpc::Trace::ErrorInfoSlot::NONE: + // slot stays kj::none + break; + } } errorInfo = kj::mv(slots); } diff --git a/src/workerd/io/trace.h b/src/workerd/io/trace.h index 32b84cbfd4f..548e1de6bb5 100644 --- a/src/workerd/io/trace.h +++ b/src/workerd/io/trace.h @@ -647,10 +647,8 @@ LogErrorInfo cloneLogErrorInfo(const LogErrorInfo& src); // Describes a log event struct Log final { - explicit Log(kj::Date timestamp, - LogLevel logLevel, - kj::String message, - LogErrorInfo errorInfo = kj::none); + explicit Log( + kj::Date timestamp, LogLevel logLevel, kj::String message, LogErrorInfo errorInfo = kj::none); Log(rpc::Trace::Log::Reader reader); Log(Log&&) noexcept = default; KJ_DISALLOW_COPY(Log); diff --git a/src/workerd/io/validation.h b/src/workerd/io/validation.h new file mode 100644 index 00000000000..091a886a0ab --- /dev/null +++ b/src/workerd/io/validation.h @@ -0,0 +1,31 @@ +// Copyright (c) 2017-2022 Cloudflare, Inc. +// Licensed under the Apache 2.0 license found in the LICENSE file or at: +// https://opensource.org/licenses/Apache-2.0 + +#pragma once + +#include +#include +#include + +namespace workerd { + +// Interface for reporting errors encountered while validating a Worker's configuration (e.g. +// compatibility flags, exported entrypoints, Durable Object classes). +class ValidationErrorReporter { + public: + virtual void addError(kj::String error) = 0; + + // Report that the Worker implements a stateless entrypoint (e.g. WorkerEntrypoint or plain + // object export) with the given export name and methods. + virtual void addEntrypoint( + kj::Maybe exportName, kj::Array methods) = 0; + + // Report that the Worker exports a Durable Object class with the given name. + virtual void addActorClass(kj::StringPtr exportName) = 0; + + // Report that the Worker exports a Workflow class with the given name. + virtual void addWorkflowClass(kj::StringPtr exportName, kj::Array methods) = 0; +}; + +} // namespace workerd diff --git a/src/workerd/io/worker.h b/src/workerd/io/worker.h index 2b995ccbb96..9aa29bc1559 100644 --- a/src/workerd/io/worker.h +++ b/src/workerd/io/worker.h @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include @@ -106,21 +107,7 @@ class Worker: public kj::AtomicRefcounted { class Isolate; class Api; - class ValidationErrorReporter { - public: - virtual void addError(kj::String error) = 0; - - // Report that the Worker implements a stateless entrypoint (e.g. WorkerEntrypoint or plain - // object export) with the given export name and methods. - virtual void addEntrypoint( - kj::Maybe exportName, kj::Array methods) = 0; - - // Report that the Worker exports a Durable Object class with the given name. - virtual void addActorClass(kj::StringPtr exportName) = 0; - - // Report that the Worker exports a Workflow class with the given name. - virtual void addWorkflowClass(kj::StringPtr exportName, kj::Array methods) = 0; - }; + using ValidationErrorReporter = workerd::ValidationErrorReporter; class LockType; diff --git a/src/workerd/jsg/modules-new.c++ b/src/workerd/jsg/modules-new.c++ index 6dfc7d0570e..6df9c32ef68 100644 --- a/src/workerd/jsg/modules-new.c++ +++ b/src/workerd/jsg/modules-new.c++ @@ -279,8 +279,7 @@ class SyntheticModule final: public Module { exports[n++] = js.strIntern(exp); } return v8::Module::CreateSyntheticModule(js.v8Isolate, js.str(id().getHref()), - std::span>(exports.data(), exports.size()), - evaluationSteps); + std::span>(exports.data(), exports.size()), evaluationSteps); } private: diff --git a/src/workerd/jsg/modules.c++ b/src/workerd/jsg/modules.c++ index ede9aafd886..87b3cc5bd92 100644 --- a/src/workerd/jsg/modules.c++ +++ b/src/workerd/jsg/modules.c++ @@ -6,11 +6,12 @@ #include "setup.h" #include "util.h" -#include #include #include +#include + namespace workerd::jsg { namespace { From dc7e74befb00dc9ef40048f74e6bb4a6e0096930 Mon Sep 17 00:00:00 2001 From: Mike Aizatsky Date: Wed, 15 Jul 2026 12:57:11 -0700 Subject: [PATCH 095/101] just format --- src/workerd/io/trace.c++ | 20 ++++++++++---------- src/workerd/io/trace.h | 6 ++---- src/workerd/jsg/modules-new.c++ | 3 +-- src/workerd/jsg/modules.c++ | 3 ++- 4 files changed, 15 insertions(+), 17 deletions(-) diff --git a/src/workerd/io/trace.c++ b/src/workerd/io/trace.c++ index 39bc3657028..e79d6a65acb 100644 --- a/src/workerd/io/trace.c++ +++ b/src/workerd/io/trace.c++ @@ -793,8 +793,8 @@ void ErrorInfo::copyTo(rpc::Trace::ErrorInfo::Builder builder) const { } ErrorInfo ErrorInfo::clone() const { - return ErrorInfo(kj::str(name), kj::str(message), - stack.map([](const kj::String& s) { return kj::str(s); })); + return ErrorInfo( + kj::str(name), kj::str(message), stack.map([](const kj::String& s) { return kj::str(s); })); } LogErrorInfo cloneLogErrorInfo(const LogErrorInfo& src) { @@ -855,14 +855,14 @@ Log::Log(rpc::Trace::Log::Reader reader) auto slots = kj::heapArray>(listReader.size()); for (auto i: kj::zeroTo(listReader.size())) { auto slotReader = listReader[i]; - switch (slotReader.which()) { - case rpc::Trace::ErrorInfoSlot::INFO: - slots[i] = tracing::ErrorInfo(slotReader.getInfo()); - break; - case rpc::Trace::ErrorInfoSlot::NONE: - // slot stays kj::none - break; - } + switch (slotReader.which()) { + case rpc::Trace::ErrorInfoSlot::INFO: + slots[i] = tracing::ErrorInfo(slotReader.getInfo()); + break; + case rpc::Trace::ErrorInfoSlot::NONE: + // slot stays kj::none + break; + } } errorInfo = kj::mv(slots); } diff --git a/src/workerd/io/trace.h b/src/workerd/io/trace.h index 32b84cbfd4f..548e1de6bb5 100644 --- a/src/workerd/io/trace.h +++ b/src/workerd/io/trace.h @@ -647,10 +647,8 @@ LogErrorInfo cloneLogErrorInfo(const LogErrorInfo& src); // Describes a log event struct Log final { - explicit Log(kj::Date timestamp, - LogLevel logLevel, - kj::String message, - LogErrorInfo errorInfo = kj::none); + explicit Log( + kj::Date timestamp, LogLevel logLevel, kj::String message, LogErrorInfo errorInfo = kj::none); Log(rpc::Trace::Log::Reader reader); Log(Log&&) noexcept = default; KJ_DISALLOW_COPY(Log); diff --git a/src/workerd/jsg/modules-new.c++ b/src/workerd/jsg/modules-new.c++ index 6dfc7d0570e..6df9c32ef68 100644 --- a/src/workerd/jsg/modules-new.c++ +++ b/src/workerd/jsg/modules-new.c++ @@ -279,8 +279,7 @@ class SyntheticModule final: public Module { exports[n++] = js.strIntern(exp); } return v8::Module::CreateSyntheticModule(js.v8Isolate, js.str(id().getHref()), - std::span>(exports.data(), exports.size()), - evaluationSteps); + std::span>(exports.data(), exports.size()), evaluationSteps); } private: diff --git a/src/workerd/jsg/modules.c++ b/src/workerd/jsg/modules.c++ index ede9aafd886..87b3cc5bd92 100644 --- a/src/workerd/jsg/modules.c++ +++ b/src/workerd/jsg/modules.c++ @@ -6,11 +6,12 @@ #include "setup.h" #include "util.h" -#include #include #include +#include + namespace workerd::jsg { namespace { From 9f2514ea4ec25ecd43337d1eee43e11d3e27a2fc Mon Sep 17 00:00:00 2001 From: Mike Aizatsky Date: Wed, 15 Jul 2026 11:59:42 -0700 Subject: [PATCH 096/101] hardening IoContext& in api/streams --- src/workerd/api/streams/common.h | 5 -- src/workerd/api/streams/compression.c++ | 15 ++-- .../streams/readable-source-adapter-test.c++ | 78 ++++++++++++------- .../api/streams/readable-source-adapter.c++ | 38 +++++---- .../api/streams/readable-source-adapter.h | 2 +- src/workerd/api/streams/readable-source.c++ | 18 ++--- src/workerd/api/streams/readable-source.h | 3 +- src/workerd/api/streams/readable.c++ | 30 ++++--- src/workerd/api/streams/readable.h | 9 +-- src/workerd/api/streams/standard.c++ | 43 ++++------ src/workerd/api/streams/standard.h | 10 +-- .../streams/writable-sink-adapter-test.c++ | 77 ++++++++++-------- .../api/streams/writable-sink-adapter.c++ | 26 ++++--- .../api/streams/writable-sink-adapter.h | 3 +- .../api/streams/writable-sink-test.c++ | 2 +- src/workerd/api/streams/writable-sink.c++ | 25 +++--- src/workerd/api/streams/writable-sink.h | 3 +- src/workerd/api/streams/writable.c++ | 38 ++++----- src/workerd/api/streams/writable.h | 4 +- 19 files changed, 225 insertions(+), 204 deletions(-) diff --git a/src/workerd/api/streams/common.h b/src/workerd/api/streams/common.h index f614c8e7b83..4824116975a 100644 --- a/src/workerd/api/streams/common.h +++ b/src/workerd/api/streams/common.h @@ -959,9 +959,4 @@ jsg::Promise rejectedMaybeHandledPromise(jsg::Lock& js, jsg::JsValue reason, return kj::mv(prp.promise); } -inline kj::Maybe tryGetIoContext() { - // TODO(cleanup): This function is obsolete; callers should just call IoContext::tryCurrent() - return IoContext::tryCurrent(); -} - } // namespace workerd::api diff --git a/src/workerd/api/streams/compression.c++ b/src/workerd/api/streams/compression.c++ index bd59669f886..449d2634229 100644 --- a/src/workerd/api/streams/compression.c++ +++ b/src/workerd/api/streams/compression.c++ @@ -538,13 +538,12 @@ class CompressionStreamAdapter final: public kj::Refcounted, public ReadableStreamSource, public WritableStreamSink { public: - explicit CompressionStreamAdapter(kj::Rc> impl) - : impl(kj::mv(impl)), - ioContext(IoContext::current()) {} + explicit CompressionStreamAdapter(kj::Rc> impl): impl(kj::mv(impl)) {} // ReadableStreamSource implementation kj::Promise tryRead(void* buffer, size_t minBytes, size_t maxBytes) override { - return impl->tryRead(buffer, minBytes, maxBytes).attach(ioContext.registerPendingEvent()); + return impl->tryRead(buffer, minBytes, maxBytes) + .attach(ioContext.assertLive().registerPendingEvent()); } void cancel(kj::Exception reason) override { @@ -554,15 +553,15 @@ class CompressionStreamAdapter final: public kj::Refcounted, // WritableStreamSink implementation kj::Promise write(kj::ArrayPtr buffer) override { - return impl->write(buffer).attach(ioContext.registerPendingEvent()); + return impl->write(buffer).attach(ioContext.assertLive().registerPendingEvent()); } kj::Promise write(kj::ArrayPtr> pieces) override { - return impl->write(pieces).attach(ioContext.registerPendingEvent()); + return impl->write(pieces).attach(ioContext.assertLive().registerPendingEvent()); } kj::Promise end() override { - return impl->end().attach(ioContext.registerPendingEvent()); + return impl->end().attach(ioContext.assertLive().registerPendingEvent()); } void abort(kj::Exception reason) override { @@ -571,7 +570,7 @@ class CompressionStreamAdapter final: public kj::Refcounted, private: kj::Rc> impl; - IoContext& ioContext; + kj::WeakRc ioContext = IoContext::current().getWeakRef(); }; kj::Rc> createCompressionStreamImpl( diff --git a/src/workerd/api/streams/readable-source-adapter-test.c++ b/src/workerd/api/streams/readable-source-adapter-test.c++ index 816e2009781..66f8159f942 100644 --- a/src/workerd/api/streams/readable-source-adapter-test.c++ +++ b/src/workerd/api/streams/readable-source-adapter-test.c++ @@ -1076,7 +1076,8 @@ KJ_TEST("KjAdapter constructor with valid normal ReadableStream") { fixture.runInIoContext([&](const TestFixture::Environment& env) { auto stream = createFiniteBytesReadableStream(env.js, 16 * 1024); KJ_ASSERT(!stream->isLocked(), "Stream should not be locked before adapter construction"); - auto adapter = kj::heap(env.js, env.context, stream.addRef()); + auto adapter = + kj::heap(env.js, env.context.getWeakRef(), stream.addRef()); KJ_ASSERT(stream->isLocked(), "Stream should be locked after adapter construction"); // The size is known because we provided expectedLength in the source. @@ -1108,7 +1109,8 @@ KJ_TEST("KjAdapter constructor with valid byob ReadableStream") { fixture.runInIoContext([&](const TestFixture::Environment& env) { auto stream = createFiniteByobReadableStream(env.js, 16 * 1024); KJ_ASSERT(!stream->isLocked(), "Stream should not be locked before adapter construction"); - auto adapter = kj::heap(env.js, env.context, stream.addRef()); + auto adapter = + kj::heap(env.js, env.context.getWeakRef(), stream.addRef()); KJ_ASSERT(stream->isLocked(), "Stream should be locked after adapter construction"); // The size is known because we provided expectedLength in the source. @@ -1132,7 +1134,8 @@ KJ_TEST("KjAdapter constructor with valid ReadableStream manual cancel") { fixture.runInIoContext([&](const TestFixture::Environment& env) { auto stream = createFiniteBytesReadableStream(env.js, 16 * 1024); KJ_ASSERT(!stream->isLocked(), "Stream should not be locked before adapter construction"); - auto adapter = kj::heap(env.js, env.context, stream.addRef()); + auto adapter = + kj::heap(env.js, env.context.getWeakRef(), stream.addRef()); KJ_ASSERT(stream->isLocked(), "Stream should be locked after adapter construction"); adapter->cancel(KJ_EXCEPTION(FAILED, "Manual cancel")); @@ -1158,7 +1161,7 @@ KJ_TEST("KjAdapter constructor with locked/disturbed stream fails") { auto stream = createFiniteBytesReadableStream(env.js, 16 * 1024); auto reader = stream->getReader(env.js, kj::none); try { - kj::heap(env.js, env.context, stream.addRef()); + kj::heap(env.js, env.context.getWeakRef(), stream.addRef()); KJ_FAIL_ASSERT("Should not be able to get adapter"); } catch (...) { // Expected. @@ -1175,7 +1178,7 @@ KJ_TEST("KjAdapter constructor with locked/disturbed stream fails") { KJ_ASSERT(stream->isDisturbed()); try { - kj::heap(env.js, env.context, stream.addRef()); + kj::heap(env.js, env.context.getWeakRef(), stream.addRef()); KJ_FAIL_ASSERT("Should not be able to get adapter"); } catch (...) { // Expected. @@ -1199,7 +1202,8 @@ KJ_TEST("KjAdapter read with valid buffer and byte ranges") { fixture.runInIoContext([&](const TestFixture::Environment& env) { auto stream = createFiniteBytesReadableStream(env.js, 1024, &counter); KJ_ASSERT(!stream->isLocked(), "Stream should not be locked before adapter construction"); - auto adapter = kj::heap(env.js, env.context, stream.addRef()); + auto adapter = + kj::heap(env.js, env.context.getWeakRef(), stream.addRef()); KJ_ASSERT(stream->isLocked(), "Stream should be locked after adapter construction"); auto buffer = kj::heapArray(2049); @@ -1242,7 +1246,8 @@ KJ_TEST("KjAdapter read with left over (source provides more than requested)") { fixture.runInIoContext([&](const TestFixture::Environment& env) { auto stream = createFiniteBytesReadableStream(env.js, 1024, &counter); KJ_ASSERT(!stream->isLocked(), "Stream should not be locked before adapter construction"); - auto adapter = kj::heap(env.js, env.context, stream.addRef()); + auto adapter = + kj::heap(env.js, env.context.getWeakRef(), stream.addRef()); KJ_ASSERT(stream->isLocked(), "Stream should be locked after adapter construction"); auto buffer = kj::heapArray(1000); @@ -1284,7 +1289,8 @@ KJ_TEST("KjAdapter read with clamped minBytes (minBytes=0)") { fixture.runInIoContext([&](const TestFixture::Environment& env) { auto stream = createFiniteBytesReadableStream(env.js, 5, &counter); KJ_ASSERT(!stream->isLocked(), "Stream should not be locked before adapter construction"); - auto adapter = kj::heap(env.js, env.context, stream.addRef()); + auto adapter = + kj::heap(env.js, env.context.getWeakRef(), stream.addRef()); KJ_ASSERT(stream->isLocked(), "Stream should be locked after adapter construction"); auto buffer = kj::heapArray(3); @@ -1309,7 +1315,8 @@ KJ_TEST("KjAdapter read with clamped minBytes (minBytes > maxBytes)") { fixture.runInIoContext([&](const TestFixture::Environment& env) { auto stream = createFiniteBytesReadableStream(env.js, 5, &counter); KJ_ASSERT(!stream->isLocked(), "Stream should not be locked before adapter construction"); - auto adapter = kj::heap(env.js, env.context, stream.addRef()); + auto adapter = + kj::heap(env.js, env.context.getWeakRef(), stream.addRef()); KJ_ASSERT(stream->isLocked(), "Stream should be locked after adapter construction"); auto buffer = kj::heapArray(3); @@ -1334,7 +1341,8 @@ KJ_TEST("KjAdapter read with zero length buffer") { fixture.runInIoContext([&](const TestFixture::Environment& env) { auto stream = createFiniteBytesReadableStream(env.js, 5, &counter); KJ_ASSERT(!stream->isLocked(), "Stream should not be locked before adapter construction"); - auto adapter = kj::heap(env.js, env.context, stream.addRef()); + auto adapter = + kj::heap(env.js, env.context.getWeakRef(), stream.addRef()); KJ_ASSERT(stream->isLocked(), "Stream should be locked after adapter construction"); auto buffer = kj::heapArray(0); @@ -1358,7 +1366,8 @@ KJ_TEST("KjAdapter forbid concurrent reads") { fixture.runInIoContext([&](const TestFixture::Environment& env) { auto stream = createFiniteBytesReadableStream(env.js, 5, &counter); - auto adapter = kj::heap(env.js, env.context, stream.addRef()); + auto adapter = + kj::heap(env.js, env.context.getWeakRef(), stream.addRef()); auto buffer = kj::heapArray(2); @@ -1385,7 +1394,8 @@ KJ_TEST("KjAdapter cancel in-flight reads") { fixture.runInIoContext([&](const TestFixture::Environment& env) { auto stream = createFiniteBytesReadableStream(env.js, 5, &counter); - auto adapter = kj::heap(env.js, env.context, stream.addRef()); + auto adapter = + kj::heap(env.js, env.context.getWeakRef(), stream.addRef()); auto buffer = kj::heapArray(2); @@ -1410,7 +1420,8 @@ KJ_TEST("KjAdapter read errored stream") { fixture.runInIoContext([&](const TestFixture::Environment& env) { auto stream = createErroredStream(env.js); - auto adapter = kj::heap(env.js, env.context, stream.addRef()); + auto adapter = + kj::heap(env.js, env.context.getWeakRef(), stream.addRef()); auto buffer = kj::heapArray(2); @@ -1440,7 +1451,8 @@ KJ_TEST("KjAdapter read closed stream") { fixture.runInIoContext([&](const TestFixture::Environment& env) { auto stream = createClosedStream(env.js); - auto adapter = kj::heap(env.js, env.context, stream.addRef()); + auto adapter = + kj::heap(env.js, env.context.getWeakRef(), stream.addRef()); auto buffer = kj::heapArray(2); @@ -1464,7 +1476,8 @@ KJ_TEST("KjAdapter pumpTo") { // which means the stream size must be <= bufferSize / 2. With bufferSize = 1024 // (for streams < 4096 bytes), we need total size <= 512 bytes. auto stream = createFiniteBytesReadableStream(env.js, 50); - auto adapter = kj::heap(env.js, env.context, stream.addRef()); + auto adapter = + kj::heap(env.js, env.context.getWeakRef(), stream.addRef()); return adapter->pumpTo(*writableSink, EndAfterPump::YES).attach(kj::mv(adapter)); }); @@ -1497,7 +1510,8 @@ KJ_TEST("KjAdapter pumpTo (no end)") { fixture.runInIoContext([&](const TestFixture::Environment& env) { // Use the same size constraint as the previous test. auto stream = createFiniteBytesReadableStream(env.js, 50); - auto adapter = kj::heap(env.js, env.context, stream.addRef()); + auto adapter = + kj::heap(env.js, env.context.getWeakRef(), stream.addRef()); return adapter->pumpTo(*writableSink, EndAfterPump::NO).attach(kj::mv(adapter)); }); @@ -1529,7 +1543,8 @@ KJ_TEST("KjAdapter pumpTo (errored)") { fixture.runInIoContext([&](const TestFixture::Environment& env) { auto stream = createErroredStream(env.js); - auto adapter = kj::heap(env.js, env.context, stream.addRef()); + auto adapter = + kj::heap(env.js, env.context.getWeakRef(), stream.addRef()); return env.context.waitForDeferredProxy(adapter->pumpTo(*writableSink, EndAfterPump::NO)) .then([]() -> kj::Promise { @@ -1550,7 +1565,8 @@ KJ_TEST("KjAdapter pumpTo (error sink)") { fixture.runInIoContext([&](const TestFixture::Environment& env) { auto stream = createFiniteBytesReadableStream(env.js, 1000); - auto adapter = kj::heap(env.js, env.context, stream.addRef()); + auto adapter = + kj::heap(env.js, env.context.getWeakRef(), stream.addRef()); return env.context.waitForDeferredProxy(adapter->pumpTo(*writableSink, EndAfterPump::NO)) .then([]() -> kj::Promise { @@ -1592,9 +1608,10 @@ KJ_TEST("KjAdapter MinReadPolicy IMMEDIATE behavior") { StreamQueuingStrategy{.highWaterMark = 0}); // Test IMMEDIATE policy - should return as soon as minBytes is satisfied - auto adapter = kj::heap(env.js, env.context, stream.addRef(), - ReadableSourceKjAdapter::Options{ - .minReadPolicy = ReadableSourceKjAdapter::MinReadPolicy::IMMEDIATE}); + auto adapter = + kj::heap(env.js, env.context.getWeakRef(), stream.addRef(), + ReadableSourceKjAdapter::Options{ + .minReadPolicy = ReadableSourceKjAdapter::MinReadPolicy::IMMEDIATE}); auto buffer = kj::heapArray(2048); @@ -1647,9 +1664,10 @@ KJ_TEST("KjAdapter MinReadPolicy OPPORTUNISTIC behavior") { StreamQueuingStrategy{.highWaterMark = 0}); // Test OPPORTUNISTIC policy - should try to fill buffer more completely - auto adapter = kj::heap(env.js, env.context, stream.addRef(), - ReadableSourceKjAdapter::Options{ - .minReadPolicy = ReadableSourceKjAdapter::MinReadPolicy::OPPORTUNISTIC}); + auto adapter = + kj::heap(env.js, env.context.getWeakRef(), stream.addRef(), + ReadableSourceKjAdapter::Options{ + .minReadPolicy = ReadableSourceKjAdapter::MinReadPolicy::OPPORTUNISTIC}); auto buffer = kj::heapArray(2048); @@ -1679,7 +1697,8 @@ KJ_TEST("KjAdapter readAllBytes") { fixture.runInIoContext([&](const TestFixture::Environment& env) -> kj::Promise { auto stream = createFiniteBytesReadableStream(env.js, 1024); - auto adapter = kj::heap(env.js, env.context, stream.addRef()); + auto adapter = + kj::heap(env.js, env.context.getWeakRef(), stream.addRef()); auto bytes = co_await adapter->readAllBytes(kj::maxValue).attach(kj::mv(adapter)); kj::FixedArray expected; expected.asPtr().first(1024).fill(97); // 'a' @@ -1706,7 +1725,8 @@ KJ_TEST("KjAdapter readAllBytes (limit exceeded)") { fixture.runInIoContext([&](const TestFixture::Environment& env) -> kj::Promise { auto stream = createFiniteBytesReadableStream(env.js, 1024); - auto adapter = kj::heap(env.js, env.context, stream.addRef()); + auto adapter = + kj::heap(env.js, env.context.getWeakRef(), stream.addRef()); try { co_await adapter->readAllBytes(100).attach(kj::mv(adapter)); KJ_FAIL_ASSERT("should have failed"); @@ -1725,7 +1745,8 @@ KJ_TEST("KjAdapter readAllText") { fixture.runInIoContext([&](const TestFixture::Environment& env) -> kj::Promise { auto stream = createFiniteBytesReadableStream(env.js, 2048); - auto adapter = kj::heap(env.js, env.context, stream.addRef()); + auto adapter = + kj::heap(env.js, env.context.getWeakRef(), stream.addRef()); auto text = co_await adapter->readAllText(kj::maxValue).attach(kj::mv(adapter)); kj::FixedArray expected; @@ -1753,7 +1774,8 @@ KJ_TEST("KjAdapter readAllText (limit exceeded)") { fixture.runInIoContext([&](const TestFixture::Environment& env) -> kj::Promise { auto stream = createFiniteBytesReadableStream(env.js, 1024); - auto adapter = kj::heap(env.js, env.context, stream.addRef()); + auto adapter = + kj::heap(env.js, env.context.getWeakRef(), stream.addRef()); try { co_await adapter->readAllText(100).attach(kj::mv(adapter)); KJ_FAIL_ASSERT("should have failed"); diff --git a/src/workerd/api/streams/readable-source-adapter.c++ b/src/workerd/api/streams/readable-source-adapter.c++ index 72a2817225b..13033b7993b 100644 --- a/src/workerd/api/streams/readable-source-adapter.c++ +++ b/src/workerd/api/streams/readable-source-adapter.c++ @@ -472,7 +472,7 @@ kj::Maybe ReadableStreamSourceJsAdapter::try // =============================================================================================== struct ReadableSourceKjAdapter::Active { - IoContext& ioContext; + kj::WeakRc ioContext; jsg::Ref stream; jsg::Ref reader; kj::Canceler canceler; @@ -526,7 +526,7 @@ struct ReadableSourceKjAdapter::Active { Canceled>; InnerState state; - Active(jsg::Lock& js, IoContext& ioContext, jsg::Ref stream); + Active(jsg::Lock& js, kj::WeakRc ioContext, jsg::Ref stream); KJ_DISALLOW_COPY_AND_MOVE(Active); ~Active() noexcept(false); @@ -675,8 +675,8 @@ kj::Maybe> copyFromSource( } // namespace ReadableSourceKjAdapter::Active::Active( - jsg::Lock& js, IoContext& ioContext, jsg::Ref stream) - : ioContext(ioContext), + jsg::Lock& js, kj::WeakRc ioContext, jsg::Ref stream) + : ioContext(kj::mv(ioContext)), stream(kj::mv(stream)), reader(initReader(js, this->stream)), state(InnerState::create()) {} @@ -696,9 +696,10 @@ void ReadableSourceKjAdapter::Active::cancel(kj::Exception reason) { // If the previous read indicated that it was the last read, then // the reader will have already been dropped. We do not need to // cancel it here. - ioContext.addTask( - ioContext.run([readable = kj::mv(stream), reader = kj::mv(reader), - exception = kj::mv(reason)](jsg::Lock& js, IoContext& ioContext) mutable { + auto& ctx = ioContext.assertLive(); + ctx.addTask( + ctx.run([readable = kj::mv(stream), reader = kj::mv(reader), exception = kj::mv(reason)]( + jsg::Lock& js, IoContext& ioContext) mutable { auto error = js.exceptionToJsValue(kj::mv(exception)); auto promise = reader->cancel(js, error.getHandle(js)); return ioContext.awaitJs(js, kj::mv(promise)); @@ -706,9 +707,11 @@ void ReadableSourceKjAdapter::Active::cancel(kj::Exception reason) { } } -ReadableSourceKjAdapter::ReadableSourceKjAdapter( - jsg::Lock& js, IoContext& ioContext, jsg::Ref stream, Options options) - : state(KjState::create(kj::heap(js, ioContext, kj::mv(stream)))), +ReadableSourceKjAdapter::ReadableSourceKjAdapter(jsg::Lock& js, + kj::WeakRc ioContext, + jsg::Ref stream, + Options options) + : state(KjState::create(kj::heap(js, kj::mv(ioContext), kj::mv(stream)))), options(options), selfRef( kj::rc>(kj::Badge{}, *this)) {} @@ -894,9 +897,10 @@ kj::Promise ReadableSourceKjAdapter::readImpl( // while we are in the promise chain. Instead, we capture a weak // reference to the adapter itself and check that we are still alive // and active before trying to update any state. - active.ioContext.run([context = kj::mv(context), self = selfRef.addRef(), - minReadPolicy = options.minReadPolicy](jsg::Lock& js, - IoContext& ioContext) mutable -> kj::Promise { + active.ioContext.assertLive().run( + [context = kj::mv(context), self = selfRef.addRef(), + minReadPolicy = options.minReadPolicy]( + jsg::Lock& js, IoContext& ioContext) mutable -> kj::Promise { // Perform the actual read. return ioContext.awaitJs(js, readInternal(js, kj::mv(context), minReadPolicy)) .then([self = kj::mv(self)](kj::Own context) mutable -> kj::Promise { @@ -1092,7 +1096,7 @@ kj::Promise ReadableSourceKjAdapter::pumpToImpl( // Initialize the pump by releasing the default reader and creating a DrainingReader. // This requires the isolate lock. - co_await active->ioContext.run( + co_await active->ioContext.assertLive().run( [&active, &maybeReader](jsg::Lock& js) mutable -> kj::Promise { // Release the existing reader's lock so we can create a DrainingReader. active->reader->releaseLock(js); @@ -1117,8 +1121,8 @@ kj::Promise ReadableSourceKjAdapter::pumpToImpl( // from the stream as possible each time we are holding the isolate lock // to minimize the number of times we need to re-enter the lock. DrainingReader* readerPtr = reader.get(); - DrainingReadResult result = - co_await active->ioContext.run([readerPtr](jsg::Lock& js, IoContext& ioContext) mutable { + DrainingReadResult result = co_await active->ioContext.assertLive().run( + [readerPtr](jsg::Lock& js, IoContext& ioContext) mutable { // Use a 256KB limit to allow periodic yielding to the event loop, // preventing a fast producer from monopolizing the thread. This limit // only affects subsequent pump iterations after the initial buffer drain. @@ -1157,7 +1161,7 @@ kj::Promise ReadableSourceKjAdapter::pumpToImpl( // If there was an error, cancel the reader and propagate the exception. KJ_IF_SOME(exception, pendingException) { DrainingReader* readerPtr = reader.get(); - co_await active->ioContext.run( + co_await active->ioContext.assertLive().run( [readerPtr, ex = exception.clone()](jsg::Lock& js, IoContext& ioContext) mutable { auto error = js.exceptionToJsValue(kj::mv(ex)); return ioContext.awaitJs(js, readerPtr->cancel(js, error.getHandle(js))); diff --git a/src/workerd/api/streams/readable-source-adapter.h b/src/workerd/api/streams/readable-source-adapter.h index 7bca8298cf2..503d61fc04c 100644 --- a/src/workerd/api/streams/readable-source-adapter.h +++ b/src/workerd/api/streams/readable-source-adapter.h @@ -314,7 +314,7 @@ class ReadableSourceKjAdapter final: public ReadableSource { }; ReadableSourceKjAdapter(jsg::Lock& js, - IoContext& ioContext, + kj::WeakRc ioContext, jsg::Ref stream, Options options = {.minReadPolicy = MinReadPolicy::OPPORTUNISTIC}); ~ReadableSourceKjAdapter() noexcept(false); diff --git a/src/workerd/api/streams/readable-source.c++ b/src/workerd/api/streams/readable-source.c++ index b763171f04c..eecf39f0e3a 100644 --- a/src/workerd/api/streams/readable-source.c++ +++ b/src/workerd/api/streams/readable-source.c++ @@ -637,18 +637,18 @@ class ReadableSourceImpl: public ReadableSource { // of the operations on the stream. class NoDeferredProxySource final: public ReadableSourceWrapper { public: - NoDeferredProxySource(kj::Own inner, IoContext& ioctx) + NoDeferredProxySource(kj::Own inner, kj::WeakRc ioctx) : ReadableSourceWrapper(kj::mv(inner)), - ioctx(ioctx) {} + ioctx(kj::mv(ioctx)) {} kj::Promise read(kj::ArrayPtr buffer, size_t minBytes = 1) override { - auto pending = ioctx.registerPendingEvent(); + auto pending = ioctx.assertLive().registerPendingEvent(); co_return co_await getInner().read(buffer, minBytes); } kj::Promise> pumpTo( WritableSink& output, EndAfterPump end = EndAfterPump::YES) override { - auto pending = ioctx.registerPendingEvent(); + auto pending = ioctx.assertLive().registerPendingEvent(); auto [proxyTask] = co_await getInner().pumpTo(output, end); co_await proxyTask; } @@ -656,13 +656,13 @@ class NoDeferredProxySource final: public ReadableSourceWrapper { Tee tee(size_t limit) override { auto tee = getInner().tee(limit); return Tee{ - .branch1 = kj::heap(kj::mv(tee.branch1), ioctx), - .branch2 = kj::heap(kj::mv(tee.branch2), ioctx), + .branch1 = kj::heap(kj::mv(tee.branch1), ioctx.clone()), + .branch2 = kj::heap(kj::mv(tee.branch2), ioctx.clone()), }; } private: - IoContext& ioctx; + kj::WeakRc ioctx; }; // A ReadableSource implementation that lazily wraps an innner Gzip or Brotli @@ -759,8 +759,8 @@ kj::Own newReadableSourceFromBytes( } kj::Own newIoContextWrappedReadableSource( - IoContext& ioctx, kj::Own inner) { - return kj::heap(kj::mv(inner), ioctx); + kj::WeakRc ioctx, kj::Own inner) { + return kj::heap(kj::mv(inner), kj::mv(ioctx)); } kj::Own newReadableSourceFromProducer( diff --git a/src/workerd/api/streams/readable-source.h b/src/workerd/api/streams/readable-source.h index 2b4804f725f..70516051ca7 100644 --- a/src/workerd/api/streams/readable-source.h +++ b/src/workerd/api/streams/readable-source.h @@ -4,6 +4,7 @@ #include #include +#include namespace kj { class AsyncInputStream; @@ -198,7 +199,7 @@ kj::Own newReadableSourceFromBytes( // Creates a ReadableSource that wraps the given source and prevents deferred proxying. kj::Own newIoContextWrappedReadableSource( - IoContext& ioctx, kj::Own inner); + kj::WeakRc ioctx, kj::Own inner); // Creates a ReadableSource that calls the given producer function to produce data // on each read (useful primarily for testing). diff --git a/src/workerd/api/streams/readable.c++ b/src/workerd/api/streams/readable.c++ index 2b29f1dd95b..d699babe3e3 100644 --- a/src/workerd/api/streams/readable.c++ +++ b/src/workerd/api/streams/readable.c++ @@ -15,8 +15,7 @@ namespace workerd::api { ReaderImpl::ReaderImpl(ReadableStreamController::Reader& reader) - : ioContext(tryGetIoContext()), - reader(reader), + : reader(reader), state(ReaderState::create()) {} ReaderImpl::~ReaderImpl() noexcept(false) { @@ -258,8 +257,6 @@ void ReadableStreamBYOBReader::visitForGc(jsg::GcVisitor& visitor) { // ====================================================================================== // DrainingReader implementation -DrainingReader::DrainingReader(): ioContext(tryGetIoContext()) {} - DrainingReader::~DrainingReader() noexcept(false) { KJ_IF_SOME(stream, state.tryGet()) { stream->getController().releaseReader(*this, kj::none); @@ -388,8 +385,7 @@ ReadableStream::ReadableStream(IoContext& ioContext, kj::Own controller) - : ioContext(tryGetIoContext()), - controller(kj::mv(controller)) { + : controller(kj::mv(controller)) { getController().setOwnerRef(PtrTarget::addWeakToThis()); } @@ -461,9 +457,9 @@ ReadableStream::Reader ReadableStream::getReader( jsg::Ref ReadableStream::values( jsg::Lock& js, jsg::Optional options) { static const auto defaultOptions = ValuesOptions{}; - return js.alloc(AsyncIteratorState{.ioContext = ioContext, - .reader = ReadableStreamDefaultReader::constructor(js, JSG_THIS), - .preventCancel = options.orDefault(defaultOptions).preventCancel.orDefault(false)}); + return js.alloc( + AsyncIteratorState{.reader = ReadableStreamDefaultReader::constructor(js, JSG_THIS), + .preventCancel = options.orDefault(defaultOptions).preventCancel.orDefault(false)}); } jsg::Ref ReadableStream::pipeThrough( @@ -634,9 +630,9 @@ namespace { // beyond the destruction of the IoContext, if it is being used for deferred proxying. class NoDeferredProxyReadableStream final: public ReadableStreamSource { public: - NoDeferredProxyReadableStream(kj::Own inner, IoContext& ioctx) + NoDeferredProxyReadableStream(kj::Own inner, kj::WeakRc ioctx) : inner(kj::mv(inner)), - ioctx(ioctx) {} + ioctx(kj::mv(ioctx)) {} kj::Promise tryRead(void* buffer, size_t minBytes, size_t maxBytes) override { return inner->tryRead(buffer, minBytes, maxBytes); @@ -647,7 +643,8 @@ class NoDeferredProxyReadableStream final: public ReadableStreamSource { // we use `ioctx.waitForDeferredProxy()`, which returns a single promise covering both parts // (and, importantly, registering pending events where needed). Then, we add a noop deferred // proxy to the end of that. - return addNoopDeferredProxy(ioctx.waitForDeferredProxy(inner->pumpTo(kj::mv(output), end))); + return addNoopDeferredProxy( + ioctx.assertLive().waitForDeferredProxy(inner->pumpTo(kj::mv(output), end))); } StreamEncoding getPreferredEncoding() override { @@ -665,15 +662,15 @@ class NoDeferredProxyReadableStream final: public ReadableStreamSource { kj::Maybe tryTee(uint64_t limit) override { return inner->tryTee(limit).map([&](Tee tee) { return Tee{.branches = { - kj::heap(kj::mv(tee.branches[0]), ioctx), - kj::heap(kj::mv(tee.branches[1]), ioctx), + kj::heap(kj::mv(tee.branches[0]), ioctx.clone()), + kj::heap(kj::mv(tee.branches[1]), ioctx.clone()), }}; }); } private: kj::Own inner; - IoContext& ioctx; + kj::WeakRc ioctx; }; } // namespace @@ -752,7 +749,8 @@ jsg::Ref ReadableStream::deserialize( kj::Own in = ioctx.getExternalPusher()->unwrapStream(rs.getStream()); return js.alloc(ioctx, - kj::heap(newSystemStream(kj::mv(in), encoding, ioctx), ioctx)); + kj::heap( + newSystemStream(kj::mv(in), encoding, ioctx), ioctx.getWeakRef())); } kj::StringPtr ReaderImpl::jsgGetMemoryName() const { diff --git a/src/workerd/api/streams/readable.h b/src/workerd/api/streams/readable.h index f0a30992725..e3679b6c8db 100644 --- a/src/workerd/api/streams/readable.h +++ b/src/workerd/api/streams/readable.h @@ -80,7 +80,7 @@ class ReaderImpl final { Closed, Released>; - kj::Maybe ioContext; + kj::WeakRc ioContext = IoContext::tryGetWeakRefForCurrent(); ReadableStreamController::Reader& reader; ReaderState state; @@ -226,7 +226,7 @@ class ReadableStreamBYOBReader: public jsg::Object, // This is intended for optimized pipe operations. class DrainingReader: public ReadableStreamController::Reader { public: - explicit DrainingReader(); + DrainingReader() = default; // Factory method to create and lock to a stream. Returns nullptr if stream is locked. static kj::Maybe> create(jsg::Lock& js, ReadableStream& stream); @@ -258,7 +258,7 @@ class DrainingReader: public ReadableStreamController::Reader { using Attached = jsg::Ref; struct Released {}; - kj::Maybe ioContext; + kj::WeakRc ioContext = IoContext::tryGetWeakRefForCurrent(); kj::OneOf state = Initial(); kj::Maybe>> closedPromise; }; @@ -267,7 +267,6 @@ class ReadableStream: public kj::PtrTarget, public jsg::Object { private: struct AsyncIteratorState { - kj::Maybe ioContext; jsg::Ref reader; bool preventCancel; }; @@ -477,7 +476,7 @@ class ReadableStream: public kj::PtrTarget, public jsg::Object { void visitForMemoryInfo(jsg::MemoryTracker& tracker) const; private: - kj::Maybe ioContext; + kj::WeakRc ioContext = IoContext::tryGetWeakRefForCurrent(); kj::Own controller; // Used to signal when this ReadableStream reads EOF. This signal is required for TCP sockets. diff --git a/src/workerd/api/streams/standard.c++ b/src/workerd/api/streams/standard.c++ index f66a544f5c9..00ce9bdbbc2 100644 --- a/src/workerd/api/streams/standard.c++ +++ b/src/workerd/api/streams/standard.c++ @@ -726,7 +726,7 @@ class ReadableStreamJsController final: public ReadableStreamController { KJ_DISALLOW_COPY_AND_MOVE(ReadableStreamJsController); - explicit ReadableStreamJsController(); + ReadableStreamJsController() = default; explicit ReadableStreamJsController(StreamStates::Closed closed); explicit ReadableStreamJsController(StreamStates::Errored errored); explicit ReadableStreamJsController(jsg::Lock& js, ValueReadable& consumer); @@ -813,7 +813,7 @@ class ReadableStreamJsController final: public ReadableStreamController { private: // If the stream was created within the scope of a request, we want to treat it as I/O // and make sure it is not advanced from the scope of a different request. - kj::Maybe ioContext; + kj::WeakRc ioContext = IoContext::tryGetWeakRefForCurrent(); kj::Weak owner; // Initial state before setup() is called. @@ -875,7 +875,7 @@ class WritableStreamJsController final: public WritableStreamController { using Controller = jsg::Ref; - explicit WritableStreamJsController(); + WritableStreamJsController() = default; explicit WritableStreamJsController(StreamStates::Closed closed); @@ -965,7 +965,7 @@ class WritableStreamJsController final: public WritableStreamController { private: jsg::Promise pipeLoop(jsg::Lock& js); - kj::Maybe ioContext; + kj::WeakRc ioContext = IoContext::tryGetWeakRefForCurrent(); kj::Weak owner; // Initial state before setup() is called. @@ -2236,8 +2236,7 @@ struct ByteReadable final: public kj::PtrTarget, ReadableStreamDefaultController::ReadableStreamDefaultController( UnderlyingSource underlyingSource, StreamQueuingStrategy queuingStrategy) - : ioContext(tryGetIoContext()), - impl(kj::mv(underlyingSource), kj::mv(queuingStrategy)) {} + : impl(kj::mv(underlyingSource), kj::mv(queuingStrategy)) {} kj::Maybe ReadableStreamDefaultController::getMaybeErrorState( jsg::Lock& js) { @@ -2352,8 +2351,7 @@ void ReadableStreamBYOBRequest::visitForGc(jsg::GcVisitor& visitor) { ReadableStreamBYOBRequest::ReadableStreamBYOBRequest(jsg::Lock& js, kj::Own readRequest, kj::Rc> controller) - : ioContext(tryGetIoContext()), - maybeImpl(Impl(js, kj::mv(readRequest), kj::mv(controller))) {} + : maybeImpl(Impl(js, kj::mv(readRequest), kj::mv(controller))) {} kj::Maybe ReadableStreamBYOBRequest::getAtLeast() { KJ_IF_SOME(impl, maybeImpl) { @@ -2502,7 +2500,6 @@ ReadableByteStreamController::ReadableByteStreamController( UnderlyingSource underlyingSource, StreamQueuingStrategy queuingStrategy) : weakSelf(kj::rc>( kj::Badge{}, *this)), - ioContext(tryGetIoContext()), impl(kj::mv(underlyingSource), kj::mv(queuingStrategy)) {} ReadableByteStreamController::~ReadableByteStreamController() noexcept(false) { @@ -2618,25 +2615,19 @@ kj::Own ReadableByteStreamController::getConsumer( // ====================================================================================== -ReadableStreamJsController::ReadableStreamJsController(): ioContext(tryGetIoContext()) {} - -ReadableStreamJsController::ReadableStreamJsController(StreamStates::Closed closed) - : ioContext(tryGetIoContext()) { +ReadableStreamJsController::ReadableStreamJsController(StreamStates::Closed closed) { state.transitionTo(); } -ReadableStreamJsController::ReadableStreamJsController(StreamStates::Errored errored) - : ioContext(tryGetIoContext()) { +ReadableStreamJsController::ReadableStreamJsController(StreamStates::Errored errored) { state.transitionTo(kj::mv(errored)); } -ReadableStreamJsController::ReadableStreamJsController(jsg::Lock& js, ValueReadable& consumer) - : ioContext(tryGetIoContext()) { +ReadableStreamJsController::ReadableStreamJsController(jsg::Lock& js, ValueReadable& consumer) { state.transitionTo>(consumer.clone(js, *this)); } -ReadableStreamJsController::ReadableStreamJsController(jsg::Lock& js, ByteReadable& consumer) - : ioContext(tryGetIoContext()) { +ReadableStreamJsController::ReadableStreamJsController(jsg::Lock& js, ByteReadable& consumer) { state.transitionTo>(consumer.clone(js, *this)); } @@ -3586,8 +3577,7 @@ kj::Promise> ReadableStreamJsController::pumpTo( WritableStreamDefaultController::WritableStreamDefaultController( jsg::Lock& js, kj::Weak owner, jsg::Ref abortSignal) - : ioContext(tryGetIoContext()), - impl(js, kj::mv(owner), kj::mv(abortSignal)) {} + : impl(js, kj::mv(owner), kj::mv(abortSignal)) {} jsg::Promise WritableStreamDefaultController::abort(jsg::Lock& js, jsg::JsValue reason) { return impl.abort(js, JSG_THIS, reason); @@ -3647,8 +3637,6 @@ WritableStreamDefaultController::~WritableStreamDefaultController() noexcept(fal } // ====================================================================================== -WritableStreamJsController::WritableStreamJsController(): ioContext(tryGetIoContext()) {} - WritableStreamJsController::~WritableStreamJsController() noexcept(false) { // Clear algorithms to break circular references during destruction KJ_IF_SOME(controller, state.tryGetUnsafe()) { @@ -3661,13 +3649,11 @@ WritableStreamJsController::~WritableStreamJsController() noexcept(false) { maybeAbortPromise = kj::none; } -WritableStreamJsController::WritableStreamJsController(StreamStates::Closed closed) - : ioContext(tryGetIoContext()) { +WritableStreamJsController::WritableStreamJsController(StreamStates::Closed closed) { state.transitionTo(); } -WritableStreamJsController::WritableStreamJsController(StreamStates::Errored errored) - : ioContext(tryGetIoContext()) { +WritableStreamJsController::WritableStreamJsController(StreamStates::Errored errored) { state.transitionTo(kj::mv(errored)); } @@ -4120,8 +4106,7 @@ void WritableStreamJsController::visitForGc(jsg::GcVisitor& visitor) { // ======================================================================================= TransformStreamDefaultController::TransformStreamDefaultController(jsg::Lock& js) - : ioContext(tryGetIoContext()), - startPromise(js.newPromiseAndResolver()) {} + : startPromise(js.newPromiseAndResolver()) {} kj::Maybe TransformStreamDefaultController::getDesiredSize() { KJ_IF_SOME(readableController, tryGetReadableController()) { diff --git a/src/workerd/api/streams/standard.h b/src/workerd/api/streams/standard.h index 7c869bc3b1d..1d836757b57 100644 --- a/src/workerd/api/streams/standard.h +++ b/src/workerd/api/streams/standard.h @@ -490,7 +490,7 @@ class ReadableStreamDefaultController: public jsg::Object { kj::Maybe getMaybeErrorState(jsg::Lock& js); private: - kj::Maybe ioContext; + kj::WeakRc ioContext = IoContext::tryGetWeakRefForCurrent(); ReadableImpl impl; void visitForGc(jsg::GcVisitor& visitor); @@ -560,7 +560,7 @@ class ReadableStreamBYOBRequest: public jsg::Object { void updateView(jsg::Lock& js); }; - kj::Maybe ioContext; + kj::WeakRc ioContext = IoContext::tryGetWeakRefForCurrent(); kj::Maybe maybeImpl; void visitForGc(jsg::GcVisitor& visitor); @@ -627,7 +627,7 @@ class ReadableByteStreamController: public jsg::Object { private: kj::Rc> weakSelf; - kj::Maybe ioContext; + kj::WeakRc ioContext = IoContext::tryGetWeakRefForCurrent(); ReadableImpl impl; kj::Maybe> maybeByobRequest; @@ -694,7 +694,7 @@ class WritableStreamDefaultController: public jsg::Object { void clearAlgorithms(); private: - kj::Maybe ioContext; + kj::WeakRc ioContext = IoContext::tryGetWeakRefForCurrent(); WritableImpl impl; void visitForGc(jsg::GcVisitor& visitor); @@ -785,7 +785,7 @@ class TransformStreamDefaultController: public jsg::Object { jsg::Promise performTransform(jsg::Lock& js, jsg::JsValue chunk); void setBackpressure(jsg::Lock& js, bool newBackpressure); - kj::Maybe ioContext; + kj::WeakRc ioContext = IoContext::tryGetWeakRefForCurrent(); jsg::PromiseResolverPair startPromise; kj::Maybe tryGetReadableController(); diff --git a/src/workerd/api/streams/writable-sink-adapter-test.c++ b/src/workerd/api/streams/writable-sink-adapter-test.c++ index 46a61c5cdab..cd48e2eb452 100644 --- a/src/workerd/api/streams/writable-sink-adapter-test.c++ +++ b/src/workerd/api/streams/writable-sink-adapter-test.c++ @@ -69,8 +69,8 @@ KJ_TEST("Basic construction with default options") { TestFixture fixture; fixture.runInIoContext([&](const TestFixture::Environment& env) { - auto sink = - newIoContextWrappedWritableSink(env.context, newWritableSink(newNullOutputStream())); + auto sink = newIoContextWrappedWritableSink( + env.context.getWeakRef(), newWritableSink(newNullOutputStream())); auto adapter = kj::heap(env.js, env.context, kj::mv(sink)); KJ_ASSERT(!adapter->isClosed(), "Adapter should not be closed upon construction"); @@ -92,8 +92,8 @@ KJ_TEST("Construction with custom highWaterMark option") { TestFixture fixture; fixture.runInIoContext([&](const TestFixture::Environment& env) { - auto sink = - newIoContextWrappedWritableSink(env.context, newWritableSink(newNullOutputStream())); + auto sink = newIoContextWrappedWritableSink( + env.context.getWeakRef(), newWritableSink(newNullOutputStream())); auto adapter = kj::heap(env.js, env.context, kj::mv(sink), WritableStreamSinkJsAdapter::Options{.highWaterMark = 100}); @@ -106,8 +106,8 @@ KJ_TEST("Construction with detachOnWrite=true option") { TestFixture fixture; fixture.runInIoContext([&](const TestFixture::Environment& env) { - auto sink = - newIoContextWrappedWritableSink(env.context, newWritableSink(newNullOutputStream())); + auto sink = newIoContextWrappedWritableSink( + env.context.getWeakRef(), newWritableSink(newNullOutputStream())); auto adapter = kj::heap(env.js, env.context, kj::mv(sink), WritableStreamSinkJsAdapter::Options{ .detachOnWrite = true, @@ -121,8 +121,8 @@ KJ_TEST("Construction with all custom options combined") { TestFixture fixture; fixture.runInIoContext([&](const TestFixture::Environment& env) { - auto sink = - newIoContextWrappedWritableSink(env.context, newWritableSink(newNullOutputStream())); + auto sink = newIoContextWrappedWritableSink( + env.context.getWeakRef(), newWritableSink(newNullOutputStream())); auto adapter = kj::heap(env.js, env.context, kj::mv(sink), WritableStreamSinkJsAdapter::Options{ .highWaterMark = 100, @@ -138,8 +138,8 @@ KJ_TEST("Basic end() operation completes successfully") { TestFixture fixture; fixture.runInIoContext([&](const TestFixture::Environment& env) { - auto sink = - newIoContextWrappedWritableSink(env.context, newWritableSink(newNullOutputStream())); + auto sink = newIoContextWrappedWritableSink( + env.context.getWeakRef(), newWritableSink(newNullOutputStream())); auto adapter = kj::heap(env.js, env.context, kj::mv(sink)); auto endPromise = adapter->end(env.js); @@ -194,8 +194,8 @@ KJ_TEST("Basic abort() operation") { TestFixture fixture; fixture.runInIoContext([&](const TestFixture::Environment& env) { - auto sink = - newIoContextWrappedWritableSink(env.context, newWritableSink(newNullOutputStream())); + auto sink = newIoContextWrappedWritableSink( + env.context.getWeakRef(), newWritableSink(newNullOutputStream())); auto adapter = kj::heap(env.js, env.context, kj::mv(sink)); adapter->abort(env.js, env.js.str("Abort reason"_kj)); @@ -234,8 +234,8 @@ KJ_TEST("Abort from closing state supersedes close") { TestFixture fixture; fixture.runInIoContext([&](const TestFixture::Environment& env) { - auto sink = - newIoContextWrappedWritableSink(env.context, newWritableSink(newNullOutputStream())); + auto sink = newIoContextWrappedWritableSink( + env.context.getWeakRef(), newWritableSink(newNullOutputStream())); auto adapter = kj::heap(env.js, env.context, kj::mv(sink)); auto endPromise = adapter->end(env.js); @@ -281,8 +281,8 @@ KJ_TEST("Abort from closed state") { TestFixture fixture; fixture.runInIoContext([&](const TestFixture::Environment& env) { - auto sink = - newIoContextWrappedWritableSink(env.context, newWritableSink(newNullOutputStream())); + auto sink = newIoContextWrappedWritableSink( + env.context.getWeakRef(), newWritableSink(newNullOutputStream())); auto adapter = kj::heap(env.js, env.context, kj::mv(sink)); auto endPromise = adapter->end(env.js); @@ -302,8 +302,8 @@ KJ_TEST("Abort rejects ready promise with abort reason") { TestFixture fixture; fixture.runInIoContext([&](const TestFixture::Environment& env) { - auto sink = - newIoContextWrappedWritableSink(env.context, newWritableSink(newNullOutputStream())); + auto sink = newIoContextWrappedWritableSink( + env.context.getWeakRef(), newWritableSink(newNullOutputStream())); auto adapter = kj::heap(env.js, env.context, kj::mv(sink), WritableStreamSinkJsAdapter::Options{ .highWaterMark = 1, @@ -836,8 +836,8 @@ KJ_TEST("Creating adapter and dropping it with pending operations") { TestFixture fixture; fixture.runInIoContext([&](const TestFixture::Environment& env) { - auto sink = - newIoContextWrappedWritableSink(env.context, newWritableSink(newNullOutputStream())); + auto sink = newIoContextWrappedWritableSink( + env.context.getWeakRef(), newWritableSink(newNullOutputStream())); auto adapter = kj::heap(env.js, env.context, kj::mv(sink)); adapter->write(env.js, env.js.str("data"_kj)); @@ -853,8 +853,8 @@ KJ_TEST("Dropping the IoContext with pending operations and using the adapter in kj::Maybe> adapter; fixture.runInIoContext([&](const TestFixture::Environment& env) { - auto sink = - newIoContextWrappedWritableSink(env.context, newWritableSink(newNullOutputStream())); + auto sink = newIoContextWrappedWritableSink( + env.context.getWeakRef(), newWritableSink(newNullOutputStream())); adapter = kj::heap(env.js, env.context, kj::mv(sink)); auto& adapterRef = *KJ_ASSERT_NONNULL(adapter); @@ -948,7 +948,8 @@ KJ_TEST("WritableStreamSinkKjAdapter construction") { fixture.runInIoContext([&](const TestFixture::Environment& env) { auto stream = createSimpleWritableStream(env.js, context); - auto adapter = kj::heap(env.js, env.context, kj::mv(stream)); + auto adapter = + kj::heap(env.js, env.context.getWeakRef(), kj::mv(stream)); }); } @@ -964,7 +965,8 @@ KJ_TEST("WritableStreamSinkKjAdapter construction with locked stream") { auto writer = stream->getWriter(env.js); try { - auto adapter = kj::heap(env.js, env.context, kj::mv(stream)); + auto adapter = + kj::heap(env.js, env.context.getWeakRef(), kj::mv(stream)); KJ_FAIL_ASSERT("Construction with locked stream should have thrown"); } catch (...) { auto ex = kj::getCaughtExceptionAsKj(); @@ -984,7 +986,8 @@ KJ_TEST("WritableStreamSinkKjAdapter construction with closed stream") { auto stream = createSimpleWritableStream(env.js, context); stream->close(env.js); - auto adapter = kj::heap(env.js, env.context, kj::mv(stream)); + auto adapter = + kj::heap(env.js, env.context.getWeakRef(), kj::mv(stream)); }); } @@ -999,7 +1002,8 @@ KJ_TEST("WritableStreamSinkKjAdapter construction with errored stream") { auto stream = createSimpleWritableStream(env.js, context); stream->abort(env.js, env.js.str("Abort reason"_kj)); - auto adapter = kj::heap(env.js, env.context, kj::mv(stream)); + auto adapter = + kj::heap(env.js, env.context.getWeakRef(), kj::mv(stream)); }); } @@ -1012,7 +1016,8 @@ KJ_TEST("WritableStreamSinkKjAdapter construction with immediate end") { fixture.runInIoContext([&](const TestFixture::Environment& env) { auto stream = createSimpleWritableStream(env.js, context); - auto adapter = kj::heap(env.js, env.context, kj::mv(stream)); + auto adapter = + kj::heap(env.js, env.context.getWeakRef(), kj::mv(stream)); return adapter->end().attach(kj::mv(adapter)); }); } @@ -1026,7 +1031,8 @@ KJ_TEST("WritableStreamSinkKjAdapter construction with immediate abort") { fixture.runInIoContext([&](const TestFixture::Environment& env) { auto stream = createSimpleWritableStream(env.js, context); - auto adapter = kj::heap(env.js, env.context, kj::mv(stream)); + auto adapter = + kj::heap(env.js, env.context.getWeakRef(), kj::mv(stream)); adapter->abort(KJ_EXCEPTION(DISCONNECTED, "Abort reason")); }); } @@ -1041,7 +1047,8 @@ KJ_TEST("WritableStreamSinkKjAdapter single write") { fixture.runInIoContext([&](const TestFixture::Environment& env) { auto stream = createSimpleWritableStream(env.js, context); - auto adapter = kj::heap(env.js, env.context, kj::mv(stream)); + auto adapter = + kj::heap(env.js, env.context.getWeakRef(), kj::mv(stream)); buffer.fill('a'); return adapter->write(buffer.asPtr()).then([&adapter = *adapter]() { @@ -1066,7 +1073,8 @@ KJ_TEST("WritableStreamSinkKjAdapter zero-length write") { fixture.runInIoContext([&](const TestFixture::Environment& env) { auto stream = createSimpleWritableStream(env.js, context); - auto adapter = kj::heap(env.js, env.context, kj::mv(stream)); + auto adapter = + kj::heap(env.js, env.context.getWeakRef(), kj::mv(stream)); return adapter->write(buffer).then([&adapter = *adapter]() { return adapter.end(); @@ -1089,7 +1097,8 @@ KJ_TEST("WritableStreamSinkKjAdapter concurrent writes forbidden") { try { fixture.runInIoContext([&](const TestFixture::Environment& env) { auto stream = createSimpleWritableStream(env.js, context); - auto adapter = kj::heap(env.js, env.context, kj::mv(stream)); + auto adapter = + kj::heap(env.js, env.context.getWeakRef(), kj::mv(stream)); buffer.asPtr().fill('a'); @@ -1115,7 +1124,8 @@ KJ_TEST("WritableStreamSinkKjAdapter write after close") { try { fixture.runInIoContext([&](const TestFixture::Environment& env) { auto stream = createSimpleWritableStream(env.js, context); - auto adapter = kj::heap(env.js, env.context, kj::mv(stream)); + auto adapter = + kj::heap(env.js, env.context.getWeakRef(), kj::mv(stream)); buffer.asPtr().fill('a'); @@ -1139,7 +1149,8 @@ KJ_TEST("WritableStreamSinkKjAdapter single errored") { fixture.runInIoContext([&](const TestFixture::Environment& env) { auto stream = createErroredStream(env.js); - auto adapter = kj::heap(env.js, env.context, kj::mv(stream)); + auto adapter = + kj::heap(env.js, env.context.getWeakRef(), kj::mv(stream)); buffer.fill('a'); return adapter->write(buffer.asPtr()) diff --git a/src/workerd/api/streams/writable-sink-adapter.c++ b/src/workerd/api/streams/writable-sink-adapter.c++ index 2a333970998..ee02072901f 100644 --- a/src/workerd/api/streams/writable-sink-adapter.c++ +++ b/src/workerd/api/streams/writable-sink-adapter.c++ @@ -146,7 +146,7 @@ WritableStreamSinkJsAdapter::WritableStreamSinkJsAdapter(jsg::Lock& js, : WritableStreamSinkJsAdapter(js, ioContext, newIoContextWrappedWritableSink( - ioContext, newEncodedWritableSink(encoding, kj::mv(stream))), + ioContext.getWeakRef(), newEncodedWritableSink(encoding, kj::mv(stream))), kj::mv(options)) {} WritableStreamSinkJsAdapter::~WritableStreamSinkJsAdapter() noexcept(false) { @@ -504,7 +504,7 @@ kj::Maybe WritableStreamSinkJsAdapt // ================================================================================================ struct WritableStreamSinkKjAdapter::Active { - IoContext& ioContext; + kj::WeakRc ioContext; jsg::Ref stream; jsg::Ref writer; kj::Canceler canceler; @@ -519,7 +519,7 @@ struct WritableStreamSinkKjAdapter::Active { // Prevent abort() from being called multiple times. bool aborted = false; - Active(jsg::Lock& js, IoContext& ioContext, jsg::Ref stream); + Active(jsg::Lock& js, kj::WeakRc ioContext, jsg::Ref stream); KJ_DISALLOW_COPY_AND_MOVE(Active); ~Active() noexcept(false); @@ -534,8 +534,8 @@ jsg::Ref initWriter(jsg::Lock& js, jsg::Ref stream) - : ioContext(ioContext), + jsg::Lock& js, kj::WeakRc ioContext, jsg::Ref stream) + : ioContext(kj::mv(ioContext)), stream(kj::mv(stream)), writer(initWriter(js, this->stream)) {} @@ -547,8 +547,9 @@ void WritableStreamSinkKjAdapter::Active::abort(kj::Exception reason) { if (aborted) return; aborted = true; canceler.cancel(reason.clone()); - ioContext.addTask(ioContext.run([writable = kj::mv(stream), writer = kj::mv(writer), - exception = reason.clone()](jsg::Lock& js) mutable { + auto& ctx = ioContext.assertLive(); + ctx.addTask(ctx.run([writable = kj::mv(stream), writer = kj::mv(writer), + exception = reason.clone()](jsg::Lock& js) mutable { auto& ioContext = IoContext::current(); auto error = js.exceptionToJsValue(kj::mv(exception)); auto promise = writer->abort(js, error.getHandle(js)); @@ -557,8 +558,8 @@ void WritableStreamSinkKjAdapter::Active::abort(kj::Exception reason) { } WritableStreamSinkKjAdapter::WritableStreamSinkKjAdapter( - jsg::Lock& js, IoContext& ioContext, jsg::Ref stream) - : state(KjState::create(kj::heap(js, ioContext, kj::mv(stream)))), + jsg::Lock& js, kj::WeakRc ioContext, jsg::Ref stream) + : state(KjState::create(kj::heap(js, kj::mv(ioContext), kj::mv(stream)))), selfRef(kj::rc>( kj::Badge{}, *this)) {} @@ -596,8 +597,9 @@ kj::Promise WritableStreamSinkKjAdapter::write( active.writePending = true; return active.canceler - .wrap(active.ioContext.run([self = selfRef.addRef(), writer = active.writer.addRef(), - pieces = pieces](jsg::Lock& js) mutable -> kj::Promise { + .wrap(active.ioContext.assertLive().run( + [self = selfRef.addRef(), writer = active.writer.addRef(), pieces = pieces]( + jsg::Lock& js) mutable -> kj::Promise { size_t totalAmount = 0; for (auto piece: pieces) { totalAmount += piece.size(); @@ -660,7 +662,7 @@ kj::Promise WritableStreamSinkKjAdapter::end() { } active.closePending = true; return active.canceler - .wrap(active.ioContext.run( + .wrap(active.ioContext.assertLive().run( [self = selfRef.addRef(), writer = active.writer.addRef()](jsg::Lock& js) mutable { auto promise = writer->close(js); return IoContext::current().awaitJs(js, kj::mv(promise)); diff --git a/src/workerd/api/streams/writable-sink-adapter.h b/src/workerd/api/streams/writable-sink-adapter.h index 5a1262577f6..e4c674f0585 100644 --- a/src/workerd/api/streams/writable-sink-adapter.h +++ b/src/workerd/api/streams/writable-sink-adapter.h @@ -405,7 +405,8 @@ class WritableStreamSinkJsAdapter final { // class WritableStreamSinkKjAdapter final: public WritableSink { public: - WritableStreamSinkKjAdapter(jsg::Lock& js, IoContext& ioContext, jsg::Ref stream); + WritableStreamSinkKjAdapter( + jsg::Lock& js, kj::WeakRc ioContext, jsg::Ref stream); ~WritableStreamSinkKjAdapter() noexcept(false); // Attempts to write the given buffer to the underlying stream. diff --git a/src/workerd/api/streams/writable-sink-test.c++ b/src/workerd/api/streams/writable-sink-test.c++ index ad2276be5b0..0bfdd76ece8 100644 --- a/src/workerd/api/streams/writable-sink-test.c++ +++ b/src/workerd/api/streams/writable-sink-test.c++ @@ -550,7 +550,7 @@ KJ_TEST("IoContext aware wrapper") { fixture.runInIoContext([&](const auto& environment) -> kj::Promise { IoContext& ioContext = environment.context; - auto wrapper = newIoContextWrappedWritableSink(ioContext, kj::mv(sink)); + auto wrapper = newIoContextWrappedWritableSink(ioContext.getWeakRef(), kj::mv(sink)); co_await wrapper->write("some data"_kjb); co_await wrapper->end(); }); diff --git a/src/workerd/api/streams/writable-sink.c++ b/src/workerd/api/streams/writable-sink.c++ index c6a1c7f8d8f..7dd4b49b637 100644 --- a/src/workerd/api/streams/writable-sink.c++ +++ b/src/workerd/api/streams/writable-sink.c++ @@ -234,36 +234,39 @@ class EncodedAsyncOutputStream final: public WritableSinkImpl { // A wrapper around a WritableSink that registers pending events with an IoContext. class IoContextWritableSinkWrapper: public WritableSinkWrapper { public: - IoContextWritableSinkWrapper(IoContext& ioContext, kj::Own inner) + IoContextWritableSinkWrapper(kj::WeakRc ioContext, kj::Own inner) : WritableSinkWrapper(kj::mv(inner)), - ioContext(ioContext) {} + ioContext(kj::mv(ioContext)) {} kj::Promise write(kj::ArrayPtr buffer) override { - auto pending = ioContext.registerPendingEvent(); - KJ_IF_SOME(p, ioContext.waitForOutputLocksIfNecessary()) { + auto& ctx = ioContext.assertLive(); + auto pending = ctx.registerPendingEvent(); + KJ_IF_SOME(p, ctx.waitForOutputLocksIfNecessary()) { co_await p; } co_await getInner().write(buffer); } kj::Promise write(kj::ArrayPtr> pieces) override { - auto pending = ioContext.registerPendingEvent(); - KJ_IF_SOME(p, ioContext.waitForOutputLocksIfNecessary()) { + auto& ctx = ioContext.assertLive(); + auto pending = ctx.registerPendingEvent(); + KJ_IF_SOME(p, ctx.waitForOutputLocksIfNecessary()) { co_await p; } co_await getInner().write(pieces); } kj::Promise end() override { - auto pending = ioContext.registerPendingEvent(); - KJ_IF_SOME(p, ioContext.waitForOutputLocksIfNecessary()) { + auto& ctx = ioContext.assertLive(); + auto pending = ctx.registerPendingEvent(); + KJ_IF_SOME(p, ctx.waitForOutputLocksIfNecessary()) { co_await p; } co_await getInner().end(); } private: - IoContext& ioContext; + kj::WeakRc ioContext; }; } // namespace @@ -289,8 +292,8 @@ kj::Own newEncodedWritableSink( } kj::Own newIoContextWrappedWritableSink( - IoContext& ioContext, kj::Own inner) { - return kj::heap(ioContext, kj::mv(inner)); + kj::WeakRc ioContext, kj::Own inner) { + return kj::heap(kj::mv(ioContext), kj::mv(inner)); } } // namespace workerd::api::streams diff --git a/src/workerd/api/streams/writable-sink.h b/src/workerd/api/streams/writable-sink.h index b518009c1e7..f0cc7e65340 100644 --- a/src/workerd/api/streams/writable-sink.h +++ b/src/workerd/api/streams/writable-sink.h @@ -3,6 +3,7 @@ #include #include +#include namespace kj { class AsyncOutputStream; @@ -136,7 +137,7 @@ kj::Own newEncodedWritableSink( // Wraps a WritableSink such that each write()/end() call on the returned sink will // register as a pending event on the IoContext. kj::Own newIoContextWrappedWritableSink( - IoContext& ioContext, kj::Own inner); + kj::WeakRc ioContext, kj::Own inner); } // namespace api::streams } // namespace workerd diff --git a/src/workerd/api/streams/writable.c++ b/src/workerd/api/streams/writable.c++ index 61642d954d1..bf0b6a96194 100644 --- a/src/workerd/api/streams/writable.c++ +++ b/src/workerd/api/streams/writable.c++ @@ -10,9 +10,7 @@ namespace workerd::api { -WritableStreamDefaultWriter::WritableStreamDefaultWriter() - : ioContext(tryGetIoContext()), - state(WriterState::create()) {} +WritableStreamDefaultWriter::WritableStreamDefaultWriter(): state(WriterState::create()) {} WritableStreamDefaultWriter::~WritableStreamDefaultWriter() noexcept(false) { KJ_IF_SOME(attached, state.tryGetActiveUnsafe()) { @@ -185,8 +183,7 @@ WritableStream::WritableStream(IoContext& ioContext, kj::mv(maybeClosureWaitable))) {} WritableStream::WritableStream(kj::Own controller) - : ioContext(tryGetIoContext()), - controller(kj::mv(controller)) { + : controller(kj::mv(controller)) { getController().setOwnerRef(PtrTarget::addWeakToThis()); } @@ -334,8 +331,9 @@ class WritableStreamRpcAdapter final: public capnp::ExplicitEndOutputStream { // a lot slower class WritableStreamJsRpcAdapter final: public capnp::ExplicitEndOutputStream { public: - WritableStreamJsRpcAdapter(IoContext& context, jsg::Ref writer) - : context(context), + WritableStreamJsRpcAdapter( + kj::WeakRc context, jsg::Ref writer) + : context(kj::mv(context)), writer(kj::mv(writer)) {} ~WritableStreamJsRpcAdapter() noexcept(false) { @@ -365,8 +363,9 @@ class WritableStreamJsRpcAdapter final: public capnp::ExplicitEndOutputStream { // hopefully improve the situation here. if (!ended) { KJ_IF_SOME(writer, this->writer) { - context.addTask(context.run([writer = kj::mv(writer), exception = cancellationException()]( - Worker::Lock& lock) mutable { + auto& ctx = context.assertLive(); + ctx.addTask(ctx.run([writer = kj::mv(writer), exception = cancellationException()]( + Worker::Lock& lock) mutable { jsg::Lock& js = lock; auto ex = js.exceptionToJsValue(kj::mv(exception)); return IoContext::current().awaitJs(lock, writer->abort(lock, ex.getHandle(js))); @@ -389,8 +388,8 @@ class WritableStreamJsRpcAdapter final: public capnp::ExplicitEndOutputStream { } auto w = kj::mv(obj.writer); KJ_IF_SOME(writer, w) { - obj.context.addTask( - obj.context.run([writer = kj::mv(writer), exception = cancellationException()]( + auto& ctx = obj.context.assertLive(); + ctx.addTask(ctx.run([writer = kj::mv(writer), exception = cancellationException()]( Worker::Lock& lock) mutable { jsg::Lock& js = lock; auto ex = js.exceptionToJsValue(kj::mv(exception)); @@ -406,10 +405,10 @@ class WritableStreamJsRpcAdapter final: public capnp::ExplicitEndOutputStream { return KJ_EXCEPTION(FAILED, "Write after stream has been closed."); } if (buffer == nullptr) return kj::READY_NOW; - return canceler.wrap(context.run([this, buffer](Worker::Lock& lock) mutable { + return canceler.wrap(context.assertLive().run([this, buffer](Worker::Lock& lock) mutable { auto& writer = getInner(); auto ab = jsg::JsArrayBuffer::create(lock, buffer); - return context.awaitJs(lock, writer.write(lock, jsg::JsValue(ab))); + return context.assertLive().awaitJs(lock, writer.write(lock, jsg::JsValue(ab))); })); } @@ -422,7 +421,8 @@ class WritableStreamJsRpcAdapter final: public capnp::ExplicitEndOutputStream { amount += piece.size(); } if (amount == 0) return kj::READY_NOW; - return canceler.wrap(context.run([this, amount, pieces](Worker::Lock& lock) mutable { + return canceler.wrap( + context.assertLive().run([this, amount, pieces](Worker::Lock& lock) mutable { auto& writer = getInner(); // Sadly, we have to allocate and copy here. Our received set of buffers are only // guaranteed to live until the returned promise is resolved, but the application code @@ -437,7 +437,7 @@ class WritableStreamJsRpcAdapter final: public capnp::ExplicitEndOutputStream { ptr.write(piece); } - return context.awaitJs(lock, writer.write(lock, jsg::JsValue(ab))); + return context.assertLive().awaitJs(lock, writer.write(lock, jsg::JsValue(ab))); })); } @@ -468,13 +468,13 @@ class WritableStreamJsRpcAdapter final: public capnp::ExplicitEndOutputStream { return KJ_EXCEPTION(FAILED, "End after stream has been closed."); } ended = true; - return canceler.wrap(context.run([this](Worker::Lock& lock) mutable { - return context.awaitJs(lock, getInner().close(lock)); + return canceler.wrap(context.assertLive().run([this](Worker::Lock& lock) mutable { + return context.assertLive().awaitJs(lock, getInner().close(lock)); })); } private: - IoContext& context; + kj::WeakRc context; kj::Maybe> writer; kj::Canceler canceler; kj::Own> doneFulfiller; @@ -538,7 +538,7 @@ void WritableStream::serialize(jsg::Lock& js, jsg::Serializer& serializer) { // NOTE: We're counting on `getWriter()` to check that the stream is not locked and other // common checks. It's important we don't modify the WritableStream before this call. - auto wrapper = kj::heap(ioctx, getWriter(js)); + auto wrapper = kj::heap(ioctx.getWeakRef(), getWriter(js)); // Make sure this stream will be revoked if the IoContext ends. ioctx.addTask(wrapper->waitForCompletionOrRevoke().attach(ioctx.registerPendingEvent())); diff --git a/src/workerd/api/streams/writable.h b/src/workerd/api/streams/writable.h index 0448121e7b5..c78b8f6d527 100644 --- a/src/workerd/api/streams/writable.h +++ b/src/workerd/api/streams/writable.h @@ -119,7 +119,7 @@ class WritableStreamDefaultWriter: public jsg::Object, public WritableStreamCont Closed, Released>; - kj::Maybe ioContext; + kj::WeakRc ioContext = IoContext::tryGetWeakRefForCurrent(); WriterState state; inline void assertAttachedOrTerminal() const { @@ -207,7 +207,7 @@ class WritableStream: public jsg::Object, public kj::PtrTarget { void visitForMemoryInfo(jsg::MemoryTracker& tracker) const; private: - kj::Maybe ioContext; + kj::WeakRc ioContext = IoContext::tryGetWeakRefForCurrent(); kj::Own controller; void visitForGc(jsg::GcVisitor& visitor); From 66dfe4144f490cd7ab5983bb23aaa4ddf52661b9 Mon Sep 17 00:00:00 2001 From: Logan Gatlin Date: Wed, 15 Jul 2026 21:05:56 +0000 Subject: [PATCH 097/101] Fix getBuiltinModule to return default export for node built-ins in new module registry * Update getBuiltinModule test expectations to default-export semantics * Fix getBuiltinModule to return default export for node built-ins in new module registry See merge request cloudflare/ew/workerd!495 --- src/workerd/api/node/process.c++ | 6 +++++ .../process-getbuiltin-newmodreg-test.js | 23 ++++++++++++++++--- 2 files changed, 26 insertions(+), 3 deletions(-) diff --git a/src/workerd/api/node/process.c++ b/src/workerd/api/node/process.c++ index 222b2928a0c..eb6844facd1 100644 --- a/src/workerd/api/node/process.c++ +++ b/src/workerd/api/node/process.c++ @@ -25,6 +25,12 @@ jsg::JsValue ProcessModule::getBuiltinModule(jsg::Lock& js, kj::String specifier if (FeatureFlags::get(js).getNewModuleRegistry()) { KJ_IF_SOME(mod, js.resolvePublicBuiltinModule(specifier)) { + // resolvePublicBuiltinModule returns the module namespace. For Node.js + // modules we return the default export (matching require() and the legacy + // registry path below); for other built-ins we return the namespace. + if (isNode) { + return mod.get(js, "default"_kj); + } return mod; } return js.undefined(); diff --git a/src/workerd/api/node/tests/process-getbuiltin-newmodreg-test.js b/src/workerd/api/node/tests/process-getbuiltin-newmodreg-test.js index ca6fd1e6d5a..9c2b188824e 100644 --- a/src/workerd/api/node/tests/process-getbuiltin-newmodreg-test.js +++ b/src/workerd/api/node/tests/process-getbuiltin-newmodreg-test.js @@ -1,14 +1,31 @@ // Copyright (c) 2026 Cloudflare, Inc. // Licensed under the Apache 2.0 license found in the LICENSE file or at: // https://opensource.org/licenses/Apache-2.0 -import { strictEqual } from 'node:assert'; +import { strictEqual, ok } from 'node:assert'; +import processDefault from 'node:process'; export const getBuiltinModulePublicBuiltins = { test() { - // process.getBuiltinModule must be able to resolve known public built-ins. + // process.getBuiltinModule must resolve known public built-ins to their + // default export (the module value), so the type is whatever the module + // exports -- e.g. node:assert's default export is the assert function. strictEqual(typeof process.getBuiltinModule('node:process'), 'object'); strictEqual(typeof process.getBuiltinModule('node:buffer'), 'object'); - strictEqual(typeof process.getBuiltinModule('node:assert'), 'object'); + strictEqual(typeof process.getBuiltinModule('node:assert'), 'function'); + }, +}; + +export const getBuiltinModuleReturnsDefaultExport = { + test() { + // For Node.js modules, getBuiltinModule must return the default export (the + // module value), not the ES module namespace object. Regression test: the + // new module registry path previously returned the namespace, so it was not + // identical to the default import and carried a `default` property. + const gbm = process.getBuiltinModule('node:process'); + strictEqual(gbm, processDefault); + strictEqual(gbm, globalThis.process); + strictEqual(typeof gbm.nextTick, 'function'); + ok(!('default' in gbm)); }, }; From a08757eee366d818f831c3184c1ab4ee07e3d361 Mon Sep 17 00:00:00 2001 From: Logan Gatlin Date: Wed, 15 Jul 2026 21:06:42 +0000 Subject: [PATCH 098/101] Ignore query/fragment when redirecting node:process in new module registry * Ignore query/fragment when redirecting node:process in new module registry See merge request cloudflare/ew/workerd!496 --- .../api/tests/new-module-registry-test.js | 16 ++++++++++++++++ src/workerd/jsg/modules-new.c++ | 15 ++++++++++----- 2 files changed, 26 insertions(+), 5 deletions(-) diff --git a/src/workerd/api/tests/new-module-registry-test.js b/src/workerd/api/tests/new-module-registry-test.js index 2d464b91c88..670d64749e0 100644 --- a/src/workerd/api/tests/new-module-registry-test.js +++ b/src/workerd/api/tests/new-module-registry-test.js @@ -494,6 +494,22 @@ export const processRedirectAcrossResolutionRoutes = { }, }; +// Regression test: the node:process redirect ignores any query/fragment, just +// like every other built-in (which is resolved with IGNORE_SEARCH | +// IGNORE_FRAGMENTS). maybeRedirectNodeProcess previously matched the full href +// exactly, so `import('node:process?foo')` failed with +// "Module not found: node:process?foo" while e.g. `import('node:assert?foo')` +// resolved. See maybeRedirectNodeProcess in jsg/modules-new.c++. +export const processRedirectIgnoresQueryAndFragment = { + async test() { + const withQuery = await import('node:process?foo=1'); + const withFragment = await import('node:process#bar'); + strictEqual(typeof withQuery.default.nextTick, 'function'); + strictEqual(withQuery.default, processStatic); + strictEqual(withFragment.default, processStatic); + }, +}; + // TODO(now): Tests // * [x] Include tests for all known module types // * [x] ESM diff --git a/src/workerd/jsg/modules-new.c++ b/src/workerd/jsg/modules-new.c++ index 6dfc7d0570e..fa79c29d0e0 100644 --- a/src/workerd/jsg/modules-new.c++ +++ b/src/workerd/jsg/modules-new.c++ @@ -29,8 +29,13 @@ kj::Maybe checkModule(const ResolveContext& context, const Module // If the specifier is "node:process", returns the appropriate internal module // URL based on the enable_nodejs_process_v2 flag. Otherwise returns kj::none. -kj::Maybe maybeRedirectNodeProcess(Lock& js, kj::ArrayPtr spec) { - if (spec == "node:process"_kjb.asChars()) { +// Any query/fragment is ignored so that e.g. "node:process?foo" and +// "node:process#bar" redirect the same as "node:process", matching how all +// other built-ins are resolved (see IGNORE_SEARCH | IGNORE_FRAGMENTS below). +kj::Maybe maybeRedirectNodeProcess(Lock& js, const Url& spec) { + auto normalized = + spec.clone(Url::EquivalenceOption::IGNORE_SEARCH | Url::EquivalenceOption::IGNORE_FRAGMENTS); + if (normalized.getHref() == "node:process"_kjb.asChars()) { static const auto publicProcess = "node-internal:public_process"_url; static const auto legacyProcess = "node-internal:legacy_process"_url; return isNodeJsProcessV2Enabled(js) ? publicProcess : legacyProcess; @@ -458,7 +463,7 @@ class IsolateModuleRegistry final { // apply that redirect here, as a last resort, so that a worker bundle module // that intentionally shadows "node:process" (exactly as it could for any other // built-in) gets first crack at resolving it above. - KJ_IF_SOME(processUrl, maybeRedirectNodeProcess(js, context.normalizedSpecifier.getHref())) { + KJ_IF_SOME(processUrl, maybeRedirectNodeProcess(js, context.normalizedSpecifier)) { auto processSpec = kj::str(processUrl.getHref()); ResolveContext processContext = { .type = ResolveContext::Type::BUILTIN_ONLY, @@ -588,7 +593,7 @@ class IsolateModuleRegistry final { // apply that redirect here, as a last resort, so that a worker bundle module // that intentionally shadows "node:process" gets first crack at resolving it // above. - KJ_IF_SOME(processUrl, maybeRedirectNodeProcess(js, normalizedSpecifier.getHref())) { + KJ_IF_SOME(processUrl, maybeRedirectNodeProcess(js, normalizedSpecifier)) { auto processSpec = kj::str(processUrl.getHref()); ResolveContext processContext = { .type = ResolveContext::Type::BUILTIN_ONLY, @@ -794,7 +799,7 @@ class IsolateModuleRegistry final { // apply that redirect here, as a last resort, so that a worker bundle module // that intentionally shadows "node:process" gets first crack at resolving it // above. - KJ_IF_SOME(processUrl, maybeRedirectNodeProcess(js, context.normalizedSpecifier.getHref())) { + KJ_IF_SOME(processUrl, maybeRedirectNodeProcess(js, context.normalizedSpecifier)) { ResolveContext newContext{ .type = ResolveContext::Type::BUILTIN_ONLY, .source = context.source, From 06ba249e615711b0e8f8895ff117c2d3e36234f7 Mon Sep 17 00:00:00 2001 From: Logan Gatlin Date: Wed, 15 Jul 2026 21:18:51 +0000 Subject: [PATCH 099/101] Reject require() of a top-level-await module even when already evaluated * Reject require() of top-level-await module even when already evaluated The synchronous require() path checked RequireOption::NO_TOP_LEVEL_AWAIT only after the early-return that hands back the cached namespace for a module that is already kEvaluated/kEvaluating. So require() of an ESM that uses top-level await threw ERR_REQUIRE_ASYNC_MODULE-style only if the module had not already been evaluated by an earlier import; otherwise it silently returned the cached namespace. This made require()'s behavior order-dependent and inconsistent with Node.js. Perform the top-level-await check inside the already-evaluated branch as well (the module is instantiated there, so IsGraphAsync() is valid). Also strengthen the regression test to evaluate the module via import() before require()ing it. See merge request cloudflare/ew/workerd!498 --- src/workerd/api/tests/new-module-registry-test.js | 6 +++++- src/workerd/jsg/modules-new.c++ | 14 ++++++++++++-- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/src/workerd/api/tests/new-module-registry-test.js b/src/workerd/api/tests/new-module-registry-test.js index 2d464b91c88..2215cd01a6c 100644 --- a/src/workerd/api/tests/new-module-registry-test.js +++ b/src/workerd/api/tests/new-module-registry-test.js @@ -199,7 +199,11 @@ await rejects(import('cjs6'), { message: /^Top-level await is not supported/, }); -// Cannot directly require an ESM with top-level await either. +// Cannot directly require an ESM with top-level await either. This must hold +// even after the module has already been fully evaluated by a prior import: +// require()'s top-level-await rejection must not depend on evaluation order +// (regression -- the already-evaluated early-return used to skip the check). +await import('tla'); throws(() => myRequire('tla'), { message: /^Top-level await is not supported/, }); diff --git a/src/workerd/jsg/modules-new.c++ b/src/workerd/jsg/modules-new.c++ index 6dfc7d0570e..be1c3de3b08 100644 --- a/src/workerd/jsg/modules-new.c++ +++ b/src/workerd/jsg/modules-new.c++ @@ -279,8 +279,7 @@ class SyntheticModule final: public Module { exports[n++] = js.strIntern(exp); } return v8::Module::CreateSyntheticModule(js.v8Isolate, js.str(id().getHref()), - std::span>(exports.data(), exports.size()), - evaluationSteps); + std::span>(exports.data(), exports.size()), evaluationSteps); } private: @@ -709,6 +708,17 @@ class IsolateModuleRegistry final { // to a degree. Just like in Node.js, however, such circular dependencies // can still be problematic depending on how they are used. if (status == v8::Module::kEvaluated || status == v8::Module::kEvaluating) { + // require() must reject a module that uses top-level await (matching + // Node.js require(esm), which throws ERR_REQUIRE_ASYNC_MODULE) even when + // the module has already been evaluated by a prior import. Without this + // check here -- before we return the cached namespace -- whether + // require() throws would depend on evaluation order (it would throw only + // if nothing had imported the module first). The module is already + // instantiated at this point, so IsGraphAsync() is valid to query. + if ((option & RequireOption::NO_TOP_LEVEL_AWAIT) == RequireOption::NO_TOP_LEVEL_AWAIT) { + JSG_REQUIRE(!module->IsGraphAsync(), Error, + "Top-level await is not supported in this context for module: ", id); + } return maybeUnwrapDefault(js, module, moduleDef, option); } From 3ab9ffe3fcfb6653e9105a64a2be1234f68b0ef1 Mon Sep 17 00:00:00 2001 From: Logan Gatlin Date: Wed, 15 Jul 2026 21:22:38 +0000 Subject: [PATCH 100/101] Fix normalizePathEncoding dropping encoded slash after a non-%2f escape * Fix normalizePathEncoding dropping encoded slash after a non-%2f escape normalizePathEncoding's findNext helper only inspected the first "%2" occurrence in the pathname. If that first "%2" was not "%2f"/"%2F" (e.g. "%25", "%20", "%2E"), it returned kj::none and gave up, so a later "%2f" in the same chunk was percent-decoded into a real '/'. This silently turned an in-segment encoded slash into a path separator, changing the resolved module path. It also read input[pos+2] without a bounds check, which is out of range when the pathname ends in a bare "%2". Scan all "%2" occurrences and bounds-check the third character. See merge request cloudflare/ew/workerd!497 --- src/workerd/jsg/url-test.c++ | 26 ++++++++++++++++++++++++++ src/workerd/jsg/url.c++ | 12 +++++++++--- 2 files changed, 35 insertions(+), 3 deletions(-) diff --git a/src/workerd/jsg/url-test.c++ b/src/workerd/jsg/url-test.c++ index 763ebe56ca5..f1249beb9fe 100644 --- a/src/workerd/jsg/url-test.c++ +++ b/src/workerd/jsg/url-test.c++ @@ -1512,6 +1512,32 @@ KJ_TEST("Normalize path for comparison and cloning") { auto url9 = "file:///foo%2f%2F/bar"_url; url9 = url9.clone(Url::EquivalenceOption::NORMALIZE_PATH); KJ_ASSERT(url9.getHref() == "file:///foo%2F%2F/bar"_kj); + + // Regression: an encoded slash that is preceded by some OTHER "%2X" escape + // (here "%20" = space) must still be preserved. Previously + // normalizePathEncoding only inspected the first "%2" occurrence and, finding + // it was not "%2f", decoded the whole segment -- turning the later %2F into a + // real '/' (i.e. "file:///a%20%2Fb" would become "file:///a%20/b"). + auto url10 = "file:///a%20%2Fb"_url; + url10 = url10.clone(Url::EquivalenceOption::NORMALIZE_PATH); + KJ_ASSERT(url10.getHref() == "file:///a%20%2Fb"_kj); + + // Same class with a "%2E" ('.') decoy before the encoded slash. + auto url11 = "file:///p%2Eq%2Fr"_url; + url11 = url11.clone(Url::EquivalenceOption::NORMALIZE_PATH); + KJ_ASSERT(url11.getHref() == "file:///p.q%2Fr"_kj); + + // Multiple encoded slashes trailing a decoy: each must survive. + auto url12 = "file:///a%20%2Fb%2Fc"_url; + url12 = url12.clone(Url::EquivalenceOption::NORMALIZE_PATH); + KJ_ASSERT(url12.getHref() == "file:///a%20%2Fb%2Fc"_kj); + + // A pathname ending in a bare "%2" must not read past the end of the buffer. + // Normalization must be safe and idempotent (the exact canonical form of the + // dangling escape is up to the URL parser, so we only assert stability). + auto url13 = "file:///foo%2"_url.clone(Url::EquivalenceOption::NORMALIZE_PATH); + auto url13b = url13.clone(Url::EquivalenceOption::NORMALIZE_PATH); + KJ_ASSERT(url13.getHref() == url13b.getHref()); } // Regression test for AUTOVULN-CLOUDFLARE-WORKERD-387: deeply nested non-capturing groups in a diff --git a/src/workerd/jsg/url.c++ b/src/workerd/jsg/url.c++ index 9b608a10dee..6c23eb0e18b 100644 --- a/src/workerd/jsg/url.c++ +++ b/src/workerd/jsg/url.c++ @@ -60,9 +60,15 @@ kj::Array normalizePathEncoding(kj::ArrayPtr pathname) { // re-encode the pieces and then join them back together with %2F. static constexpr auto findNext = [](std::string_view input) -> kj::Maybe { - size_t pos = input.find("%2", 0); - if (pos != std::string_view::npos) { - if (input[pos + 2] == 'f' || input[pos + 2] == 'F') { + // Scan for the next "%2f"/"%2F". We must keep looking past any "%2X" that is + // not an encoded slash (e.g. "%25", "%20", "%2E"): stopping at the first + // "%2" occurrence would leave a later "%2f" to be percent-decoded into a + // real '/', silently turning an in-segment encoded slash into a path + // separator. Also guard against reading past the end when the input ends in + // a bare "%2" (no third character). + for (size_t pos = input.find("%2", 0); pos != std::string_view::npos; + pos = input.find("%2", pos + 2)) { + if (pos + 2 < input.size() && (input[pos + 2] == 'f' || input[pos + 2] == 'F')) { return pos; } } From 281677dc8909e140033f22a2531dabcdd493f3f4 Mon Sep 17 00:00:00 2001 From: Mike Aizatsky Date: Thu, 16 Jul 2026 08:55:38 -0700 Subject: [PATCH 101/101] just generate-types --- .../experimental/index.d.ts | 1 + .../generated-snapshot/experimental/index.ts | 1 + types/generated-snapshot/index.d.ts | 376 +++++++++++++++++- types/generated-snapshot/index.ts | 376 +++++++++++++++++- 4 files changed, 752 insertions(+), 2 deletions(-) diff --git a/types/generated-snapshot/experimental/index.d.ts b/types/generated-snapshot/experimental/index.d.ts index fbfab1eed84..f0a963ce096 100755 --- a/types/generated-snapshot/experimental/index.d.ts +++ b/types/generated-snapshot/experimental/index.d.ts @@ -503,6 +503,7 @@ interface ExecutionContext { readonly override?: string; }; readonly access?: CloudflareAccessContext; + mapVirtualHost(fetcher: Fetcher, port: number): string; tracing: Tracing; abort(reason?: any): void; } diff --git a/types/generated-snapshot/experimental/index.ts b/types/generated-snapshot/experimental/index.ts index 96ff8a442ed..d083717e150 100755 --- a/types/generated-snapshot/experimental/index.ts +++ b/types/generated-snapshot/experimental/index.ts @@ -505,6 +505,7 @@ export interface ExecutionContext { readonly override?: string; }; readonly access?: CloudflareAccessContext; + mapVirtualHost(fetcher: Fetcher, port: number): string; tracing: Tracing; abort(reason?: any): void; } diff --git a/types/generated-snapshot/index.d.ts b/types/generated-snapshot/index.d.ts index 9b7e909ddef..0b8560a9f2a 100755 --- a/types/generated-snapshot/index.d.ts +++ b/types/generated-snapshot/index.d.ts @@ -350,6 +350,14 @@ interface ServiceWorkerGlobalScope extends WorkerGlobalScope { ReadableByteStreamController: typeof ReadableByteStreamController; WritableStreamDefaultController: typeof WritableStreamDefaultController; TransformStreamDefaultController: typeof TransformStreamDefaultController; + Buffer: any; + process: any; + global: ServiceWorkerGlobalScope; + setImmediate( + $function: (...param0: any[]) => void, + ...args: any[] + ): Immediate; + clearImmediate(immediate: Immediate | null): void; CompressionStream: typeof CompressionStream; DecompressionStream: typeof DecompressionStream; TextEncoderStream: typeof TextEncoderStream; @@ -381,6 +389,13 @@ interface ServiceWorkerGlobalScope extends WorkerGlobalScope { FixedLengthStream: typeof FixedLengthStream; IdentityTransformStream: typeof IdentityTransformStream; HTMLRewriter: typeof HTMLRewriter; + Performance: typeof Performance; + PerformanceEntry: typeof PerformanceEntry; + PerformanceMark: typeof PerformanceMark; + PerformanceMeasure: typeof PerformanceMeasure; + PerformanceResourceTiming: typeof PerformanceResourceTiming; + PerformanceObserver: typeof PerformanceObserver; + PerformanceObserverEntryList: typeof PerformanceObserverEntryList; } declare function addEventListener( type: Type, @@ -472,6 +487,14 @@ declare const scheduler: Scheduler; declare const performance: Performance; declare const Cloudflare: Cloudflare; declare const origin: string; +declare const Buffer: any; +declare const process: any; +declare const global: ServiceWorkerGlobalScope; +declare function setImmediate( + $function: (...param0: any[]) => void, + ...args: any[] +): Immediate; +declare function clearImmediate(immediate: Immediate | null): void; declare const navigator: Navigator; interface TestController {} interface ExecutionContext { @@ -563,6 +586,11 @@ interface AlarmInvocationInfo { readonly retryCount: number; readonly scheduledTime: number; } +interface Immediate { + ref(): void; + unref(): void; + hasRef(): boolean; +} interface Cloudflare { readonly compatibilityFlags: Record; } @@ -4080,17 +4108,363 @@ interface workerdResourceLimits { * * [Cloudflare Docs Reference](https://developers.cloudflare.com/workers/runtime-apis/performance/) */ -declare abstract class Performance { +declare abstract class Performance extends EventTarget { /* [Cloudflare Docs Reference](https://developers.cloudflare.com/workers/runtime-apis/performance/#performancetimeorigin) */ get timeOrigin(): number; /* [Cloudflare Docs Reference](https://developers.cloudflare.com/workers/runtime-apis/performance/#performancenow) */ now(): number; + get eventCounts(): EventCounts; + /** + * The **`clearMarks()`** method removes all or specific PerformanceMark objects from the browser's performance timeline. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Performance/clearMarks) + */ + clearMarks(name?: string): void; + /** + * The **`clearMeasures()`** method removes all or specific PerformanceMeasure objects from the browser's performance timeline. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Performance/clearMeasures) + */ + clearMeasures(name?: string): void; + /** + * The **`clearResourceTimings()`** method removes all performance entries with an PerformanceEntry.entryType of `'resource'` from the browser's performance timeline and sets the size of the performance resource data buffer to zero. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Performance/clearResourceTimings) + */ + clearResourceTimings(): void; + /** + * The **`getEntries()`** method returns an array of all PerformanceEntry objects currently present in the performance timeline. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Performance/getEntries) + */ + getEntries(): PerformanceEntry[]; + /** + * The **`getEntriesByName()`** method returns an array of PerformanceEntry objects currently present in the performance timeline with the given _name_ and _type_. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Performance/getEntriesByName) + */ + getEntriesByName(name: string, type?: string): PerformanceEntry[]; + /** + * The **`getEntriesByType()`** method returns an array of PerformanceEntry objects currently present in the performance timeline for a given _type_. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Performance/getEntriesByType) + */ + getEntriesByType(type: string): PerformanceEntry[]; + /** + * The **`mark()`** method creates a named PerformanceMark object representing a high resolution timestamp marker in the browser's performance timeline. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Performance/mark) + */ + mark(name: string, options?: PerformanceMarkOptions): PerformanceMark; + /** + * The **`measure()`** method creates a named PerformanceMeasure object representing a time measurement between two marks in the browser's performance timeline. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Performance/measure) + */ + measure( + measureName: string, + measureOptionsOrStartMark?: PerformanceMeasureOptions | string, + maybeEndMark?: string, + ): PerformanceMeasure; + /** + * The **`setResourceTimingBufferSize()`** method sets the desired size of the browser's resource timing buffer which stores the `'resource'` performance entries. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Performance/setResourceTimingBufferSize) + */ + setResourceTimingBufferSize(size: number): void; /** * The **`toJSON()`** method of the Performance interface is a Serialization; it returns a JSON representation of the Performance object. * * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Performance/toJSON) */ toJSON(): object; + get nodeTiming(): PerformanceNodeTiming; + eventLoopUtilization(): PerformanceEventLoopUtilization; + markResourceTiming(): void; + timerify(fn: () => void): () => void; +} +interface PerformanceEventLoopUtilization { + idle: number; + active: number; + utilization: number; +} +interface PerformanceNodeTiming extends PerformanceEntry { + readonly nodeStart: number; + readonly v8Start: number; + readonly bootstrapComplete: number; + readonly environment: number; + readonly loopStart: number; + readonly loopExit: number; + readonly idleTime: number; + readonly uvMetricsInfo: UvMetricsInfo; + toJSON(): object; +} +interface UvMetricsInfo { + loopCount: number; + events: number; + eventsWaiting: number; +} +/** + * **`PerformanceMark`** is an interface for PerformanceEntry objects with an PerformanceEntry.entryType of `'mark'`. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/PerformanceMark) + */ +declare class PerformanceMark extends PerformanceEntry { + constructor(name: string, maybeOptions?: PerformanceMarkOptions); + /** + * The read-only **`detail`** property returns arbitrary metadata that was included in the mark upon construction (either when using Performance.mark or the PerformanceMark.PerformanceMark constructor). + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/PerformanceMark/detail) + */ + get detail(): any; + toJSON(): object; +} +/** + * **`PerformanceMeasure`** is an _abstract_ interface for PerformanceEntry objects with an PerformanceEntry.entryType of `'measure'`. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/PerformanceMeasure) + */ +declare abstract class PerformanceMeasure extends PerformanceEntry { + /** + * The read-only **`detail`** property returns arbitrary metadata that was included in the mark upon construction (when using Performance.measure. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/PerformanceMeasure/detail) + */ + get detail(): any; + toJSON(): object; +} +interface PerformanceMarkOptions { + detail?: any; + startTime?: number; +} +interface PerformanceMeasureOptions { + detail?: any; + start?: number; + duration?: number; + end?: number; +} +/** + * The **`PerformanceObserverEntryList`** interface is a list of PerformanceEntry that were explicitly observed via the PerformanceObserver.observe method. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/PerformanceObserverEntryList) + */ +declare abstract class PerformanceObserverEntryList { + /** + * The **`getEntries()`** method of the PerformanceObserverEntryList interface returns a list of explicitly observed PerformanceEntry objects. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/PerformanceObserverEntryList/getEntries) + */ + getEntries(): PerformanceEntry[]; + /** + * The **`getEntriesByType()`** method of the PerformanceObserverEntryList returns a list of explicitly _observed_ PerformanceEntry objects for a given PerformanceEntry.entryType. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/PerformanceObserverEntryList/getEntriesByType) + */ + getEntriesByType(type: string): PerformanceEntry[]; + /** + * The **`getEntriesByName()`** method of the PerformanceObserverEntryList interface returns a list of explicitly observed PerformanceEntry objects for a given PerformanceEntry.name and PerformanceEntry.entryType. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/PerformanceObserverEntryList/getEntriesByName) + */ + getEntriesByName(name: string, type?: string): PerformanceEntry[]; +} +/** + * The **`PerformanceEntry`** object encapsulates a single performance metric that is part of the browser's performance timeline. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/PerformanceEntry) + */ +declare abstract class PerformanceEntry { + /** + * The read-only **`name`** property of the PerformanceEntry interface is a string representing the name for a performance entry. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/PerformanceEntry/name) + */ + get name(): string; + /** + * The read-only **`entryType`** property returns a string representing the type of performance metric that this entry represents. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/PerformanceEntry/entryType) + */ + get entryType(): string; + /** + * The read-only **`startTime`** property returns the first DOMHighResTimeStamp recorded for this PerformanceEntry. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/PerformanceEntry/startTime) + */ + get startTime(): number; + /** + * The read-only **`duration`** property returns a DOMHighResTimeStamp that is the duration of the PerformanceEntry. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/PerformanceEntry/duration) + */ + get duration(): number; + /** + * The **`toJSON()`** method is a Serialization; it returns a JSON representation of the PerformanceEntry object. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/PerformanceEntry/toJSON) + */ + toJSON(): object; +} +/** + * The **`PerformanceResourceTiming`** interface enables retrieval and analysis of detailed network timing data regarding the loading of an application's resources. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/PerformanceResourceTiming) + */ +declare abstract class PerformanceResourceTiming extends PerformanceEntry { + /** + * The **`connectEnd`** read-only property returns the DOMHighResTimeStamp immediately after the browser finishes establishing the connection to the server to retrieve the resource. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/PerformanceResourceTiming/connectEnd) + */ + get connectEnd(): number; + /** + * The **`connectStart`** read-only property returns the DOMHighResTimeStamp immediately before the user agent starts establishing the connection to the server to retrieve the resource. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/PerformanceResourceTiming/connectStart) + */ + get connectStart(): number; + /** + * The **`decodedBodySize`** read-only property returns the size (in octets) received from the fetch (HTTP or cache) of the message body after removing any applied content encoding (like gzip or Brotli). + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/PerformanceResourceTiming/decodedBodySize) + */ + get decodedBodySize(): number; + /** + * The **`domainLookupEnd`** read-only property returns the DOMHighResTimeStamp immediately after the browser finishes the domain-name lookup for the resource. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/PerformanceResourceTiming/domainLookupEnd) + */ + get domainLookupEnd(): number; + /** + * The **`domainLookupStart`** read-only property returns the DOMHighResTimeStamp immediately before the browser starts the domain name lookup for the resource. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/PerformanceResourceTiming/domainLookupStart) + */ + get domainLookupStart(): number; + /** + * The **`encodedBodySize`** read-only property represents the size (in octets) received from the fetch (HTTP or cache) of the payload body before removing any applied content encodings (like gzip or Brotli). + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/PerformanceResourceTiming/encodedBodySize) + */ + get encodedBodySize(): number; + /** + * The **`fetchStart`** read-only property represents a DOMHighResTimeStamp immediately before the browser starts to fetch the resource. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/PerformanceResourceTiming/fetchStart) + */ + get fetchStart(): number; + /** + * The **`initiatorType`** read-only property is a string representing web platform feature that initiated the resource load. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/PerformanceResourceTiming/initiatorType) + */ + get initiatorType(): string; + /** + * The **`nextHopProtocol`** read-only property is a string representing the network protocol used to fetch the resource, as identified by the ALPN Protocol ID (RFC7301). + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/PerformanceResourceTiming/nextHopProtocol) + */ + get nextHopProtocol(): string; + /** + * The **`redirectEnd`** read-only property returns a DOMHighResTimeStamp immediately after receiving the last byte of the response of the last redirect. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/PerformanceResourceTiming/redirectEnd) + */ + get redirectEnd(): number; + /** + * The **`redirectStart`** read-only property returns a DOMHighResTimeStamp representing the start time of the fetch which that initiates the redirect. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/PerformanceResourceTiming/redirectStart) + */ + get redirectStart(): number; + /** + * The **`requestStart`** read-only property returns a DOMHighResTimeStamp of the time immediately before the browser starts requesting the resource from the server, cache, or local resource. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/PerformanceResourceTiming/requestStart) + */ + get requestStart(): number; + /** + * The **`responseEnd`** read-only property returns a DOMHighResTimeStamp immediately after the browser receives the last byte of the resource or immediately before the transport connection is closed, whichever comes first. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/PerformanceResourceTiming/responseEnd) + */ + get responseEnd(): number; + /** + * The **`responseStart`** read-only property returns a DOMHighResTimeStamp immediately after the browser receives the first byte of the response from the server, cache, or local resource. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/PerformanceResourceTiming/responseStart) + */ + get responseStart(): number; + /** + * The **`responseStatus`** read-only property represents the HTTP response status code returned when fetching the resource. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/PerformanceResourceTiming/responseStatus) + */ + get responseStatus(): number; + /** + * The **`secureConnectionStart`** read-only property returns a DOMHighResTimeStamp immediately before the browser starts the handshake process to secure the current connection. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/PerformanceResourceTiming/secureConnectionStart) + */ + get secureConnectionStart(): number | undefined; + /** + * The **`transferSize`** read-only property represents the size (in octets) of the fetched resource. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/PerformanceResourceTiming/transferSize) + */ + get transferSize(): number; + /** + * The **`workerStart`** read-only property of the PerformanceResourceTiming interface returns a The `workerStart` property can have the following values: - A DOMHighResTimeStamp. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/PerformanceResourceTiming/workerStart) + */ + get workerStart(): number; +} +/** + * The **`PerformanceObserver`** interface is used to observe performance measurement events and be notified of new PerformanceEntry as they are recorded in the browser's _performance timeline_. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/PerformanceObserver) + */ +declare class PerformanceObserver { + constructor(callback: any); + /** + * The **`disconnect()`** method of the PerformanceObserver interface is used to stop the performance observer from receiving any PerformanceEntry events. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/PerformanceObserver/disconnect) + */ + disconnect(): void; + /** + * The **`observe()`** method of the **PerformanceObserver** interface is used to specify the set of performance entry types to observe. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/PerformanceObserver/observe) + */ + observe(options?: PerformanceObserverObserveOptions): void; + /** + * The **`takeRecords()`** method of the PerformanceObserver interface returns the current list of PerformanceEntry objects stored in the performance observer, emptying it out. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/PerformanceObserver/takeRecords) + */ + takeRecords(): PerformanceEntry[]; + readonly supportedEntryTypes: string[]; +} +interface PerformanceObserverObserveOptions { + buffered?: boolean; + durationThreshold?: number; + entryTypes?: string[]; + type?: string; +} +interface EventCounts { + get size(): number; + get(eventType: string): number | undefined; + has(eventType: string): boolean; + entries(): IterableIterator; + keys(): IterableIterator; + values(): IterableIterator; + forEach( + param1: (param0: number, param1: string, param2: EventCounts) => void, + param2?: any, + ): void; + [Symbol.iterator](): IterableIterator; } interface Tracing { enterSpan( diff --git a/types/generated-snapshot/index.ts b/types/generated-snapshot/index.ts index 18d6b326c10..cc1021da1dd 100755 --- a/types/generated-snapshot/index.ts +++ b/types/generated-snapshot/index.ts @@ -350,6 +350,14 @@ export interface ServiceWorkerGlobalScope extends WorkerGlobalScope { ReadableByteStreamController: typeof ReadableByteStreamController; WritableStreamDefaultController: typeof WritableStreamDefaultController; TransformStreamDefaultController: typeof TransformStreamDefaultController; + Buffer: any; + process: any; + global: ServiceWorkerGlobalScope; + setImmediate( + $function: (...param0: any[]) => void, + ...args: any[] + ): Immediate; + clearImmediate(immediate: Immediate | null): void; CompressionStream: typeof CompressionStream; DecompressionStream: typeof DecompressionStream; TextEncoderStream: typeof TextEncoderStream; @@ -381,6 +389,13 @@ export interface ServiceWorkerGlobalScope extends WorkerGlobalScope { FixedLengthStream: typeof FixedLengthStream; IdentityTransformStream: typeof IdentityTransformStream; HTMLRewriter: typeof HTMLRewriter; + Performance: typeof Performance; + PerformanceEntry: typeof PerformanceEntry; + PerformanceMark: typeof PerformanceMark; + PerformanceMeasure: typeof PerformanceMeasure; + PerformanceResourceTiming: typeof PerformanceResourceTiming; + PerformanceObserver: typeof PerformanceObserver; + PerformanceObserverEntryList: typeof PerformanceObserverEntryList; } export declare function addEventListener< Type extends keyof WorkerGlobalScopeEventMap, @@ -474,6 +489,14 @@ export declare const scheduler: Scheduler; export declare const performance: Performance; export declare const Cloudflare: Cloudflare; export declare const origin: string; +export declare const Buffer: any; +export declare const process: any; +export declare const global: ServiceWorkerGlobalScope; +export declare function setImmediate( + $function: (...param0: any[]) => void, + ...args: any[] +): Immediate; +export declare function clearImmediate(immediate: Immediate | null): void; export declare const navigator: Navigator; export interface TestController {} export interface ExecutionContext { @@ -565,6 +588,11 @@ export interface AlarmInvocationInfo { readonly retryCount: number; readonly scheduledTime: number; } +export interface Immediate { + ref(): void; + unref(): void; + hasRef(): boolean; +} export interface Cloudflare { readonly compatibilityFlags: Record; } @@ -4090,17 +4118,363 @@ export interface workerdResourceLimits { * * [Cloudflare Docs Reference](https://developers.cloudflare.com/workers/runtime-apis/performance/) */ -export declare abstract class Performance { +export declare abstract class Performance extends EventTarget { /* [Cloudflare Docs Reference](https://developers.cloudflare.com/workers/runtime-apis/performance/#performancetimeorigin) */ get timeOrigin(): number; /* [Cloudflare Docs Reference](https://developers.cloudflare.com/workers/runtime-apis/performance/#performancenow) */ now(): number; + get eventCounts(): EventCounts; + /** + * The **`clearMarks()`** method removes all or specific PerformanceMark objects from the browser's performance timeline. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Performance/clearMarks) + */ + clearMarks(name?: string): void; + /** + * The **`clearMeasures()`** method removes all or specific PerformanceMeasure objects from the browser's performance timeline. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Performance/clearMeasures) + */ + clearMeasures(name?: string): void; + /** + * The **`clearResourceTimings()`** method removes all performance entries with an PerformanceEntry.entryType of `'resource'` from the browser's performance timeline and sets the size of the performance resource data buffer to zero. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Performance/clearResourceTimings) + */ + clearResourceTimings(): void; + /** + * The **`getEntries()`** method returns an array of all PerformanceEntry objects currently present in the performance timeline. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Performance/getEntries) + */ + getEntries(): PerformanceEntry[]; + /** + * The **`getEntriesByName()`** method returns an array of PerformanceEntry objects currently present in the performance timeline with the given _name_ and _type_. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Performance/getEntriesByName) + */ + getEntriesByName(name: string, type?: string): PerformanceEntry[]; + /** + * The **`getEntriesByType()`** method returns an array of PerformanceEntry objects currently present in the performance timeline for a given _type_. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Performance/getEntriesByType) + */ + getEntriesByType(type: string): PerformanceEntry[]; + /** + * The **`mark()`** method creates a named PerformanceMark object representing a high resolution timestamp marker in the browser's performance timeline. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Performance/mark) + */ + mark(name: string, options?: PerformanceMarkOptions): PerformanceMark; + /** + * The **`measure()`** method creates a named PerformanceMeasure object representing a time measurement between two marks in the browser's performance timeline. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Performance/measure) + */ + measure( + measureName: string, + measureOptionsOrStartMark?: PerformanceMeasureOptions | string, + maybeEndMark?: string, + ): PerformanceMeasure; + /** + * The **`setResourceTimingBufferSize()`** method sets the desired size of the browser's resource timing buffer which stores the `'resource'` performance entries. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Performance/setResourceTimingBufferSize) + */ + setResourceTimingBufferSize(size: number): void; /** * The **`toJSON()`** method of the Performance interface is a Serialization; it returns a JSON representation of the Performance object. * * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Performance/toJSON) */ toJSON(): object; + get nodeTiming(): PerformanceNodeTiming; + eventLoopUtilization(): PerformanceEventLoopUtilization; + markResourceTiming(): void; + timerify(fn: () => void): () => void; +} +export interface PerformanceEventLoopUtilization { + idle: number; + active: number; + utilization: number; +} +export interface PerformanceNodeTiming extends PerformanceEntry { + readonly nodeStart: number; + readonly v8Start: number; + readonly bootstrapComplete: number; + readonly environment: number; + readonly loopStart: number; + readonly loopExit: number; + readonly idleTime: number; + readonly uvMetricsInfo: UvMetricsInfo; + toJSON(): object; +} +export interface UvMetricsInfo { + loopCount: number; + events: number; + eventsWaiting: number; +} +/** + * **`PerformanceMark`** is an interface for PerformanceEntry objects with an PerformanceEntry.entryType of `'mark'`. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/PerformanceMark) + */ +export declare class PerformanceMark extends PerformanceEntry { + constructor(name: string, maybeOptions?: PerformanceMarkOptions); + /** + * The read-only **`detail`** property returns arbitrary metadata that was included in the mark upon construction (either when using Performance.mark or the PerformanceMark.PerformanceMark constructor). + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/PerformanceMark/detail) + */ + get detail(): any; + toJSON(): object; +} +/** + * **`PerformanceMeasure`** is an _abstract_ interface for PerformanceEntry objects with an PerformanceEntry.entryType of `'measure'`. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/PerformanceMeasure) + */ +export declare abstract class PerformanceMeasure extends PerformanceEntry { + /** + * The read-only **`detail`** property returns arbitrary metadata that was included in the mark upon construction (when using Performance.measure. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/PerformanceMeasure/detail) + */ + get detail(): any; + toJSON(): object; +} +export interface PerformanceMarkOptions { + detail?: any; + startTime?: number; +} +export interface PerformanceMeasureOptions { + detail?: any; + start?: number; + duration?: number; + end?: number; +} +/** + * The **`PerformanceObserverEntryList`** interface is a list of PerformanceEntry that were explicitly observed via the PerformanceObserver.observe method. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/PerformanceObserverEntryList) + */ +export declare abstract class PerformanceObserverEntryList { + /** + * The **`getEntries()`** method of the PerformanceObserverEntryList interface returns a list of explicitly observed PerformanceEntry objects. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/PerformanceObserverEntryList/getEntries) + */ + getEntries(): PerformanceEntry[]; + /** + * The **`getEntriesByType()`** method of the PerformanceObserverEntryList returns a list of explicitly _observed_ PerformanceEntry objects for a given PerformanceEntry.entryType. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/PerformanceObserverEntryList/getEntriesByType) + */ + getEntriesByType(type: string): PerformanceEntry[]; + /** + * The **`getEntriesByName()`** method of the PerformanceObserverEntryList interface returns a list of explicitly observed PerformanceEntry objects for a given PerformanceEntry.name and PerformanceEntry.entryType. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/PerformanceObserverEntryList/getEntriesByName) + */ + getEntriesByName(name: string, type?: string): PerformanceEntry[]; +} +/** + * The **`PerformanceEntry`** object encapsulates a single performance metric that is part of the browser's performance timeline. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/PerformanceEntry) + */ +export declare abstract class PerformanceEntry { + /** + * The read-only **`name`** property of the PerformanceEntry interface is a string representing the name for a performance entry. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/PerformanceEntry/name) + */ + get name(): string; + /** + * The read-only **`entryType`** property returns a string representing the type of performance metric that this entry represents. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/PerformanceEntry/entryType) + */ + get entryType(): string; + /** + * The read-only **`startTime`** property returns the first DOMHighResTimeStamp recorded for this PerformanceEntry. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/PerformanceEntry/startTime) + */ + get startTime(): number; + /** + * The read-only **`duration`** property returns a DOMHighResTimeStamp that is the duration of the PerformanceEntry. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/PerformanceEntry/duration) + */ + get duration(): number; + /** + * The **`toJSON()`** method is a Serialization; it returns a JSON representation of the PerformanceEntry object. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/PerformanceEntry/toJSON) + */ + toJSON(): object; +} +/** + * The **`PerformanceResourceTiming`** interface enables retrieval and analysis of detailed network timing data regarding the loading of an application's resources. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/PerformanceResourceTiming) + */ +export declare abstract class PerformanceResourceTiming extends PerformanceEntry { + /** + * The **`connectEnd`** read-only property returns the DOMHighResTimeStamp immediately after the browser finishes establishing the connection to the server to retrieve the resource. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/PerformanceResourceTiming/connectEnd) + */ + get connectEnd(): number; + /** + * The **`connectStart`** read-only property returns the DOMHighResTimeStamp immediately before the user agent starts establishing the connection to the server to retrieve the resource. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/PerformanceResourceTiming/connectStart) + */ + get connectStart(): number; + /** + * The **`decodedBodySize`** read-only property returns the size (in octets) received from the fetch (HTTP or cache) of the message body after removing any applied content encoding (like gzip or Brotli). + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/PerformanceResourceTiming/decodedBodySize) + */ + get decodedBodySize(): number; + /** + * The **`domainLookupEnd`** read-only property returns the DOMHighResTimeStamp immediately after the browser finishes the domain-name lookup for the resource. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/PerformanceResourceTiming/domainLookupEnd) + */ + get domainLookupEnd(): number; + /** + * The **`domainLookupStart`** read-only property returns the DOMHighResTimeStamp immediately before the browser starts the domain name lookup for the resource. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/PerformanceResourceTiming/domainLookupStart) + */ + get domainLookupStart(): number; + /** + * The **`encodedBodySize`** read-only property represents the size (in octets) received from the fetch (HTTP or cache) of the payload body before removing any applied content encodings (like gzip or Brotli). + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/PerformanceResourceTiming/encodedBodySize) + */ + get encodedBodySize(): number; + /** + * The **`fetchStart`** read-only property represents a DOMHighResTimeStamp immediately before the browser starts to fetch the resource. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/PerformanceResourceTiming/fetchStart) + */ + get fetchStart(): number; + /** + * The **`initiatorType`** read-only property is a string representing web platform feature that initiated the resource load. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/PerformanceResourceTiming/initiatorType) + */ + get initiatorType(): string; + /** + * The **`nextHopProtocol`** read-only property is a string representing the network protocol used to fetch the resource, as identified by the ALPN Protocol ID (RFC7301). + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/PerformanceResourceTiming/nextHopProtocol) + */ + get nextHopProtocol(): string; + /** + * The **`redirectEnd`** read-only property returns a DOMHighResTimeStamp immediately after receiving the last byte of the response of the last redirect. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/PerformanceResourceTiming/redirectEnd) + */ + get redirectEnd(): number; + /** + * The **`redirectStart`** read-only property returns a DOMHighResTimeStamp representing the start time of the fetch which that initiates the redirect. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/PerformanceResourceTiming/redirectStart) + */ + get redirectStart(): number; + /** + * The **`requestStart`** read-only property returns a DOMHighResTimeStamp of the time immediately before the browser starts requesting the resource from the server, cache, or local resource. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/PerformanceResourceTiming/requestStart) + */ + get requestStart(): number; + /** + * The **`responseEnd`** read-only property returns a DOMHighResTimeStamp immediately after the browser receives the last byte of the resource or immediately before the transport connection is closed, whichever comes first. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/PerformanceResourceTiming/responseEnd) + */ + get responseEnd(): number; + /** + * The **`responseStart`** read-only property returns a DOMHighResTimeStamp immediately after the browser receives the first byte of the response from the server, cache, or local resource. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/PerformanceResourceTiming/responseStart) + */ + get responseStart(): number; + /** + * The **`responseStatus`** read-only property represents the HTTP response status code returned when fetching the resource. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/PerformanceResourceTiming/responseStatus) + */ + get responseStatus(): number; + /** + * The **`secureConnectionStart`** read-only property returns a DOMHighResTimeStamp immediately before the browser starts the handshake process to secure the current connection. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/PerformanceResourceTiming/secureConnectionStart) + */ + get secureConnectionStart(): number | undefined; + /** + * The **`transferSize`** read-only property represents the size (in octets) of the fetched resource. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/PerformanceResourceTiming/transferSize) + */ + get transferSize(): number; + /** + * The **`workerStart`** read-only property of the PerformanceResourceTiming interface returns a The `workerStart` property can have the following values: - A DOMHighResTimeStamp. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/PerformanceResourceTiming/workerStart) + */ + get workerStart(): number; +} +/** + * The **`PerformanceObserver`** interface is used to observe performance measurement events and be notified of new PerformanceEntry as they are recorded in the browser's _performance timeline_. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/PerformanceObserver) + */ +export declare class PerformanceObserver { + constructor(callback: any); + /** + * The **`disconnect()`** method of the PerformanceObserver interface is used to stop the performance observer from receiving any PerformanceEntry events. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/PerformanceObserver/disconnect) + */ + disconnect(): void; + /** + * The **`observe()`** method of the **PerformanceObserver** interface is used to specify the set of performance entry types to observe. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/PerformanceObserver/observe) + */ + observe(options?: PerformanceObserverObserveOptions): void; + /** + * The **`takeRecords()`** method of the PerformanceObserver interface returns the current list of PerformanceEntry objects stored in the performance observer, emptying it out. + * + * [MDN Reference](https://developer.mozilla.org/docs/Web/API/PerformanceObserver/takeRecords) + */ + takeRecords(): PerformanceEntry[]; + readonly supportedEntryTypes: string[]; +} +export interface PerformanceObserverObserveOptions { + buffered?: boolean; + durationThreshold?: number; + entryTypes?: string[]; + type?: string; +} +export interface EventCounts { + get size(): number; + get(eventType: string): number | undefined; + has(eventType: string): boolean; + entries(): IterableIterator; + keys(): IterableIterator; + values(): IterableIterator; + forEach( + param1: (param0: number, param1: string, param2: EventCounts) => void, + param2?: any, + ): void; + [Symbol.iterator](): IterableIterator; } export interface Tracing { enterSpan(