Fix network telemetry capture across navigation

.github/workflows/server-test.yaml

@@ -103,7 +103,7 @@ jobs:
           cache-dependency-path: server/go.sum
 
       - name: Run CDP telemetry browser regressions
-        run: go test -race ./lib/cdpmonitor -run '^(TestNetworkCapture|TestTelemetryConnection|TestProxyErrorE2E)' -count=1 -v
+        run: go test -race ./lib/cdpmonitor -run '^(TestNetworkCapture|TestNetworkWireCapture|TestNetworkBodiesSurviveNavigation|TestNetworkExtensionAttribution|TestTelemetryConnection|TestProxyErrorE2E)' -count=1 -v
         working-directory: server
         env:
           KERNEL_CDPMONITOR_CHROME_E2E: "1"

server/lib/cdpmonitor/README.md

@@ -64,8 +64,8 @@ startup, telemetry toggles, and shutdown. Additional regressions delay a cleanup
 command while PUT/PATCH/GET continue, fence old capture across coalesced revisions,
 and recover page/frame listeners after socket loss or failed cleanup. They check
 future navigations and the independent user CDP connection as well. Extension
-background pages are included in discovery but are not validated by a real extension
-fixture here. A delayed-Network-reply regression also verifies that pending
+service-worker and page attribution are covered by an unpacked-extension browser
+fixture. A delayed-Network-reply regression also verifies that pending
 attachments finish recovery before optional instrumentation starts, and that
 click events arrive without navigation through subsequent telemetry toggles.
 
@@ -109,6 +109,83 @@ and fresh sessions after reconnect. Without the environment variable these tests
 skip, matching the other real-Chromium tests in this package. They exercise settled
 targets, not requests issued before their capture domains finish initializing.
 
+## Network capture fidelity
+
+Top-level navigation retains requests until their terminal events. Only requests
+belonging to the new loader seed its pending count; completions from the previous
+navigation cannot make the new navigation idle. Same-process frame requests still
+participate in their page's network-idle accounting.
+
+The collector joins `requestWillBeSentExtraInfo` to requests
+by session, request ID, and redirect hop. Cached hops without ExtraInfo do not
+consume the next hop's headers. A `network_request` waits at most one second for
+wire headers, then uses the base headers so a hanging request remains visible.
+`network_response.request_headers` retains wire request headers that arrive after
+that initial event. After body retrieval, response capture waits up to one second
+for request headers. Missing or invalid request headers fall back to the base
+headers on the request event and are omitted from the response. Capture remains
+best-effort when Chrome omits events or a target disconnects. Redirects emit a
+response for each hop, without a body. Request IDs remain the same across a
+redirect chain.
+
+Network event timestamps record wall-clock time when the CDP event is observed,
+before header collection, body retrieval, or publication waits. Response events
+use observation of `loadingFinished`, or the redirect notification for redirects.
+Timestamp differences therefore retain request duration despite delayed delivery.
+Other event types may publish during these waits, so timestamps need not increase
+in stream order.
+
+Responses, failures, and proxy errors for tracked requests publish after their
+request event. Each redirect response publishes before the next hop's request.
+Header waits run concurrently across hops and remain bounded by one second;
+publication dependencies drain in order even when capture is canceled.
+Header correlation stays active for pending redirect responses after the final
+request completes or fails, until their capture finishes.
+
+`network_response.status`, `status_text`, and `headers` retain Chrome's effective
+response, including custom reason phrases and retained cache headers. A
+revalidated cached response has `status: 200` and may carry Chrome's cached body.
+Body capture is limited to selected textual resources and remains subject to
+truncation and retrieval errors.
+
+While optional telemetry is enabled, `Network.enable` requests durable body
+storage with `maxTotalBufferSize` set to 10 MiB on each attached session. Disabling
+telemetry stops new durable capture without disabling the Network domain used by
+counters. Chrome may keep already-buffered bodies until eviction or disconnection.
+Unsupported configuration is logged and ordinary capture continues. Bodies can
+still be evicted under buffer pressure; `body_error` reports retrieval/decoding
+failures separately from intentionally excluded or empty bodies. The 8 KiB/4 KiB
+output caps are unchanged.
+
+Extension-target events carry `source.metadata.extension_id` and
+`source.metadata.kernel_extension` (`"true"` or `"false"`). The latter identifies
+the bundled extensions loaded from their fixed installation paths. Consumers can
+filter these sources while retaining application workers and custom extensions.
+Network events snapshot the request's document origin, falling back to the target
+URL, so navigating into or out of an extension page does not relabel pending
+requests. Content-script requests made through an ordinary page document are not
+attributed to an extension. Attribution is a telemetry aid, not a security boundary.
+
+Run the browser regressions with:
+
+```sh
+KERNEL_CDPMONITOR_CHROME_E2E=1 go test -race ./lib/cdpmonitor \
+  -run '^TestNetwork(WireCapture|BodiesSurviveNavigation|ExtensionAttribution)$' -count=1
+```
+
+Bundled-extension attribution also has a browser regression that loads an actual
+installed bundle and checks its Chromium-assigned ID. Run it once for each bundled
+path (`/home/kernel/extensions/kernel` and `/home/kernel/extensions/capmonster`):
+
+```sh
+KERNEL_CDPMONITOR_BUNDLED_EXTENSION_PATH=/home/kernel/extensions/kernel \
+  go test -race ./lib/cdpmonitor -run '^TestNetworkBundledExtensionAttribution$' -count=1
+```
+
+This requires the bundle at its installation path, with its normal manifest and
+assets. It is separate from the general Chromium suite because the bundles are
+not included in this source tree. An explicit path that cannot load fails the test.
+
 ## Event taxonomy
 
 **CDP-derived** (1-to-1 with a CDP notification): `console_log`, `console_error`, `network_request`, `network_response`, `network_loading_failed`, `proxy_error` (classified from a branded 502 or 403 response carrying the `X-Kernel-Proxy-Error` header), `page_tab_opened`, `page_navigation`, `page_dom_content_loaded`, `page_load`, `page_layout_shift`, `page_lcp`. `proxy_error` is an opt-in per-session/per-URL refinement of the raw `network` events: it is only observable while the network category (CDP collector) is running, so it is not a default-on alerting signal.
@@ -360,7 +437,7 @@ Unless otherwise noted, events also include the nav context fields described abo
 | Event | Fields |
 | --- | --- |
 | `network_request` | `request_id`, `loader_id`, `frame_id`, `document_url`, `method`, `url`, `headers`, `initiator_type`. Optional: `post_data`, `resource_type`, `is_redirect` + `redirect_url`. |
-| `network_response` | `request_id`, `loader_id`, `frame_id`, `method`, `url`, `status`, `headers`. Optional: `status_text`, `mime_type`, `resource_type`, `body` (truncated text body for textual MIME types). |
+| `network_response` | `request_id`, `loader_id`, `frame_id`, `method`, `url`, `status`, `headers`. Optional: `status_text`, `mime_type`, `resource_type`, `body` (truncated text body for textual MIME types), `body_error`, `request_headers`. |
 | `network_loading_failed` | `request_id`, `error_text`, `canceled`. Optional (absent when the request record was not found): `url`, `loader_id`, `frame_id`, `resource_type`. |
 | `proxy_error` | `request_id`, `code` (typed enum matching the metro header values), `status` (502, or 403 when the session's network policy blocks the destination). Optional: `raw_code` (sanitized original value when `code` is `unknown`), `url`, `loader_id`, `frame_id`, `method`, `resource_type`. Emitted when a 502 or 403 response carries the `X-Kernel-Proxy-Error` header. Unrecognized values are reported as `unknown`; all such values share one rate-limit slot. Emission is sampled to at most one per session+code+resource_type per second. WebSocket handshakes are not classified (documented non-goal). A CONNECT the proxy refuses outright, such as a `network_policy_denied` CONNECT to a port other than 443, is not classified either: Chromium does not expose the refusal, so it appears only as `network_loading_failed` with `net::ERR_TUNNEL_CONNECTION_FAILED`. |
 

server/lib/cdpmonitor/cdp_proto.go

@@ -162,6 +162,17 @@ type cdpNetworkRequestWillBeSentParams struct {
 	RenderBlockingBehavior string            `json:"renderBlockingBehavior,omitempty"`
 }
 
+// cdpNetworkRequestWillBeSentExtraInfoParams mirrors Network.requestWillBeSentExtraInfo.
+type cdpNetworkRequestWillBeSentExtraInfoParams struct {
+	RequestID                     string          `json:"requestId"`
+	AssociatedCookies             json.RawMessage `json:"associatedCookies"`
+	Headers                       json.RawMessage `json:"headers"`
+	ConnectTiming                 json.RawMessage `json:"connectTiming"`
+	ClientSecurityState           json.RawMessage `json:"clientSecurityState,omitempty"`
+	SiteHasCookieInOtherPartition bool            `json:"siteHasCookieInOtherPartition,omitempty"`
+	AppliedNetworkConditionsID    string          `json:"appliedNetworkConditionsId,omitempty"`
+}
+
 // cdpNetworkResponseReceivedParams mirrors Network.responseReceived params.
 type cdpNetworkResponseReceivedParams struct {
 	RequestID    string             `json:"requestId"`

server/lib/cdpmonitor/cdp_proto_test.go

@@ -29,6 +29,7 @@ func TestLayer1Roundtrip(t *testing.T) {
 		{"Runtime.exceptionThrown", "Runtime_exceptionThrown.json", func() any { return new(cdpRuntimeExceptionThrownParams) }},
 		{"Runtime.bindingCalled", "Runtime_bindingCalled.json", func() any { return new(cdpRuntimeBindingCalledParams) }},
 		{"Network.requestWillBeSent", "Network_requestWillBeSent.json", func() any { return new(cdpNetworkRequestWillBeSentParams) }},
+		{"Network.requestWillBeSentExtraInfo", "Network_requestWillBeSentExtraInfo.json", func() any { return new(cdpNetworkRequestWillBeSentExtraInfoParams) }},
 		{"Network.responseReceived", "Network_responseReceived.json", func() any { return new(cdpNetworkResponseReceivedParams) }},
 		{"Network.loadingFinished", "Network_loadingFinished.json", func() any { return new(cdpNetworkLoadingFinishedParams) }},
 		{"Network.loadingFailed", "Network_loadingFailed.json", func() any { return new(cdpNetworkLoadingFailedParams) }},

server/lib/cdpmonitor/cdp_test.go

@@ -390,10 +390,7 @@ func simulateRequest(m *Monitor, id string) {
 	}, "s1")
 }
 
-// simulateFinished stores minimal state and sends Network.loadingFinished.
+// simulateFinished completes the request without replacing its captured state.
 func simulateFinished(m *Monitor, id string) {
-	m.pendReqMu.Lock()
-	m.pendingRequests[networkRequestKey{"s1", id}] = networkReqState{sessionID: "s1", method: "GET", url: "https://example.com/" + id}
-	m.pendReqMu.Unlock()
 	m.handleLoadingFinished(context.Background(), cdpNetworkLoadingFinishedParams{RequestID: id}, "s1")
 }

server/lib/cdpmonitor/computed.go

@@ -138,8 +138,7 @@ func (s *computedState) stop() {
 
 // resetOnNavigation resets all state machines. Called on Page.frameNavigated.
 // Increments navSeq so any AfterFunc callbacks already running will discard their results.
-// inflight seeds netPending; callers pass 0 because each session only tracks its
-// own requests and starts fresh on navigation.
+// inflight seeds netPending with requests already observed for the new loader.
 func (s *computedState) resetOnNavigation(inflight int, ctx navContext) error {
 	s.mu.Lock()
 	defer s.mu.Unlock()
@@ -159,11 +158,8 @@ func (s *computedState) resetOnNavigation(inflight int, ctx navContext) error {
 		return err
 	}
 	s.navData = navData
-	s.navMeta = map[string]string{
-		MetadataKeyCDPSessionID: ctx.sessionID,
-		MetadataKeyTargetID:     ctx.targetID,
-		MetadataKeyTargetType:   ctx.targetType,
-	}
+	s.navMeta = targetMetadata(targetInfo{targetID: ctx.targetID, targetType: ctx.targetType, url: ctx.url})
+	s.navMeta[MetadataKeyCDPSessionID] = ctx.sessionID
 
 	stopTimer(s.netTimer)
 	s.netTimer = nil

server/lib/cdpmonitor/domains.go

@@ -133,9 +133,10 @@ func (m *Monitor) disableOptionalDomains(ctx context.Context, sessionID, scriptI
 	if !exists {
 		return nil
 	}
-	var cleanupErr error
+	cleanupErr := m.configureBodyCapture(ctx, sessionID, false)
 	if scriptID != "" {
-		_, cleanupErr = m.send(ctx, "Page.removeScriptToEvaluateOnNewDocument", map[string]any{"identifier": scriptID}, sessionID)
+		_, err := m.send(ctx, "Page.removeScriptToEvaluateOnNewDocument", map[string]any{"identifier": scriptID}, sessionID)
+		cleanupErr = errors.Join(cleanupErr, err)
 	}
 	if isPageLikeTarget(info.targetType) {
 		if dirty {

server/lib/cdpmonitor/extension_source.go

@@ -0,0 +1,36 @@
+package cdpmonitor
+
+import (
+	"crypto/sha256"
+	"encoding/hex"
+	"net/url"
+	"strconv"
+	"strings"
+)
+
+// These bundled extensions are loaded unpacked at fixed paths without a manifest
+// key. Chromium derives their IDs from the first 128 bits of SHA-256(path).
+var bundledExtensionIDs = map[string]bool{
+	unpackedExtensionID("/home/kernel/extensions/kernel"):     true,
+	unpackedExtensionID("/home/kernel/extensions/capmonster"): true,
+}
+
+func unpackedExtensionID(path string) string {
+	sum := sha256.Sum256([]byte(path))
+	return strings.Map(func(r rune) rune {
+		if r <= '9' {
+			return 'a' + r - '0'
+		}
+		return 'k' + r - 'a'
+	}, hex.EncodeToString(sum[:16]))
+}
+
+func targetMetadata(info targetInfo) map[string]string {
+	metadata := map[string]string{MetadataKeyTargetID: info.targetID, MetadataKeyTargetType: info.targetType}
+	origin, err := url.Parse(info.url)
+	if err == nil && origin.Scheme == "chrome-extension" && origin.Host != "" {
+		metadata["extension_id"] = origin.Host
+		metadata["kernel_extension"] = strconv.FormatBool(bundledExtensionIDs[origin.Host])
+	}
+	return metadata
+}