From 898dd3265ba834ae1287693d37dc62fc26328589 Mon Sep 17 00:00:00 2001 From: Rob Hogan Date: Sat, 26 Sep 2026 09:14:25 +0100 Subject: [PATCH] composeSourceMaps: Encode the composed map directly ## Context `composeSourceMaps` built its output with `source-map`'s `SourceMapGenerator`, which allocates and validates an object per mapping and serialises by string concatenation: https://github.com/react/metro/blob/b7c20552a16a251507f753c3da23cc7d18488b6c/packages/metro-source-map/src/composeSourceMaps.js#L44-L56 On a large map that was most of the remaining time. ## This change This collects the composed mappings into parallel arrays and encodes them with Metro's own `B64Builder`. Output is unchanged, including source and name ordering, de-duplication, sorting and the error for an invalid mapping. This also fixes `B64Builder.markLines`, which grew its buffer at most once and silently dropped line markers that didn't fit. Without the fix, `composeSourceMaps` truncates a gap of more than about two million generated lines between mappings, which `SourceMapGenerator` handled. Metro's own `Generator` had the same bug. That puts Metro's composer level with Expo's `@jridgewell/remapping`-based one (~680ms on the same input), with more faithful output. It's the point of this stack: it makes the case for Expo's Hermes export path using Metro's composer rather than its own. `composeSourceMaps` is 20.6% faster than #1981 (95% CI 19.7-21.2%) and 3.25x faster than `main` (2,088ms -> 646ms on our benchmark app), with byte-identical output and no change in memory. ## Benchmark AI-driven. | | Time (median) | vs previous | vs `main` | Composition memory | vs previous | vs `main` | Peak RSS | |---|---|---|---|---|---|---|---| | `main` | 2,088ms (2,082 to 2,094) | | | 1,070MB (1,070 to 1,070) | | | 1,664MB | | #1981 | 805ms (802 to 811) | | -61.4% (-61.6 to -61.1) | 911MB (910 to 912) | | -14.9% (-15.0 to -14.7) | 1,505MB | | This PR | 646ms (641 to 650) | -20.6% (-21.2 to -19.7) | -69.2% (-69.3 to -69.0) | 914MB (907 to 915) | 0.0% (-0.4 to +0.3) | -14.8% (-15.2 to -14.6) | 1,508MB | ## Changelog ``` - **[Performance]**: `composeSourceMaps` encodes its output directly rather than through `source-map` - **[Fix]**: Source maps no longer lose lines after a gap of more than about two million unmapped generated lines ``` ## Test plan Compared against #1981 on 40,000 randomly generated compositions of one to three maps, including repeated and out-of-order mappings, names, unmapped positions and invalid mappings. Every output and every error message is identical. A new `B64Builder` test marks 3,145,728 lines, which previously produced 2,097,151 line markers. Benchmark (AI-driven): Mattermost Mobile 2.45.0 (React Native 0.83.9, Metro 0.83.7), iOS release bundle, unminified: 7,086 sources, 52.6MB bundle, 82.4MB flat source map. Composed with its `hermesc -O -output-source-map` map (hermes-compiler 0.14.1, 10.7MB, 1.84M segments). Timings are `composeSourceMaps` alone, excluding JSON parsing, over 105 rounds on Node 22.13.1 on an M5 Pro. Each round runs `main`, every PR in this stack and a memory baseline, in a shuffled order. Figures are medians with 95% bootstrap confidence intervals, for changes of the per-round paired ratio. Composition memory is peak RSS less that of the same process parsing both maps without composing them (594MB). --- packages/metro-source-map/src/B64Builder.js | 2 +- .../src/__tests__/B64Builder-test.js | 6 + .../metro-source-map/src/composeSourceMaps.js | 230 +++++++++++++++--- 3 files changed, 201 insertions(+), 37 deletions(-) diff --git a/packages/metro-source-map/src/B64Builder.js b/packages/metro-source-map/src/B64Builder.js index 72e6f93406..a97bdacfb2 100644 --- a/packages/metro-source-map/src/B64Builder.js +++ b/packages/metro-source-map/src/B64Builder.js @@ -47,7 +47,7 @@ export default class B64Builder { return this; } this.hasSegment = false; - if (this.pos + n >= this.buffer.length) { + while (this.pos + n >= this.buffer.length) { this._realloc(); } while (n--) { diff --git a/packages/metro-source-map/src/__tests__/B64Builder-test.js b/packages/metro-source-map/src/__tests__/B64Builder-test.js index 57a93d7709..8df14dd4ab 100644 --- a/packages/metro-source-map/src/__tests__/B64Builder-test.js +++ b/packages/metro-source-map/src/__tests__/B64Builder-test.js @@ -42,6 +42,12 @@ test('can mark zero new lines in the generated code', () => { expect(builder.toString()).toEqual(''); }); +test('can mark more new lines than twice the initial buffer size', () => { + const lines = 3 * 1024 * 1024; + builder.startSegment(0).markLines(lines).startSegment(0); + expect(builder.toString()).toEqual('A' + ';'.repeat(lines) + 'A'); +}); + test('does not add commas when just starting a segment', () => { builder.startSegment(0); expect(builder.toString()).toEqual('A'); diff --git a/packages/metro-source-map/src/composeSourceMaps.js b/packages/metro-source-map/src/composeSourceMaps.js index c512b8d844..a0e8967c5c 100644 --- a/packages/metro-source-map/src/composeSourceMaps.js +++ b/packages/metro-source-map/src/composeSourceMaps.js @@ -10,11 +10,17 @@ */ import type {SourcePosition} from './Consumer/types'; -import type {IConsumer, MixedSourceMap} from './source-map'; +import type { + FBSourcesArray, + HermesFunctionOffsets, + IConsumer, + MixedSourceMap, +} from './source-map'; import type {Number0, Number1} from 'ob1'; +import B64Builder from './B64Builder'; import Consumer from './Consumer'; -import {SourceMapGenerator} from 'source-map'; +import {get0, get1} from 'ob1'; // TODO(t67648443): Bypass the `sort-requires` rule for this file because of a dependency cycle. Consumer; @@ -41,45 +47,189 @@ export default function composeSourceMaps( }) .reverse(); - const generator = new SourceMapGenerator({ - file: consumers[0].file, - }); + const sources: Array = []; + const names: Array = []; + const sourceIndices: Map = new Map(); + const nameIndices: Map = new Map(); + + // The composed mappings as parallel arrays, in the order the last map + // yields them. An unmapped position has source index -1 and original + // line and column 0. + const generatedLines: Array = []; + const generatedColumns: Array = []; + const mappingSources: Array = []; + const originalLines: Array = []; + const originalColumns: Array = []; + const mappingNames: Array = []; + let sorted = true; const earlierConsumers = consumers.slice(1); consumers[0].eachMapping(mapping => { - const original = - mapping.originalLine == null || mapping.originalColumn == null - ? {line: null, column: null, source: null, name: null} - : earlierConsumers.length === 0 - ? { - line: mapping.originalLine, - column: mapping.originalColumn, - source: mapping.source, - name: mapping.name, - } - : findOriginalPosition( - earlierConsumers, - mapping.originalLine, - mapping.originalColumn, - ); - generator.addMapping({ - generated: { - line: mapping.generatedLine, - column: mapping.generatedColumn, - }, - original: - original.line != null - ? { - line: original.line, - column: original.column, - } - : null, - source: original.source, - name: original.name, - }); + let originalLine: ?number = null; + let originalColumn: ?number = null; + let source: ?string = null; + let name: ?string = null; + const mappingOriginalLine = mapping.originalLine; + const mappingOriginalColumn = mapping.originalColumn; + if (mappingOriginalLine != null && mappingOriginalColumn != null) { + if (earlierConsumers.length === 0) { + originalLine = get1(mappingOriginalLine); + originalColumn = get0(mappingOriginalColumn); + source = mapping.source; + name = mapping.name; + } else { + const original = findOriginalPosition( + earlierConsumers, + mappingOriginalLine, + mappingOriginalColumn, + ); + originalLine = original.line; + originalColumn = original.column; + source = original.source; + name = original.name; + } + } + const generatedLine = get1(mapping.generatedLine); + const generatedColumn = get0(mapping.generatedColumn); + const hasOriginal = originalLine != null; + if (!( + generatedLine > 0 && + generatedColumn >= 0 && + (originalLine != null + ? originalLine > 0 && (originalColumn ?? 0) >= 0 && source + : !source && !name) + )) { + throw new Error( + 'Invalid mapping: ' + + JSON.stringify({ + generated: {line: generatedLine, column: generatedColumn}, + source, + original: hasOriginal + ? {line: originalLine, column: originalColumn} + : null, + name, + }), + ); + } + + const count = generatedLines.length; + if ( + count > 0 && + (generatedLine < generatedLines[count - 1] || + (generatedLine === generatedLines[count - 1] && + generatedColumn < generatedColumns[count - 1])) + ) { + sorted = false; + } + generatedLines.push(generatedLine); + generatedColumns.push(generatedColumn); + if (originalLine != null && source != null) { + let sourceIndex = sourceIndices.get(source); + if (sourceIndex == null) { + sourceIndex = sources.length; + sources.push(source); + sourceIndices.set(source, sourceIndex); + } + mappingSources.push(sourceIndex); + originalLines.push(originalLine); + originalColumns.push(originalColumn ?? 0); + } else { + mappingSources.push(-1); + originalLines.push(0); + originalColumns.push(0); + } + if (name != null) { + let nameIndex = nameIndices.get(name); + if (nameIndex == null) { + nameIndex = names.length; + names.push(name); + nameIndices.set(name, nameIndex); + } + mappingNames.push(nameIndex); + } else { + mappingNames.push(-1); + } }); - const composedMap = generator.toJSON(); + // Orders two composed mappings by generated position, then source, + // original position and name. + const compare = (a: number, b: number): number => + generatedLines[a] - generatedLines[b] || + generatedColumns[a] - generatedColumns[b] || + compareStrings( + mappingSources[a] === -1 ? null : sources[mappingSources[a]], + mappingSources[b] === -1 ? null : sources[mappingSources[b]], + ) || + originalLines[a] - originalLines[b] || + originalColumns[a] - originalColumns[b] || + compareStrings( + mappingNames[a] === -1 ? null : names[mappingNames[a]], + mappingNames[b] === -1 ? null : names[mappingNames[b]], + ); + + const count = generatedLines.length; + let order: ?Array = null; + if (!sorted) { + order = Array.from({length: count}, (_, i) => i); + order.sort(compare); + } + + // Encode the mappings, dropping any identical to the one before it. + const builder = new B64Builder(); + let previousIndex = -1; + let previousGeneratedLine = 1; + let previousGeneratedColumn = 0; + let previousSource = 0; + let previousOriginalLine = 0; + let previousOriginalColumn = 0; + let previousName = 0; + for (let position = 0; position < count; position++) { + const index = order == null ? position : order[position]; + const generatedLine = generatedLines[index]; + if (generatedLine !== previousGeneratedLine) { + builder.markLines(generatedLine - previousGeneratedLine); + previousGeneratedLine = generatedLine; + previousGeneratedColumn = 0; + } else if (previousIndex !== -1 && compare(index, previousIndex) === 0) { + previousIndex = index; + continue; + } + previousIndex = index; + + builder.startSegment(generatedColumns[index] - previousGeneratedColumn); + previousGeneratedColumn = generatedColumns[index]; + const sourceIndex = mappingSources[index]; + if (sourceIndex !== -1) { + builder.append(sourceIndex - previousSource); + previousSource = sourceIndex; + // Original lines are 0-based in the encoding. + builder.append(originalLines[index] - 1 - previousOriginalLine); + previousOriginalLine = originalLines[index] - 1; + builder.append(originalColumns[index] - previousOriginalColumn); + previousOriginalColumn = originalColumns[index]; + const nameIndex = mappingNames[index]; + if (nameIndex !== -1) { + builder.append(nameIndex - previousName); + previousName = nameIndex; + } + } + } + + const composedMap: { + version: number, + sources: Array, + names: Array, + mappings: string, + file?: string, + sourcesContent?: Array, + x_facebook_sources?: FBSourcesArray, + x_hermes_function_offsets?: HermesFunctionOffsets, + x_google_ignoreList?: Array, + } = {version: 3, sources, names, mappings: builder.toString()}; + const {file} = consumers[0]; + if (file != null) { + composedMap.file = file; + } composedMap.sourcesContent = composedMap.sources.map(source => consumers[consumers.length - 1].sourceContentFor(source, true), @@ -103,6 +253,14 @@ export default function composeSourceMaps( return composedMap; } +function compareStrings(a: ?string, b: ?string): number { + if (a === b) { + return 0; + } + // $FlowFixMe[invalid-compare] Matches `source-map`'s ordering, null included + return a > b ? 1 : -1; +} + function findOriginalPosition( consumers: ReadonlyArray, generatedLine: Number1,