diff --git a/packages/metro-config/src/defaults/index.js b/packages/metro-config/src/defaults/index.js index 2d7df87df2..95a4a0ead6 100644 --- a/packages/metro-config/src/defaults/index.js +++ b/packages/metro-config/src/defaults/index.js @@ -135,6 +135,7 @@ const getDefaultValues = (projectRoot: ?string): ConfigT => ({ unstable_disableNormalizePseudoGlobals: false, unstable_renameRequire: true, unstable_compactOutput: false, + unstable_compactSourceMaps: false, unstable_memoizeInlineRequires: false, unstable_workerThreads: false, }, diff --git a/packages/metro-source-map/src/__tests__/source-map-test.js b/packages/metro-source-map/src/__tests__/source-map-test.js index 664163966c..f5499ca5cb 100644 --- a/packages/metro-source-map/src/__tests__/source-map-test.js +++ b/packages/metro-source-map/src/__tests__/source-map-test.js @@ -9,8 +9,17 @@ * @oncall react_native */ +import type {BabelDecodedMap, MetroSourceMapSegmentTuple} from '../source-map'; + import Generator from '../Generator'; -import {fromRawMappings, toBabelSegments, toSegmentTuple} from '../source-map'; +import { + fromRawMappings, + isVlqMap, + toBabelSegments, + toSegmentTuple, + vlqMapFromBabelDecodedMap, + vlqMapFromTuples, +} from '../source-map'; describe('flattening mappings / compacting', () => { test('flattens simple mappings', () => { @@ -167,3 +176,216 @@ describe('build map from raw mappings', () => { }); const lines = (n: number) => Array(n).join('\n'); + +function makeVlqMap( + mappings: string, + names: ReadonlyArray, +): {readonly mappings: string, readonly names: ReadonlyArray} { + return { + mappings, + names, + }; +} + +describe('isVlqMap', () => { + test('returns false for null', () => { + expect(isVlqMap(null)).toBe(false); + }); + + test('returns false for tuple array', () => { + expect(isVlqMap([[1, 2, 3, 4]])).toBe(false); + }); + + test('returns true for VlqMap', () => { + expect(isVlqMap(makeVlqMap('AAAA', []))).toBe(true); + }); + + test('returns false for plain object without string mappings', () => { + // $FlowFixMe[incompatible-type] Testing runtime behavior with invalid type + expect(isVlqMap({mappings: 123, names: []})).toBe(false); + }); +}); + +describe('fromRawMappings with VlqMap', () => { + // Shared tuple definitions. We build two parallel module lists from these — + // one storing decoded tuples, one storing the equivalent VLQ — and assert the + // serialized flat map is byte-identical, i.e. VLQ storage is transparent. + const tuples0: Array = [ + [1, 2], + [3, 4, 5, 6, 'apples'], + [7, 8, 9, 10], + [11, 12, 13, 14, 'pears'], + ]; + const tuples1: Array = [ + [1, 2], + [3, 4, 15, 16, 'bananas'], + ]; + + const tupleModules = [ + { + code: lines(11), + functionMap: {names: [''], mappings: 'AAA'}, + map: tuples0, + source: 'code1', + path: 'path1', + isIgnored: false, + }, + { + code: lines(3), + functionMap: null, + map: tuples1, + source: 'code2', + path: 'path2', + isIgnored: true, + }, + ]; + + const vlqModules = [ + {...tupleModules[0], map: vlqMapFromTuples(tuples0)}, + {...tupleModules[1], map: vlqMapFromTuples(tuples1)}, + ]; + + test('produces a flat (non-indexed) map for VlqMap inputs', () => { + const map = fromRawMappings(vlqModules).toMap(); + expect(typeof map.mappings).toBe('string'); + expect(map.sources).toEqual(['path1', 'path2']); + expect(map.version).toBe(3); + }); + + test('VlqMap input serializes byte-identically to tuple input', () => { + expect(fromRawMappings(vlqModules).toString()).toBe( + fromRawMappings(tupleModules).toString(), + ); + expect(fromRawMappings(vlqModules).toMap()).toEqual( + fromRawMappings(tupleModules).toMap(), + ); + }); + + test('preserves functionMap and ignoreList from VlqMap modules', () => { + const map = fromRawMappings(vlqModules).toMap(); + expect(map.x_facebook_sources).toEqual([ + [{names: [''], mappings: 'AAA'}], + null, + ]); + expect(map.x_google_ignoreList).toEqual([1]); + }); + + test('handles mixed tuple and VlqMap modules identically to all-tuple', () => { + const mixed = [tupleModules[0], vlqModules[1]]; + expect(fromRawMappings(mixed).toString()).toBe( + fromRawMappings(tupleModules).toString(), + ); + }); + + test('applies offsetLines identically for VlqMap and tuple inputs', () => { + expect(fromRawMappings(vlqModules, 8).toString()).toBe( + fromRawMappings(tupleModules, 8).toString(), + ); + }); + + test('excludeSource option omits sourcesContent', () => { + const map = fromRawMappings(vlqModules).toMap(undefined, { + excludeSource: true, + }); + expect(map.sourcesContent).toBeUndefined(); + }); +}); + +describe('vlqMapFromTuples', () => { + // Decode via Metro's existing string->tuples path, the inverse of + // vlqMapFromTuples. + const decode = (vlqMap: { + readonly mappings: string, + readonly names: ReadonlyArray, + }) => + toBabelSegments({ + version: 3, + sources: [''], + names: [...vlqMap.names], + mappings: vlqMap.mappings, + }).map(toSegmentTuple); + + test('encodes tuples into a VlqMap', () => { + const vlqMap = vlqMapFromTuples([ + [1, 2], + [3, 4, 5, 6, 'apples'], + [7, 8, 9, 10], + [11, 12, 13, 14, 'pears'], + ]); + expect(isVlqMap(vlqMap)).toBe(true); + expect(typeof vlqMap.mappings).toBe('string'); + expect(vlqMap.names).toEqual(['apples', 'pears']); + }); + + test('round-trips via toBabelSegments + toSegmentTuple', () => { + const tuples = [ + [1, 2], + [3, 4, 5, 6, 'apples'], + [7, 8, 9, 10], + [11, 12, 13, 14, 'pears'], + [11, 20, 30, 40], + ]; + expect(decode(vlqMapFromTuples(tuples))).toEqual(tuples); + }); + + test('round-trips multi-line, multi-segment maps', () => { + const tuples = [ + [1, 0, 1, 0], + [1, 8, 1, 4, 'foo'], + [2, 0, 2, 0], + [3, 4, 3, 2, 'bar'], + [5, 0], + ]; + expect(decode(vlqMapFromTuples(tuples))).toEqual(tuples); + }); + + test('encodes an empty map', () => { + const vlqMap = vlqMapFromTuples([]); + expect(vlqMap.mappings).toBe(''); + expect(decode(vlqMap)).toEqual([]); + }); +}); + +describe('vlqMapFromBabelDecodedMap', () => { + test('matches vlqMapFromTuples, appending a terminator when needed', () => { + // Decoded format: grouped by generated line (0-based), source lines 0-based. + const decodedMap: BabelDecodedMap = { + names: ['foo'], + mappings: [ + [[0, 0, 0, 0]], // gen 1:0 -> src 1:0 + [[2, 0, 0, 4, 0]], // gen 2:2 -> src 1:4 name 'foo' + [[0]], // gen 3:0 generated-only + ], + }; + // Equivalent Metro tuples (source lines 1-based) + terminator at gen 3:5. + const tuples: Array = [ + [1, 0, 1, 0], + [2, 2, 1, 4, 'foo'], + [3, 0], + [3, 5], + ]; + expect(vlqMapFromBabelDecodedMap(decodedMap, [3, 5])).toEqual( + vlqMapFromTuples(tuples), + ); + }); + + test('does not append a terminator already present', () => { + const decodedMap: BabelDecodedMap = { + names: [], + mappings: [[[0], [5]]], + }; + expect(vlqMapFromBabelDecodedMap(decodedMap, [1, 5])).toEqual( + vlqMapFromTuples([ + [1, 0], + [1, 5], + ]), + ); + }); + + test('handles an empty decoded map (terminator only)', () => { + const decodedMap: BabelDecodedMap = {names: [], mappings: []}; + expect(vlqMapFromBabelDecodedMap(decodedMap, [1, 0])).toEqual( + vlqMapFromTuples([[1, 0]]), + ); + }); +}); diff --git a/packages/metro-source-map/src/source-map.js b/packages/metro-source-map/src/source-map.js index b0f98553e9..6bed2ab9e3 100644 --- a/packages/metro-source-map/src/source-map.js +++ b/packages/metro-source-map/src/source-map.js @@ -21,6 +21,7 @@ import { generateFunctionMap, } from './generateFunctionMap'; import Generator from './Generator'; +import nullthrows from 'nullthrows'; // $FlowFixMe[untyped-import] - source-map import SourceMap from 'source-map'; @@ -53,6 +54,11 @@ export type BabelDecodedMap = { ... }; +export type VlqMap = { + readonly mappings: string, + readonly names: ReadonlyArray, +}; + export type HermesFunctionOffsets = {[number]: ReadonlyArray, ...}; export type FBSourcesArray = ReadonlyArray; @@ -123,18 +129,26 @@ type SourceMapConsumerMapping = { name: ?string, }; +export type RawMappingsModule = { + readonly map: ?ReadonlyArray | VlqMap, + readonly functionMap: ?FBSourceFunctionMap, + readonly path: string, + readonly source: string, + readonly code: string, + readonly isIgnored: boolean, + readonly lineCount?: number, +}; + +function isVlqMap( + map: ?ReadonlyArray | VlqMap, +): implies map is VlqMap { + return map != null && !Array.isArray(map) && typeof map.mappings === 'string'; +} + function fromRawMappingsImpl( isBlocking: boolean, onDone: Generator => void, - modules: ReadonlyArray<{ - readonly map: ?ReadonlyArray, - readonly functionMap: ?FBSourceFunctionMap, - readonly path: string, - readonly source: string, - readonly code: string, - readonly isIgnored: boolean, - readonly lineCount?: number, - }>, + modules: ReadonlyArray, offsetLines: number, ): void { const modulesToProcess = modules.slice(); @@ -146,15 +160,18 @@ function fromRawMappingsImpl( return true; } - const mod = modulesToProcess.shift(); - // $FlowFixMe[incompatible-use] + const mod = nullthrows(modulesToProcess.shift()); const {code, map} = mod; - if (Array.isArray(map)) { - // $FlowFixMe[incompatible-type] + if (isVlqMap(map)) { + // Modules may store their map compactly as VLQ. Decode it back to tuples + // just-in-time so it can be folded into the flat Generator like any other + // module. Decoding one module at a time keeps the transient tuple arrays + // short-lived, preserving the memory win of VLQ storage. + addMappingsForFile(generator, decodeVlqMap(map), mod, carryOver); + } else if (Array.isArray(map)) { addMappingsForFile(generator, map, mod, carryOver); } else if (map != null) { throw new Error( - // $FlowFixMe[incompatible-use] `Unexpected module with full source map found: ${mod.path}`, ); } @@ -197,15 +214,7 @@ function fromRawMappingsImpl( * the resulting bundle, e.g. by some prefix code. */ function fromRawMappings( - modules: ReadonlyArray<{ - readonly map: ?ReadonlyArray, - readonly functionMap: ?FBSourceFunctionMap, - readonly path: string, - readonly source: string, - readonly code: string, - readonly isIgnored: boolean, - readonly lineCount?: number, - }>, + modules: ReadonlyArray, offsetLines: number = 0, ): Generator { let generator: void | Generator; @@ -224,15 +233,7 @@ function fromRawMappings( } async function fromRawMappingsNonBlocking( - modules: ReadonlyArray<{ - readonly map: ?ReadonlyArray, - readonly functionMap: ?FBSourceFunctionMap, - readonly path: string, - readonly source: string, - readonly code: string, - readonly isIgnored: boolean, - readonly lineCount?: number, - }>, + modules: ReadonlyArray, offsetLines: number = 0, ): Promise { return new Promise(resolve => { @@ -344,16 +345,8 @@ function tuplesFromBabelDecodedMap( function addMappingsForFile( generator: Generator, - mappings: Array, - module: { - readonly code: string, - readonly functionMap: ?FBSourceFunctionMap, - readonly map: ?Array, - readonly path: string, - readonly source: string, - readonly isIgnored: boolean, - readonly lineCount?: number, - }, + mappings: ReadonlyArray, + module: RawMappingsModule, carryOver: number, ) { generator.startFile(module.path, module.source, module.functionMap, { @@ -400,6 +393,110 @@ const newline = /\r\n?|\n|\u2028|\u2029/g; const countLines = (string: string): number => (string.match(newline) || []).length + 1; +/** + * Decodes a compact VLQ map back into raw mapping tuples — the inverse of + * `vlqMapFromTuples`, reusing Metro's existing source-map consumer. + */ +function decodeVlqMap(vlqMap: VlqMap): Array { + return toBabelSegments({ + version: 3, + sources: [''], + names: [...vlqMap.names], + mappings: vlqMap.mappings, + }).map(toSegmentTuple); +} + +/** + * Encodes raw mapping tuples into a compact VLQ `mappings` string + `names` + * table. Decode the inverse via `decodeVlqMap` (or `toBabelSegments` + + * `toSegmentTuple`). Storing maps in this form uses far less memory than the + * equivalent decoded tuple arrays. + */ +function vlqMapFromTuples( + mappings: ReadonlyArray, +): VlqMap { + const generator = new Generator(); + generator.startFile('', '', null); + for (const mapping of mappings) { + addMapping(generator, mapping, 0); + } + generator.endFile(); + const map = generator.toMap(); + return {mappings: map.mappings, names: map.names}; +} + +/** + * Encodes a `VlqMap` directly from a Babel/gen-mapping "decoded" source map + * (`result.decodedMap` from `@babel/generator`), without ever materialising the + * intermediate `Array`. + * + * `@babel/generator` computes `decodedMap` eagerly while generating, so reusing + * it avoids the separate, more expensive `result.rawMappings` decode (which + * allocates a flat array of segment objects) plus the per-segment tuple + * allocation that `vlqMapFromTuples` would otherwise consume. The result is + * byte-identical to `vlqMapFromTuples(decoded -> tuples)`. + * + * `terminatingMapping` is a `[generatedLine1Based, generatedColumn0Based]` + * generated-only mapping appended at the end (matching the transform worker's + * `countLinesAndTerminateMap`) unless the last real mapping already sits there. + */ +function vlqMapFromBabelDecodedMap( + decodedMap: BabelDecodedMap, + terminatingMapping: [number, number], +): VlqMap { + const generator = new Generator(); + generator.startFile('', '', null); + const {mappings, names} = decodedMap; + let lastGeneratedLine = -1; + let lastGeneratedColumn = -1; + for (let line = 0, n = mappings.length; line < n; ++line) { + // Decoded mappings are grouped by generated line (0-based); Generator + // expects 1-based generated lines. + const generatedLine = line + 1; + const segments = mappings[line]; + for (let i = 0, m = segments.length; i < m; ++i) { + const segment = segments[i]; + const generatedColumn = segment[0]; + switch (segment.length) { + case 1: + generator.addSimpleMapping(generatedLine, generatedColumn); + break; + case 4: + // Decoded source lines are 0-based; Generator expects 1-based. + generator.addSourceMapping( + generatedLine, + generatedColumn, + segment[2] + 1, + segment[3], + ); + break; + case 5: + generator.addNamedSourceMapping( + generatedLine, + generatedColumn, + segment[2] + 1, + segment[3], + names[segment[4]], + ); + break; + default: + throw new Error(`Invalid mapping: [${segment.join(', ')}]`); + } + lastGeneratedLine = generatedLine; + lastGeneratedColumn = generatedColumn; + } + } + if ( + lastGeneratedLine !== terminatingMapping[0] || + lastGeneratedColumn !== terminatingMapping[1] + ) { + generator.addSimpleMapping(terminatingMapping[0], terminatingMapping[1]); + } + generator.endFile(); + const map = generator.toMap(); + return {mappings: map.mappings, names: map.names}; +} + export { BundleBuilder, composeSourceMaps, @@ -409,10 +506,13 @@ export { fromRawMappings, fromRawMappingsNonBlocking, functionMapBabelPlugin, + isVlqMap, normalizeSourcePath, toBabelSegments, toSegmentTuple, tuplesFromBabelDecodedMap, + vlqMapFromBabelDecodedMap, + vlqMapFromTuples, }; /** diff --git a/packages/metro-source-map/types/source-map.d.ts b/packages/metro-source-map/types/source-map.d.ts index e7f58e0c6c..dcc9aec967 100644 --- a/packages/metro-source-map/types/source-map.d.ts +++ b/packages/metro-source-map/types/source-map.d.ts @@ -6,7 +6,7 @@ * * @noformat * @oncall react_native - * @generated SignedSource<> + * @generated SignedSource<<9ec89353742743678e422f0bf81e488d>> * * This file was translated from Flow by scripts/generateTypeScriptDefinitions.js * Original file: packages/metro-source-map/src/source-map.js @@ -43,6 +43,10 @@ export type BabelDecodedMap = { readonly mappings: ReadonlyArray>; readonly names: ReadonlyArray; }; +export type VlqMap = { + readonly mappings: string; + readonly names: ReadonlyArray; +}; export type HermesFunctionOffsets = { [$$Key$$: number]: ReadonlyArray; }; @@ -92,6 +96,20 @@ export type IndexMap = { readonly x_google_ignoreList?: void; }; export type MixedSourceMap = IndexMap | BasicSourceMap; +export type RawMappingsModule = { + readonly map: + | (null | undefined | ReadonlyArray) + | VlqMap; + readonly functionMap: null | undefined | FBSourceFunctionMap; + readonly path: string; + readonly source: string; + readonly code: string; + readonly isIgnored: boolean; + readonly lineCount?: number; +}; +declare function isVlqMap( + map: (null | undefined | ReadonlyArray) | VlqMap, +): map is VlqMap; /** * Creates a source map from modules with "raw mappings", i.e. an array of * tuples with either 2, 4, or 5 elements: @@ -100,27 +118,11 @@ export type MixedSourceMap = IndexMap | BasicSourceMap; * the resulting bundle, e.g. by some prefix code. */ declare function fromRawMappings( - modules: ReadonlyArray<{ - readonly map: null | undefined | ReadonlyArray; - readonly functionMap: null | undefined | FBSourceFunctionMap; - readonly path: string; - readonly source: string; - readonly code: string; - readonly isIgnored: boolean; - readonly lineCount?: number; - }>, + modules: ReadonlyArray, offsetLines?: number, ): Generator; declare function fromRawMappingsNonBlocking( - modules: ReadonlyArray<{ - readonly map: null | undefined | ReadonlyArray; - readonly functionMap: null | undefined | FBSourceFunctionMap; - readonly path: string; - readonly source: string; - readonly code: string; - readonly isIgnored: boolean; - readonly lineCount?: number; - }>, + modules: ReadonlyArray, offsetLines?: number, ): Promise; /** @@ -146,6 +148,34 @@ declare function toSegmentTuple( declare function tuplesFromBabelDecodedMap( decodedMap: BabelDecodedMap, ): Array; +/** + * Encodes raw mapping tuples into a compact VLQ `mappings` string + `names` + * table. Decode the inverse via `decodeVlqMap` (or `toBabelSegments` + + * `toSegmentTuple`). Storing maps in this form uses far less memory than the + * equivalent decoded tuple arrays. + */ +declare function vlqMapFromTuples( + mappings: ReadonlyArray, +): VlqMap; +/** + * Encodes a `VlqMap` directly from a Babel/gen-mapping "decoded" source map + * (`result.decodedMap` from `@babel/generator`), without ever materialising the + * intermediate `Array`. + * + * `@babel/generator` computes `decodedMap` eagerly while generating, so reusing + * it avoids the separate, more expensive `result.rawMappings` decode (which + * allocates a flat array of segment objects) plus the per-segment tuple + * allocation that `vlqMapFromTuples` would otherwise consume. The result is + * byte-identical to `vlqMapFromTuples(decoded -> tuples)`. + * + * `terminatingMapping` is a `[generatedLine1Based, generatedColumn0Based]` + * generated-only mapping appended at the end (matching the transform worker's + * `countLinesAndTerminateMap`) unless the last real mapping already sits there. + */ +declare function vlqMapFromBabelDecodedMap( + decodedMap: BabelDecodedMap, + terminatingMapping: [number, number], +): VlqMap; export { BundleBuilder, composeSourceMaps, @@ -155,10 +185,13 @@ export { fromRawMappings, fromRawMappingsNonBlocking, functionMapBabelPlugin, + isVlqMap, normalizeSourcePath, toBabelSegments, toSegmentTuple, tuplesFromBabelDecodedMap, + vlqMapFromBabelDecodedMap, + vlqMapFromTuples, }; /** * Backwards-compatibility with CommonJS consumers using interopRequireDefault. diff --git a/packages/metro-transform-worker/src/__tests__/index-test.js b/packages/metro-transform-worker/src/__tests__/index-test.js index 81aba4c9f0..b55466d83c 100644 --- a/packages/metro-transform-worker/src/__tests__/index-test.js +++ b/packages/metro-transform-worker/src/__tests__/index-test.js @@ -32,6 +32,8 @@ import type {JsTransformerConfig, JsTransformOptions} from '../index'; import typeof * as TransformerType from '../index'; import typeof FSType from 'fs'; +import {vlqMapFromTuples} from 'metro-source-map'; + const {Buffer} = require('buffer'); const path = require('path'); @@ -409,6 +411,56 @@ test('uses a reserved dependency map name and prevents it from being minified', `); }); +test('unstable_compactSourceMaps emits a VlqMap byte-identical to the tuple path', async () => { + const source = Buffer.from( + [ + 'function foo(aaa, bbb) {', + ' const ccc = aaa + bbb;', + ' return ccc * 2;', + '}', + 'export default function entry(items) {', + ' return items.map(x => x.value).filter(Boolean);', + '}', + '', + ].join('\n'), + 'utf8', + ); + + // Default path stores decoded tuples (line-counted + terminated). + const tupleResult = await Transformer.transform( + {...baseConfig, unstable_compactSourceMaps: false}, + '/root', + 'local/file.js', + source, + {...baseTransformOptions, experimentalImportSupport: true}, + ); + // Compact path encodes VLQ straight from Babel's decoded map (no tuples). + const vlqResult = await Transformer.transform( + {...baseConfig, unstable_compactSourceMaps: true}, + '/root', + 'local/file.js', + source, + {...baseTransformOptions, experimentalImportSupport: true}, + ); + + const tupleMap = tupleResult.output[0].data.map; + const vlqMap = vlqResult.output[0].data.map; + + // Generated code and line count are unaffected by map storage. + expect(vlqResult.output[0].data.code).toBe(tupleResult.output[0].data.code); + expect(vlqResult.output[0].data.lineCount).toBe( + tupleResult.output[0].data.lineCount, + ); + + if (Array.isArray(vlqMap) || !Array.isArray(tupleMap)) { + throw new Error('Expected a VlqMap (compact) and a tuple array (default)'); + } + // The compact fast path is byte-identical to re-encoding the tuple output. + expect(vlqMap).toEqual(vlqMapFromTuples(tupleMap)); + expect(typeof vlqMap.mappings).toBe('string'); + expect(vlqMap.mappings.length).toBeGreaterThan(0); +}); + test('throws if the reserved dependency map name appears in the input', async () => { await expect( Transformer.transform( diff --git a/packages/metro-transform-worker/src/index.js b/packages/metro-transform-worker/src/index.js index ea93e87de4..9e57da874a 100644 --- a/packages/metro-transform-worker/src/index.js +++ b/packages/metro-transform-worker/src/index.js @@ -20,6 +20,7 @@ import type { BasicSourceMap, FBSourceFunctionMap, MetroSourceMapSegmentTuple, + VlqMap, } from 'metro-source-map'; import type { ImportExportPluginOptions, @@ -47,6 +48,8 @@ import { toBabelSegments, toSegmentTuple, tuplesFromBabelDecodedMap, + vlqMapFromBabelDecodedMap, + vlqMapFromTuples, } from 'metro-source-map'; import metroTransformPlugins from 'metro-transform-plugins'; import collectDependencies from 'metro/private/ModuleGraph/worker/collectDependencies'; @@ -111,6 +114,8 @@ export type JsTransformerConfig = Readonly<{ unstable_nonMemoizedInlineRequires?: ReadonlyArray, /** Whether to rename scoped `require` functions to `_$$_REQUIRE`, usually an extraneous operation when serializing to iife (default). */ unstable_renameRequire?: boolean, + /** Store source maps as compact VLQ-encoded strings (`VlqMap`) instead of decoded tuple arrays. Reduces source-map memory ~51% on the heap. Opt-in; changes `JsOutput.data.map` for consumers. */ + unstable_compactSourceMaps?: boolean, }>; export type {CustomTransformOptions} from 'metro-babel-transformer'; @@ -169,7 +174,7 @@ export type JsOutput = Readonly<{ data: Readonly<{ code: string, lineCount: number, - map: Array, + map: Array | VlqMap, functionMap: ?FBSourceFunctionMap, }>, type: JSFileType, @@ -472,29 +477,49 @@ async function transformJS( file.code, ); - // Derive tuples from Babel's eagerly-computed decoded map rather than - // `result.rawMappings`, which would trigger a second, more expensive decode - // (`allMappings`). Byte-identical to `result.rawMappings.map(toSegmentTuple)`. - let map = result.decodedMap - ? tuplesFromBabelDecodedMap(result.decodedMap) - : []; let code = result.code; + let map: Array | VlqMap; + let lineCount: number; + + if (config.unstable_compactSourceMaps === true && !minify) { + // Dominant path (e.g. Hermes, which doesn't minify): encode the compact VLQ + // map straight from Babel's eagerly-computed decoded map, never + // materialising tuples. Byte-identical to the tuple path below. + const {lineCount: lines, lastLineColumn} = countLines(code); + lineCount = lines; + map = vlqMapFromBabelDecodedMap( + result.decodedMap ?? {mappings: [], names: []}, + [lines, lastLineColumn], + ); + } else { + // Derive tuples from Babel's eagerly-computed decoded map rather than + // `result.rawMappings`, which would trigger a second, more expensive decode + // (`allMappings`). Byte-identical to `result.rawMappings.map(toSegmentTuple)`. + let tuples = result.decodedMap + ? tuplesFromBabelDecodedMap(result.decodedMap) + : []; + + if (minify) { + // The minifier returns its own map (not Babel's `decodedMap`), so the + // fast path above can't apply; re-encode the resulting tuples if compact. + ({map: tuples, code} = await minifyCode( + config, + projectRoot, + file.filename, + result.code, + file.code, + tuples, + reserved, + )); + } - if (minify) { - ({map, code} = await minifyCode( - config, - projectRoot, - file.filename, - result.code, - file.code, - map, - reserved, - )); + ({lineCount, map: tuples} = countLinesAndTerminateMap(code, tuples)); + map = + config.unstable_compactSourceMaps === true + ? vlqMapFromTuples(tuples) + : tuples; } - let lineCount; - ({lineCount, map} = countLinesAndTerminateMap(code, map)); - const output: Array = [ { data: { @@ -622,9 +647,13 @@ async function transformJSON( let lineCount; ({lineCount, map} = countLinesAndTerminateMap(code, map)); + // The JSON path builds tuples directly (no Babel `decodedMap`), so when + // compact we re-encode the finished tuples to VLQ. + const outputMap = + config.unstable_compactSourceMaps === true ? vlqMapFromTuples(map) : map; const output: Array = [ { - data: {code, functionMap: null, lineCount, map}, + data: {code, functionMap: null, lineCount, map: outputMap}, type: jsType, }, ]; @@ -761,12 +790,9 @@ export const getCacheKey = ( ].join('$'); }; -function countLinesAndTerminateMap( - code: string, - map: ReadonlyArray, -): { +function countLines(code: string): { lineCount: number, - map: Array, + lastLineColumn: number, } { const NEWLINE = /\r\n?|\n|\u2028|\u2029/g; let lineCount = 1; @@ -777,9 +803,19 @@ function countLinesAndTerminateMap( lineCount++; lastLineStart = match.index + match[0].length; } - const lastLineLength = code.length - lastLineStart; + return {lineCount, lastLineColumn: code.length - lastLineStart}; +} + +function countLinesAndTerminateMap( + code: string, + map: ReadonlyArray, +): { + lineCount: number, + map: Array, +} { + const {lineCount, lastLineColumn} = countLines(code); const lastLineIndex1Based = lineCount; - const lastLineNextColumn0Based = lastLineLength; + const lastLineNextColumn0Based = lastLineColumn; // If there isn't a mapping at one-past-the-last column of the last line, // add one that maps to nothing. This ensures out-of-bounds lookups hit the diff --git a/packages/metro-transform-worker/types/index.d.ts b/packages/metro-transform-worker/types/index.d.ts index ce18c8f5a5..cd3e2f681f 100644 --- a/packages/metro-transform-worker/types/index.d.ts +++ b/packages/metro-transform-worker/types/index.d.ts @@ -6,7 +6,7 @@ * * @noformat * @oncall react_native - * @generated SignedSource<<165882da0b131608da36b1cbd00ecf28>> + * @generated SignedSource<<9cf6eca6abe0d86fd41c697473f45aff>> * * This file was translated from Flow by scripts/generateTypeScriptDefinitions.js * Original file: packages/metro-transform-worker/src/index.js @@ -23,6 +23,7 @@ import type { BasicSourceMap, FBSourceFunctionMap, MetroSourceMapSegmentTuple, + VlqMap, } from 'metro-source-map'; import type {TransformResultDependency} from 'metro/private/DeltaBundler'; import type {AllowOptionalDependencies} from 'metro/private/DeltaBundler/types'; @@ -68,6 +69,8 @@ export type JsTransformerConfig = Readonly<{ unstable_nonMemoizedInlineRequires?: ReadonlyArray; /** Whether to rename scoped `require` functions to `_$$_REQUIRE`, usually an extraneous operation when serializing to iife (default). */ unstable_renameRequire?: boolean; + /** Store source maps as compact VLQ-encoded strings (`VlqMap`) instead of decoded tuple arrays. Reduces source-map memory ~51% on the heap. Opt-in; changes `JsOutput.data.map` for consumers. */ + unstable_compactSourceMaps?: boolean; }>; export type {CustomTransformOptions} from 'metro-babel-transformer'; export type JsTransformOptions = Readonly<{ @@ -90,7 +93,7 @@ export type JsOutput = Readonly<{ data: Readonly<{ code: string; lineCount: number; - map: Array; + map: Array | VlqMap; functionMap: null | undefined | FBSourceFunctionMap; }>; type: JSFileType; diff --git a/packages/metro/src/DeltaBundler/Serializers/getExplodedSourceMap.js b/packages/metro/src/DeltaBundler/Serializers/getExplodedSourceMap.js index 45c88a83a8..b1d301f3c2 100644 --- a/packages/metro/src/DeltaBundler/Serializers/getExplodedSourceMap.js +++ b/packages/metro/src/DeltaBundler/Serializers/getExplodedSourceMap.js @@ -13,12 +13,13 @@ import type {Module} from '../types'; import type { FBSourceFunctionMap, MetroSourceMapSegmentTuple, + VlqMap, } from 'metro-source-map'; import {getJsOutput, isJsModule} from './helpers/js'; export type ExplodedSourceMap = ReadonlyArray<{ - readonly map: Array, + readonly map: Array | VlqMap, readonly firstLine1Based: number, readonly functionMap: ?FBSourceFunctionMap, readonly path: string, diff --git a/packages/metro/src/DeltaBundler/Serializers/helpers/getSourceMapInfo.js b/packages/metro/src/DeltaBundler/Serializers/helpers/getSourceMapInfo.js index d70aa79ec1..8513ee4e7b 100644 --- a/packages/metro/src/DeltaBundler/Serializers/helpers/getSourceMapInfo.js +++ b/packages/metro/src/DeltaBundler/Serializers/helpers/getSourceMapInfo.js @@ -13,6 +13,7 @@ import type {Module} from '../../types'; import type { FBSourceFunctionMap, MetroSourceMapSegmentTuple, + VlqMap, } from 'metro-source-map'; import {getJsOutput} from './js'; @@ -25,7 +26,7 @@ export default function getSourceMapInfo( getSourceUrl: ?(module: Module<>) => string, }, ): { - readonly map: Array, + readonly map: Array | VlqMap, readonly functionMap: ?FBSourceFunctionMap, readonly code: string, readonly path: string, diff --git a/packages/metro/src/Server/__tests__/symbolicate-test.js b/packages/metro/src/Server/__tests__/symbolicate-test.js new file mode 100644 index 0000000000..38e26e3892 --- /dev/null +++ b/packages/metro/src/Server/__tests__/symbolicate-test.js @@ -0,0 +1,101 @@ +/** + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + * + * @flow strict-local + * @format + * @oncall react_native + */ + +import type {ExplodedSourceMap} from '../../DeltaBundler/Serializers/getExplodedSourceMap'; +import type {InputConfigT} from 'metro-config'; +import type {MetroSourceMapSegmentTuple, VlqMap} from 'metro-source-map'; + +import symbolicate from '../symbolicate'; +import {getDefaultConfig, mergeConfig} from 'metro-config'; +import {vlqMapFromTuples} from 'metro-source-map'; + +// symbolicate() only reads `config.symbolicator`. Stub metro-config so the test +// stays independent of the full config pipeline (and the Node version it needs). +jest.mock('metro-config', () => ({ + getDefaultConfig: {getDefaultValues: () => ({})}, + mergeConfig: (base, override) => ({...base, ...override}), +})); + +const config = mergeConfig(getDefaultConfig.getDefaultValues('/'), { + symbolicator: { + customizeFrame: () => null, + customizeStack: stack => stack, + }, +} as InputConfigT); + +// genLine1Based, genCol0Based, srcLine1Based, srcCol0Based[, name] +const TUPLES: Array = [ + [1, 0, 10, 4], + [1, 8, 10, 12, 'greet'], + [2, 0, 11, 0], +]; + +function makeMap( + map: Array | VlqMap, +): ExplodedSourceMap { + return [ + { + firstLine1Based: 1, + functionMap: null, + map, + path: 'foo.js', + }, + ]; +} + +test('symbolicates a frame against a decoded tuple map', async () => { + const [frame] = await symbolicate( + [{file: 'bundle.js', lineNumber: 1, column: 8, methodName: null}], + [['bundle.js', makeMap(TUPLES)]], + config, + null, + ); + expect(frame).toMatchObject({file: 'foo.js', lineNumber: 10, column: 12}); +}); + +test('VLQ map symbolicates identically to its decoded tuples', async () => { + const frame = [ + {file: 'bundle.js', lineNumber: 1, column: 8, methodName: null}, + ]; + + const [fromTuples] = await symbolicate( + frame, + [['bundle.js', makeMap(TUPLES)]], + config, + null, + ); + const [fromVlq] = await symbolicate( + frame, + [['bundle.js', makeMap(vlqMapFromTuples(TUPLES))]], + config, + null, + ); + + expect(fromVlq).toEqual(fromTuples); + expect(fromVlq).toMatchObject({file: 'foo.js', lineNumber: 10, column: 12}); +}); + +test('reuses a single VLQ map across multiple frames in the same module', async () => { + const explodedMap = makeMap(vlqMapFromTuples(TUPLES)); + + const out = await symbolicate( + [ + {file: 'bundle.js', lineNumber: 1, column: 0, methodName: null}, + {file: 'bundle.js', lineNumber: 1, column: 8, methodName: null}, + {file: 'bundle.js', lineNumber: 2, column: 0, methodName: null}, + ], + [['bundle.js', explodedMap]], + config, + null, + ); + + expect(out.map(f => f.lineNumber)).toEqual([10, 10, 11]); +}); diff --git a/packages/metro/src/Server/symbolicate.js b/packages/metro/src/Server/symbolicate.js index b219d39661..9452367dca 100644 --- a/packages/metro/src/Server/symbolicate.js +++ b/packages/metro/src/Server/symbolicate.js @@ -12,10 +12,12 @@ import type { FBSourceFunctionMap, MetroSourceMapSegmentTuple, + VlqMap, } from '../../../metro-source-map/src/source-map'; import type {ExplodedSourceMap} from '../DeltaBundler/Serializers/getExplodedSourceMap'; import type {ConfigT} from 'metro-config'; +import {toBabelSegments, toSegmentTuple} from 'metro-source-map'; import {greatestLowerBound} from 'metro-source-map/private/Consumer/search'; import {SourceMetadataMapConsumer} from 'metro-symbolicate/private/Symbolication'; @@ -35,6 +37,26 @@ export type StackFrameOutput = Readonly; type ExplodedSourceMapModule = ExplodedSourceMap[number]; type Position = {readonly line1Based: number, column0Based: number}; +function ensureDecodedMap( + map: Array | VlqMap, + decodedMapCache: Map>, +): Array { + if (Array.isArray(map)) { + return map; + } + let decoded = decodedMapCache.get(map); + if (decoded == null) { + decoded = toBabelSegments({ + version: 3, + sources: [''], + names: [...map.names], + mappings: map.mappings, + }).map(toSegmentTuple); + decodedMapCache.set(map, decoded); + } + return decoded; +} + function createFunctionNameGetter( module: ExplodedSourceMapModule, ): Position => ?string { @@ -70,12 +92,19 @@ export default async function symbolicate( { readonly firstLine1Based: number, readonly functionMap: ?FBSourceFunctionMap, - readonly map: Array, + readonly map: Array | VlqMap, readonly path: string, }, (Position) => ?string, >(); + // Decoded VLQ maps are cached only for the duration of this request, then + // discarded. The cache dedupes decoding across frames that resolve to the + // same module, while keeping the (large) decoded tuples short-lived — the + // VlqMaps themselves are retained by the long-lived module graph, so caching + // beyond request scope would defeat the memory savings of storing them as VLQ. + const decodedMapCache = new Map>(); + function findModule(frame: StackFrameInput): ?ExplodedSourceMapModule { const map = mapsByUrl.get(frame.file); if (!map || frame.lineNumber == null) { @@ -96,19 +125,18 @@ export default async function symbolicate( frame: StackFrameInput, module: ExplodedSourceMapModule, ): ?Position { - if ( - module.map == null || - frame.lineNumber == null || - frame.column == null - ) { + const lineNumber = frame.lineNumber; + const column = frame.column; + if (module.map == null || lineNumber == null || column == null) { return null; } + const decodedMap = ensureDecodedMap(module.map, decodedMapCache); const generatedPosInModule = { - line1Based: frame.lineNumber - module.firstLine1Based + 1, - column0Based: frame.column, + line1Based: lineNumber - module.firstLine1Based + 1, + column0Based: column, }; const mappingIndex = greatestLowerBound( - module.map, + decodedMap, generatedPosInModule, (target, candidate) => { if (target.line1Based === candidate[0]) { @@ -120,7 +148,7 @@ export default async function symbolicate( if (mappingIndex == null) { return null; } - const mapping = module.map[mappingIndex]; + const mapping = decodedMap[mappingIndex]; if ( mapping[0] !== generatedPosInModule.line1Based || mapping.length < 4 /* no source line/column info */ @@ -140,7 +168,7 @@ export default async function symbolicate( module: { readonly firstLine1Based: number, readonly functionMap: ?FBSourceFunctionMap, - readonly map: Array, + readonly map: Array | VlqMap, readonly path: string, }, ): ?string { @@ -160,11 +188,6 @@ export default async function symbolicate( if (!module) { return {...frame}; } - if (!Array.isArray(module.map)) { - throw new Error( - `Unexpected module with serialized source map found: ${module.path}`, - ); - } const originalPos = findOriginalPos(frame, module); if (!originalPos) { return {...frame}; diff --git a/packages/metro/types/DeltaBundler/Serializers/getExplodedSourceMap.d.ts b/packages/metro/types/DeltaBundler/Serializers/getExplodedSourceMap.d.ts index a9d62b738d..5f4804a75b 100644 --- a/packages/metro/types/DeltaBundler/Serializers/getExplodedSourceMap.d.ts +++ b/packages/metro/types/DeltaBundler/Serializers/getExplodedSourceMap.d.ts @@ -6,7 +6,7 @@ * * @noformat * @oncall react_native - * @generated SignedSource<<623892927b76c4f68802bb69f19d9974>> + * @generated SignedSource<<2f0ab0435f64798986366df74674d02a>> * * This file was translated from Flow by scripts/generateTypeScriptDefinitions.js * Original file: packages/metro/src/DeltaBundler/Serializers/getExplodedSourceMap.js @@ -19,10 +19,11 @@ import type {Module} from '../types'; import type { FBSourceFunctionMap, MetroSourceMapSegmentTuple, + VlqMap, } from 'metro-source-map'; export type ExplodedSourceMap = ReadonlyArray<{ - readonly map: Array; + readonly map: Array | VlqMap; readonly firstLine1Based: number; readonly functionMap: null | undefined | FBSourceFunctionMap; readonly path: string; diff --git a/packages/metro/types/DeltaBundler/Serializers/helpers/getSourceMapInfo.d.ts b/packages/metro/types/DeltaBundler/Serializers/helpers/getSourceMapInfo.d.ts index e154274d74..b30c4224f8 100644 --- a/packages/metro/types/DeltaBundler/Serializers/helpers/getSourceMapInfo.d.ts +++ b/packages/metro/types/DeltaBundler/Serializers/helpers/getSourceMapInfo.d.ts @@ -6,7 +6,7 @@ * * @noformat * @oncall react_native - * @generated SignedSource<> + * @generated SignedSource<> * * This file was translated from Flow by scripts/generateTypeScriptDefinitions.js * Original file: packages/metro/src/DeltaBundler/Serializers/helpers/getSourceMapInfo.js @@ -19,6 +19,7 @@ import type {Module} from '../../types'; import type { FBSourceFunctionMap, MetroSourceMapSegmentTuple, + VlqMap, } from 'metro-source-map'; declare function getSourceMapInfo( @@ -29,7 +30,7 @@ declare function getSourceMapInfo( getSourceUrl: null | undefined | ((module: Module) => string); }, ): { - readonly map: Array; + readonly map: Array | VlqMap; readonly functionMap: null | undefined | FBSourceFunctionMap; readonly code: string; readonly path: string;