From 3c9a3bae87439d35a9bd2de8efaf24138aaff939 Mon Sep 17 00:00:00 2001 From: luojiyin Date: Mon, 28 Sep 2026 13:46:46 +0800 Subject: [PATCH] refactor: record indented code mappings during parsing --- scripts/bench-source-map.mjs | 60 ------ src/source-map/build-source-map.ts | 17 +- src/source-map/code-segments.ts | 283 -------------------------- src/source-map/recording-extension.ts | 58 +++++- 4 files changed, 53 insertions(+), 365 deletions(-) delete mode 100644 src/source-map/code-segments.ts diff --git a/scripts/bench-source-map.mjs b/scripts/bench-source-map.mjs index ac66097..2e1d062 100644 --- a/scripts/bench-source-map.mjs +++ b/scripts/bench-source-map.mjs @@ -12,24 +12,10 @@ // Smoke (CI): node scripts/bench-source-map.mjs --smoke import { performance } from 'node:perf_hooks'; import { createRequire } from 'node:module'; -import { build } from 'esbuild'; const require = createRequire(import.meta.url); const { parseMdWithSourceMap } = require('../dist/lint-md-parser.cjs'); -// Build the internal entry in memory. This keeps benchmark-only exports out of -// the package bundle. -const [{ text: codeSegmentsModule }] = (await build({ - entryPoints: ['src/source-map/code-segments.ts'], - bundle: true, - format: 'esm', - platform: 'node', - write: false, -})).outputFiles; -const { buildCodeSegments } = await import( - `data:text/javascript;base64,${Buffer.from(codeSegmentsModule).toString('base64')}` -); - const UNIT = '&\\('; // one character reference + one escape => 2 segments const SMOKE = process.argv.includes('--smoke'); // On the 256 KiB alternating-atomic input, the binary-search lookup finishes @@ -41,10 +27,6 @@ const URL_BUILD_SIZES_KIB = [16, 32, 64, 128, 256, 512]; // search grows beyond 6× from 128 KiB to 512 KiB; this larger interval makes // the quadratic term dominate while leaving headroom for CI scheduling noise. const SMOKE_URL_BUILD_RATIO_MAX = 6; -const INDENTED_CODE_SIZES_KIB = [1, 16, 64, 256]; -// A 4x prefix grows close to 4x after the suffix lookup becomes linear. The -// old repeated-slice search grows close to 16x. -const SMOKE_INDENTED_CODE_RATIO_MAX = 10; /** Build an input of roughly `kib` kibibytes made of repeated UNIT. */ function makeInput(kib) { @@ -58,22 +40,6 @@ function makeUrlInput(kib) { return '[x](' + '&'.repeat(kib * 1024) + ')'; } -/** Build the fallback shape produced by a tab-based list continuation. */ -function makeIndentedCodeFallbackFixture(kib) { - const prefixLength = kib * 1024; - const md = `${' \t'.repeat(Math.ceil(prefixLength / 2)).slice(0, prefixLength)}x`; - return { - md, - node: { - value: 'x', - position: { - start: { line: 1, column: prefixLength + 1, offset: prefixLength }, - end: { line: 1, column: prefixLength + 2, offset: md.length }, - }, - }, - }; -} - /** The first (and only) text node of the parsed document. */ function firstTextNode(root) { let found; @@ -228,29 +194,3 @@ if (SMOKE) { ); } } - -console.log('\n=== Indented code fallback construction ==='); -const indentedCodeResults = []; -for (const kib of INDENTED_CODE_SIZES_KIB) { - const { md, node } = makeIndentedCodeFallbackFixture(kib); - const result = timeMedian(`${kib} KiB whitespace prefix`, () => { - const mapping = buildCodeSegments(md, node); - if (!mapping) - throw new Error('indented code benchmark did not produce a mapping'); - }); - indentedCodeResults.push(result); - console.log(` ${result.label.padEnd(28)} ${fmt(result.ms)}`); -} - -if (SMOKE) { - const smaller = indentedCodeResults[2]; // 64 KiB - const larger = indentedCodeResults[3]; // 256 KiB - const ratio = larger.ms / smaller.ms; - console.log(` ${'64 → 256 KiB growth'.padEnd(28)} ${ratio.toFixed(2)}x`); - if (ratio > SMOKE_INDENTED_CODE_RATIO_MAX) { - throw new Error( - `benchmark smoke failed: indented code construction grew ${ratio.toFixed(2)}x ` - + `from 64 KiB to 256 KiB (budget ${SMOKE_INDENTED_CODE_RATIO_MAX}x)`, - ); - } -} diff --git a/src/source-map/build-source-map.ts b/src/source-map/build-source-map.ts index 813244b..fa522cb 100644 --- a/src/source-map/build-source-map.ts +++ b/src/source-map/build-source-map.ts @@ -14,7 +14,6 @@ import type { PositionedMarkdownRoot, } from '../types'; import { getParserExtensions } from '../remark-config'; -import { buildCodeSegments } from './code-segments'; import { SourceMapConsistencyError, SourceMapUnavailableError, @@ -42,7 +41,7 @@ interface RecordingState { segments: WeakMap /** inlineCode node -> parser-recorded value segments. */ inlineCodeSegments: WeakMap - /** code node -> parser-recorded or indented-code value segments. */ + /** code node -> parser-recorded value segments. */ codeSegments: WeakMap /** code node -> source point for an empty value. */ emptyCodeOffsets: WeakMap @@ -471,20 +470,6 @@ export const parseMdWithSourceMap = (md: string): ParsedMarkdownDocument => { const sourceGapPrefixes = new WeakMap(); function indexNode(node: TraversableNode): void { - if ( - node.type === 'code' - && hasStringField(node, 'value') - && !state.codeSegments.has(node) - ) { - const mapping = buildCodeSegments(md, node); - if (mapping) { - state.codeSegments.set(node, mapping.segments); - if (mapping.emptyOffset !== undefined) { - state.emptyCodeOffsets.set(node, mapping.emptyOffset); - } - } - } - if ( (node.type === 'link' || node.type === 'definition') && hasStringField(node, 'url') diff --git a/src/source-map/code-segments.ts b/src/source-map/code-segments.ts deleted file mode 100644 index cc459a2..0000000 --- a/src/source-map/code-segments.ts +++ /dev/null @@ -1,283 +0,0 @@ -import type { ParsedPosition } from '../types'; -import type { MarkdownSourceMapSegment, SourceSpan } from './types'; - -interface CodeSegments { - segments: MarkdownSourceMapSegment[] - emptyOffset?: number -} - -function lineEnd(md: string, start: number, limit: number): number { - let offset = start; - while (offset < limit) { - const char = md.charCodeAt(offset); - if (char === 13) - return offset + (md.charCodeAt(offset + 1) === 10 ? 2 : 1); - if (char === 10) - return offset + 1; - offset++; - } - return limit; -} - -function lineStart(md: string, start: number, end: number): number { - let offset = end; - while (offset > start) { - const char = md.charCodeAt(offset - 1); - if (char === 10 || char === 13) - break; - offset--; - } - return offset; -} - -function lineContentEnd(md: string, start: number, end: number): number { - if (end <= start) - return end; - if (md.charCodeAt(end - 1) === 10) { - return end - (md.charCodeAt(end - 2) === 13 ? 2 : 1); - } - return md.charCodeAt(end - 1) === 13 ? end - 1 : end; -} - -function blockQuoteDepth(md: string, start: number, end: number): number { - let depth = 0; - for (let offset = start; offset < end; offset++) { - if (md.charCodeAt(offset) === 62 /* > */) - depth++; - } - return depth; -} - -function skipBlockQuoteMarkers( - md: string, - start: number, - end: number, - depth: number, -): number | undefined { - let offset = start; - for (let index = 0; index < depth; index++) { - while (offset < end && (md.charCodeAt(offset) === 32 || md.charCodeAt(offset) === 9)) { - offset++; - } - if (md.charCodeAt(offset) !== 62) - return undefined; - offset++; - if (md.charCodeAt(offset) === 32 || md.charCodeAt(offset) === 9) - offset++; - } - return offset; -} - -function trimTrailingLineEnding(md: string, spans: SourceSpan[]): void { - const last = spans[spans.length - 1]; - if (!last) - return; - if (md.charCodeAt(last.end - 1) === 10) { - last.end -= md.charCodeAt(last.end - 2) === 13 ? 2 : 1; - } - else if (md.charCodeAt(last.end - 1) === 13) { - last.end--; - } - if (last.start === last.end) - spans.pop(); -} - -function equalRange( - left: string, - leftStart: number, - right: string, - rightStart: number, - length: number, -): boolean { - for (let i = 0; i < length; i++) { - if (left.charCodeAt(leftStart + i) !== right.charCodeAt(rightStart + i)) - return false; - } - return true; -} - -function segmentsFromSpans( - md: string, - spans: SourceSpan[], - value: string, -): MarkdownSourceMapSegment[] | undefined { - const segments: MarkdownSourceMapSegment[] = []; - let valueOffset = 0; - for (const span of spans) { - if (span.start >= span.end) - continue; - const length = span.end - span.start; - if (!equalRange(md, span.start, value, valueOffset, length)) - return undefined; - const previous = segments[segments.length - 1]; - if (previous && previous.sourceEnd === span.start && previous.valueEnd === valueOffset) { - previous.sourceEnd = span.end; - previous.valueEnd += length; - } - else { - segments.push({ - valueStart: valueOffset, - valueEnd: valueOffset + length, - sourceStart: span.start, - sourceEnd: span.end, - kind: 'literal', - }); - } - valueOffset += length; - } - return valueOffset === value.length ? segments : undefined; -} - -function buildIndentedCodeSegmentsFromIndentation( - md: string, - node: { value: string; position?: ParsedPosition }, -): CodeSegments | undefined { - const position = node.position; - if (!position) - return undefined; - const start = position.start.offset; - const end = position.end.offset; - const physicalLineStart = lineStart(md, 0, start); - const quoteDepth = blockQuoteDepth(md, physicalLineStart, start); - const initialContentStart = quoteDepth === 0 - ? physicalLineStart - : skipBlockQuoteMarkers(md, physicalLineStart, start, quoteDepth); - if (initialContentStart === undefined) - return undefined; - const listContinuationIndent = start - initialContentStart; - const spans: SourceSpan[] = []; - let offset = start; - let firstLine = true; - while (offset < end) { - const endOfLine = lineEnd(md, offset, end); - let contentStart = offset; - if (!firstLine && quoteDepth > 0) { - const afterMarkers = skipBlockQuoteMarkers(md, offset, endOfLine, quoteDepth); - if (afterMarkers === undefined) - return undefined; - contentStart = afterMarkers; - } - if (!firstLine && listContinuationIndent > 0) { - let removedContinuation = 0; - while ( - removedContinuation < listContinuationIndent - && md.charCodeAt(contentStart) === 32 - ) { - contentStart++; - removedContinuation++; - } - if ( - removedContinuation !== listContinuationIndent - && contentStart < lineContentEnd(md, offset, endOfLine) - ) { - return undefined; - } - } - let indentation = 0; - while (indentation < 4) { - const char = md.charCodeAt(contentStart); - if (char === 32) { - contentStart++; - indentation++; - } - else if (char === 9) { - contentStart++; - indentation += 4 - (indentation % 4); - } - else { - break; - } - } - // A non-blank line with fewer than four indentation columns needs - // parser-specific virtual-space accounting. Do not fabricate it. - if (indentation < 4 && contentStart < lineContentEnd(md, offset, endOfLine)) - return undefined; - spans.push({ start: contentStart, end: endOfLine }); - offset = endOfLine; - firstLine = false; - } - trimTrailingLineEnding(md, spans); - const segments = segmentsFromSpans(md, spans, node.value); - if (!segments) - return undefined; - return { - segments, - emptyOffset: start + Math.min(4, end - start), - }; -} - -function buildIndentedCodeSegmentsFromValueLines( - md: string, - node: { value: string; position?: ParsedPosition }, -): CodeSegments | undefined { - const position = node.position; - if (!position) - return undefined; - const start = position.start.offset; - const end = position.end.offset; - const physicalLineStart = lineStart(md, 0, start); - const quoteDepth = blockQuoteDepth(md, physicalLineStart, start); - const spans: SourceSpan[] = []; - let offset = physicalLineStart; - let valueOffset = 0; - - while (valueOffset < node.value.length) { - if (offset >= end) - return undefined; - const valueLineEnd = lineEnd(node.value, valueOffset, node.value.length); - const valueContentEnd = lineContentEnd( - node.value, - valueOffset, - valueLineEnd, - ); - const endOfLine = lineEnd(md, offset, end); - const contentStart = quoteDepth === 0 - ? offset - : skipBlockQuoteMarkers(md, offset, endOfLine, quoteDepth); - if (contentStart === undefined) - return undefined; - const contentEnd = lineContentEnd(md, contentStart, endOfLine); - // The parsed value line must be a literal suffix of the physical source line. - const valueLength = valueContentEnd - valueOffset; - const sourceStart = contentEnd - valueLength; - if (sourceStart < contentStart) - return undefined; - for (let prefixOffset = contentStart; prefixOffset < sourceStart; prefixOffset++) { - const char = md.charCodeAt(prefixOffset); - if (char !== 32 && char !== 9) - return undefined; - } - if (!equalRange(md, sourceStart, node.value, valueOffset, valueLength)) - return undefined; - spans.push({ start: sourceStart, end: endOfLine }); - offset = endOfLine; - valueOffset = valueLineEnd; - } - - trimTrailingLineEnding(md, spans); - const segments = segmentsFromSpans(md, spans, node.value); - if (!segments) - return undefined; - return { - segments, - emptyOffset: spans[0]?.start ?? start, - }; -} - -function buildIndentedCodeSegments( - md: string, - node: { value: string; position?: ParsedPosition }, -): CodeSegments | undefined { - return buildIndentedCodeSegmentsFromIndentation(md, node) - // A list continuation can start inside a tab's virtual columns, so its - // positioned node start is not sufficient to replay the physical prefix. - // Recover only literal suffixes; normalization remains rejected below. - ?? buildIndentedCodeSegmentsFromValueLines(md, node); -} - -export function buildCodeSegments( - md: string, - node: { value: string; position?: ParsedPosition }, -): CodeSegments | undefined { - return buildIndentedCodeSegments(md, node); -} diff --git a/src/source-map/recording-extension.ts b/src/source-map/recording-extension.ts index fe6a93e..4df25b8 100644 --- a/src/source-map/recording-extension.ts +++ b/src/source-map/recording-extension.ts @@ -23,9 +23,12 @@ interface PendingConstruct { sourceEnd: number } -interface FencedCodeRecording { +interface CodeValueRecording { segments: MarkdownSourceMapSegment[] valueLength: number +} + +interface FencedCodeRecording extends CodeValueRecording { emptyOffset: number sawOpeningLineEnding: boolean sawContentAfterOpening: boolean @@ -143,7 +146,7 @@ interface CompileContext { * this relies on are: * * - token event handler names (enter/exit): `data`, `codeText`, `codeTextData`, - * `codeFenced`, `codeFencedFence`, `codeFencedFenceSequence`, + * `codeFenced`, `codeFencedFence`, `codeFencedFenceSequence`, `codeIndented`, * `codeFlowValue`, `blockQuotePrefix`, `characterEscape` / * `characterEscapeValue`, `characterReference` / `characterReferenceValue`, * `lineEnding`, `autolinkProtocol`, `autolinkEmail`, @@ -165,6 +168,7 @@ export function recordingExtension(state: RecordingState) { let pendingConstruct: PendingConstruct | undefined; let inlineCodeSegments: MarkdownSourceMapSegment[] | undefined; let fencedCodeRecording: FencedCodeRecording | undefined; + let indentedCodeRecording: CodeValueRecording | undefined; let pendingEmptyFencedCode: object | undefined; let lineIndentStart = 0; @@ -290,8 +294,11 @@ export function recordingExtension(state: RecordingState) { }); }; - const recordFencedCodeSegment = function (this: CompileContext, token: any) { - const recording = fencedCodeRecording; + const recordCodeSegment = function ( + this: CompileContext, + recording: CodeValueRecording | undefined, + token: any, + ) { if (!recording) return; const valueLength = this.sliceSerialize(token).length; @@ -355,8 +362,9 @@ export function recordingExtension(state: RecordingState) { else { fencedCodeRecording.sawContentAfterOpening = true; } - recordFencedCodeSegment.call(this, token); + recordCodeSegment.call(this, fencedCodeRecording, token); } + recordCodeSegment.call(this, indentedCodeRecording, token); } }; @@ -469,7 +477,8 @@ export function recordingExtension(state: RecordingState) { }); if (fencedCodeRecording) fencedCodeRecording.sawContentAfterOpening = true; - recordFencedCodeSegment.call(this, token); + recordCodeSegment.call(this, fencedCodeRecording, token); + recordCodeSegment.call(this, indentedCodeRecording, token); }; const onexitcodefenced = function (this: CompileContext, token: any) { @@ -506,6 +515,41 @@ export function recordingExtension(state: RecordingState) { this.exit(token); }; + const onentercodeindented = function (this: CompileContext, token: any) { + if (indentedCodeRecording) { + throw new Error('An indented code source map is already being recorded'); + } + this.enter({ type: 'code', lang: null, meta: null, value: '' }, token); + this.buffer(); + indentedCodeRecording = { + segments: [], + valueLength: 0, + }; + }; + + const onexitcodeindented = function (this: CompileContext, token: any) { + const data = this.resume(); + const node = this.stack[this.stack.length - 1]; + if (!node || node.type !== 'code') { + throw new Error('Expected code node while recording indented code'); + } + const value = data.replace(/(\r?\n|\r)$/g, ''); + node.value = value; + const recording = indentedCodeRecording; + indentedCodeRecording = undefined; + if (!recording) { + throw new Error('Missing indented code source-map recording'); + } + const segments = sliceLiteralSegments( + recording.segments, + 0, + value.length, + ); + if (segments) + state.codeSegments.set(node, segments); + this.exit(token); + }; + const onexitautolinkprotocol = function (this: CompileContext, token: any) { onexitdata.call(this, token, { sourceStart: token.start.offset, @@ -579,6 +623,7 @@ export function recordingExtension(state: RecordingState) { enter: { codeFenced: onentercodefenced, codeFencedFence: onentercodefencedfence, + codeIndented: onentercodeindented, codeFlowValue: onenterdata, codeText: onentercodetext, codeTextData: onentercodetextdata, @@ -594,6 +639,7 @@ export function recordingExtension(state: RecordingState) { codeFenced: onexitcodefenced, codeFencedFence: onexitcodefencedfence, codeFencedFenceSequence: onexitcodefencedfencesequence, + codeIndented: onexitcodeindented, codeFlowValue: onexitcodeflowvalue, codeText: onexitcodetext, codeTextData: onexitcodetextdata,