diff --git a/__tests__/source-map/inline-code.spec.ts b/__tests__/source-map/inline-code.spec.ts index 64052ac..f7e57fc 100644 --- a/__tests__/source-map/inline-code.spec.ts +++ b/__tests__/source-map/inline-code.spec.ts @@ -64,6 +64,17 @@ describe('parseMdWithSourceMap: inlineCode.value → raw source', () => { }); }); + test('maps whitespace-only content from parser events', () => { + const md = '` `'; + const { ast, sourceMap } = parseMdWithSourceMap(md); + const node = nodesOfType(ast, 'inlineCode')[0]; + expect(node.value).toBe(' '); + expect(sourceMap.getSourceRange(node, 0, node.value.length)).toEqual({ + start: { line: 1, column: 2, offset: 1 }, + end: { line: 1, column: 4, offset: 3 }, + }); + }); + test.each(['\n', '\r', '\r\n'])( 'maps %p code units individually after padding normalization', (lineEnding) => { diff --git a/src/source-map/build-source-map.ts b/src/source-map/build-source-map.ts index b1930ff..019fe03 100644 --- a/src/source-map/build-source-map.ts +++ b/src/source-map/build-source-map.ts @@ -28,16 +28,17 @@ import type { SourceSpan, } from './types'; -// Use the exact same parser extensions as `parseMd` so the AST (and therefore -// the tokenizer / mdast-extension decisions) are identical. Only the mdast -// `text`-building handlers are swapped for recording ones; every other token -// is compiled by the real `mdast-util-from-markdown` handlers. +// Use the same parser extensions as `parseMd`. +// This keeps tokenizer and mdast extension decisions identical. +// Only replace the mdast handlers needed to record mapped values. +// The replacements mirror the matching `mdast-util-from-markdown` AST behavior. +// All other tokens use the standard handlers. const { micromarkExtensions, fromMarkdownExtensions } = getParserExtensions(); interface RecordingState { /** node -> ordered, gap-free, non-overlapping segments. */ segments: WeakMap - /** inlineCode node -> value segments (see buildInlineCodeSegments). */ + /** inlineCode node -> parser-recorded value segments. */ inlineCodeSegments: WeakMap /** code node -> value segments (see buildCodeSegments). */ codeSegments: WeakMap @@ -125,101 +126,6 @@ function pointAtOffset(lineStarts: number[], md: string, offset: number): Parsed }; } -/** - * Build the source-map segments for an `inlineCode` node. - * - * `inlineCode.value` is NOT a contiguous slice of the source: the GFM code - * span algorithm (micromark `codeText` + `mdast-util-from-markdown`) strips one - * leading and one trailing whitespace unit from the content when it contains - * non-whitespace data. A unit is one space, LF, CR, or CRLF. The parser keeps - * all remaining source code units verbatim, so every surviving value code unit - * maps 1:1 to exactly one source code unit. - * - * The mapping is computed from the node's `position` (the full source span - * including the backtick delimiters) plus `value`, replicating the GFM - * resolver: identify its delimiters, then apply its single leading/trailing - * whitespace-unit rule. This couples to the same parser-sensitive behavior as - * the text mapping (see CONTRIBUTING). - * - * @returns ordered, gap-free, 1:1 segments, or undefined if the node has no - * usable position. - * - * @internal Used by buildSourceMap; not part of the public API. - */ -function buildInlineCodeSegments( - md: string, - node: { value: string; position?: ParsedPosition }, -): MarkdownSourceMapSegment[] | undefined { - const position = node.position; - if (!position || !position.start || !position.end) - return undefined; - const start = position.start.offset; - const end = position.end.offset; - if (start < 0 || end > md.length || start >= end) - return undefined; - - // Full source span including the backtick delimiters. - const full = md.slice(start, end); - - // Determine the opening / closing backtick run lengths (they must match). - let openLen = 0; - while (openLen < full.length && full.charCodeAt(openLen) === 96 /* ` */) openLen++; - let closeLen = 0; - while (closeLen < full.length && full.charCodeAt(full.length - 1 - closeLen) === 96) closeLen++; - if (openLen === 0 || closeLen === 0 || openLen !== closeLen) - return undefined; - - const interiorStart = start + openLen; - const interiorEnd = end - closeLen; - const interior = md.slice(interiorStart, interiorEnd); - - const isWhitespace = (char: number): boolean => - char === 32 || char === 10 || char === 13; - const leadingWhitespaceEnd = (): number => { - const first = interior.charCodeAt(0); - if (first === 13 && interior.charCodeAt(1) === 10) - return 2; - return isWhitespace(first) ? 1 : 0; - }; - const trailingWhitespaceStart = (): number => { - const last = interior.charCodeAt(interior.length - 1); - if (last === 10 && interior.charCodeAt(interior.length - 2) === 13) { - return interior.length - 2; - } - return isWhitespace(last) ? interior.length - 1 : interior.length; - }; - - let valueSourceStart = 0; - let valueSourceEnd = interior.length; - const leadingEnd = leadingWhitespaceEnd(); - const trailingStart = trailingWhitespaceStart(); - let hasData = false; - for (let i = 0; i < interior.length; i++) { - if (!isWhitespace(interior.charCodeAt(i))) { - hasData = true; - break; - } - } - if (leadingEnd > 0 && trailingStart < interior.length && hasData) { - valueSourceStart = leadingEnd; - valueSourceEnd = trailingStart; - } - - // Confirm that the parser did not apply an unaccounted-for transformation. - // Returning undefined is safer than fabricating a source range. - const sourceValue = interior.slice(valueSourceStart, valueSourceEnd); - if (sourceValue !== node.value || sourceValue.length === 0) - return undefined; - - return [{ - valueStart: 0, - valueEnd: sourceValue.length, - sourceStart: interiorStart + valueSourceStart, - sourceEnd: interiorStart + valueSourceEnd, - kind: 'literal', - }]; -} - function isEscapableUrlCharacter(char: number): boolean { return (char >= 33 && char <= 47) || (char >= 58 && char <= 64) @@ -562,17 +468,6 @@ export const parseMdWithSourceMap = (md: string): ParsedMarkdownDocument => { const sourceGapPrefixes = new WeakMap(); function indexNode(node: TraversableNode): void { - // The standard mdast handler compiles inline code. - // The completed node contains enough data to build its mapping. - if ( - node.type === 'inlineCode' - && hasStringField(node, 'value') - ) { - const segments = buildInlineCodeSegments(md, node); - if (segments) - state.inlineCodeSegments.set(node, segments); - } - if ( node.type === 'code' && hasStringField(node, 'value') diff --git a/src/source-map/recording-extension.ts b/src/source-map/recording-extension.ts index e5edfeb..22de3b3 100644 --- a/src/source-map/recording-extension.ts +++ b/src/source-map/recording-extension.ts @@ -5,6 +5,7 @@ import type { MarkdownSourceMapSegment, SourceSpan } from './types'; interface RecordingState { segments: WeakMap + inlineCodeSegments: WeakMap urlSourceSpans: WeakMap } @@ -32,6 +33,8 @@ const createText = () => ({ type: 'text', value: '' }); interface CompileContext { stack: Array config: { canContainEols: string[] } + enter: (node: any, token: any) => void + exit: (token: any) => void getData: (key: string) => unknown setData: (key: string, value?: unknown) => void sliceSerialize: (token: any) => string @@ -40,24 +43,25 @@ interface CompileContext { } /** - * Build a `mdast`-extension whose `text`-building handlers record, alongside - * the normal AST construction, the mapping from each `text` node's normalized - * `value` back to the raw Markdown source. + * Build a `mdast` extension that records source mappings during compilation. + * Its handlers record mappings for `text` and `inlineCode` values. * * ⚠️ This couples to `mdast-util-from-markdown` / micromark INTERNALS, not the * public remark API. Upgrading any parser-sensitive dependency is a parser * behavior upgrade — see CONTRIBUTING.md. The undocumented upstream contracts * this relies on are: * - * - token event handler names (enter/exit): `data`, `characterEscape` / + * - token event handler names (enter/exit): `data`, `codeText`, `codeTextData`, + * `characterEscape` / * `characterEscapeValue`, `characterReference` / `characterReferenceValue`, * `lineEnding`, `autolinkProtocol`, `autolinkEmail`, * `resourceDestinationString`, `definitionDestinationString`, their literal * wrappers, and `resource`. * - compile-context fields on `this` ({@link CompileContext}): `stack` (the AST - * build stack), `config.canContainEols` (whether a line ending is merged into - * text), `getData` / `setData` for the keys `characterReferenceType`, - * `atHardBreak`, `setextHeadingSlurpLineEnding`, and `sliceSerialize`. + * build stack), `enter` / `exit`, `config.canContainEols` (whether a line + * ending is merged into text), and `getData` / `setData`. + * The handlers use the `characterReferenceType`, `atHardBreak`, and + * `setextHeadingSlurpLineEnding` data keys. They also use `sliceSerialize`. * - entity decoding must match remark's own: `decodeNumericCharacterReference` / * `decodeNamedCharacterReference` are pinned to the versions remark uses so * decoding does not drift. @@ -67,6 +71,7 @@ interface CompileContext { */ export function recordingExtension(state: RecordingState) { let pendingConstruct: PendingConstruct | undefined; + let inlineCodeSegments: MarkdownSourceMapSegment[] | undefined; const takePendingConstruct = (): PendingConstruct => { if (!pendingConstruct) { @@ -162,6 +167,34 @@ export function recordingExtension(state: RecordingState) { } }; + const inlineCodeValueLength = (): number => { + const tail = inlineCodeSegments?.[inlineCodeSegments.length - 1]; + return tail?.valueEnd ?? 0; + }; + + const recordInlineCodeSegment = function (this: CompileContext, token: any) { + if (!inlineCodeSegments) + return; + const valueLength = this.sliceSerialize(token).length; + const previous = inlineCodeSegments[inlineCodeSegments.length - 1]; + if ( + previous + && previous.sourceEnd === token.start.offset + ) { + previous.valueEnd += valueLength; + previous.sourceEnd = token.end.offset; + return; + } + const valueStart = inlineCodeValueLength(); + inlineCodeSegments.push({ + valueStart, + valueEnd: valueStart + valueLength, + sourceStart: token.start.offset, + sourceEnd: token.end.offset, + kind: 'literal', + }); + }; + const onexitlineending = function (this: CompileContext, token: any) { if (this.getData('atHardBreak')) { const tail = this.stack[this.stack.length - 1].children.slice(-1)[0]; @@ -180,7 +213,49 @@ export function recordingExtension(state: RecordingState) { sourceEnd: token.end.offset, kind: 'literal', }); + recordInlineCodeSegment.call(this, token); + } + }; + + const onentercodetext = function (this: CompileContext, token: any) { + if (inlineCodeSegments) { + throw new Error('An inline code source map is already being recorded'); + } + this.enter({ type: 'inlineCode', value: '' }, token); + this.buffer(); + inlineCodeSegments = []; + }; + + const onentercodetextdata = function (this: CompileContext, token: any) { + const node = this.stack[this.stack.length - 1]; + let tail = node.children[node.children.length - 1]; + if (!tail || tail.type !== 'text') { + tail = createText(); + tail.position = { start: point(token.start) }; + node.children.push(tail); + } + this.stack.push(tail); + }; + + const onexitcodetextdata = function (this: CompileContext, token: any) { + const tail = this.stack.pop(); + const slice = this.sliceSerialize(token); + tail.value += slice; + tail.position.end = point(token.end); + recordInlineCodeSegment.call(this, token); + }; + + const onexitcodetext = function (this: CompileContext, token: any) { + const value = this.resume(); + const node = this.stack[this.stack.length - 1]; + node.value = value; + const segments = inlineCodeSegments; + const mappedValueLength = inlineCodeValueLength(); + inlineCodeSegments = undefined; + if (segments && mappedValueLength === value.length) { + state.inlineCodeSegments.set(node, segments); } + this.exit(token); }; const onexitautolinkprotocol = function (this: CompileContext, token: any) { @@ -254,6 +329,8 @@ export function recordingExtension(state: RecordingState) { return { enter: { + codeText: onentercodetext, + codeTextData: onentercodetextdata, data: onenterdata, characterEscape: onenterConstruct, characterReference: onenterConstruct, @@ -263,6 +340,8 @@ export function recordingExtension(state: RecordingState) { resourceDestinationString: onenterUrlDestination, }, exit: { + codeText: onexitcodetext, + codeTextData: onexitcodetextdata, data(this: CompileContext, token: any) { onexitdata.call(this, token, { sourceStart: token.start.offset,