diff --git a/src/delta/delta.js b/src/delta/delta.js index 1edbf4a..9718244 100644 --- a/src/delta/delta.js +++ b/src/delta/delta.js @@ -406,6 +406,8 @@ export class TextOp extends list.ListNode { encoding.writeVarString(encoder, this.insert) encoding.writeAny(encoder, this.format) encoding.writeAny(encoder, this.attribution) + // UTF-8 replaces lone surrogates with U+FFFD; JSON escapes them losslessly. + if (/[\uD800-\uDFFF]/u.test(this.insert)) encoding.writeVarString(encoder, JSON.stringify(this.insert)) }))) } @@ -4267,6 +4269,19 @@ const commonUnits = (a, ai, b, bi, max, back) => { return k } +/** + * Check within and between adjacent text ops, regardless of style. + * @param {TextOp} op + * @param {number} offset + */ +const splitsSurrogatePair = (op, offset) => { + const prev = op.prev + const next = op.next + const high = offset === 0 && $textOp.check(prev) ? prev.insert.charCodeAt(prev.length - 1) : op.insert.charCodeAt(offset - 1) + const low = offset === op.length && $textOp.check(next) ? next.insert.charCodeAt(0) : op.insert.charCodeAt(offset) + return high >= 0xd800 && high <= 0xdbff && low >= 0xdc00 && low <= 0xdfff +} + /** * Where a lockstep walk of two states stands: the current op of each side and the units of it already * consumed from the walked end. @@ -4314,6 +4329,11 @@ const commonLen = (c, max, back) => { k = nl < start || nl >= end ? 0 : back ? end - nl - 1 : nl - start + 1 max = len + k } + // Leave whole code points in the interior when a hint or op boundary divides a pair. + if ($textOp.check(a) && $textOp.check(b) && (splitsSurrogatePair(a, back ? a.length - ai - k : ai + k) || splitsSurrogatePair(b, back ? b.length - bi - k : bi + k))) { + k-- + max = len + k + } len += k ai += k bi += k @@ -4486,7 +4506,8 @@ export const diff = (d1, d2, options = {}) => { let changeset = [{ index: prefixLen, insert: cs2, remove: cs1 }] changeset = diffChangesetWithSeparator(changeset, /[\n]+/g, leftmostIndex) changeset = diffChangesetWithSeparator(changeset, patience.smartSplitRegex, leftmostIndex) - changeset = diffChangesetWithSeparator(changeset, /./g, leftmostIndex) + // Match code points, including line terminators; emitted lengths remain UTF-16. + changeset = diffChangesetWithSeparator(changeset, /./gsu, leftmostIndex) applyChangesetToDelta(d, changeset, c, prefixLen, n - suffixLen, options) applyAttrDiff(d, d1, d2, options) } diff --git a/src/delta/delta.test.js b/src/delta/delta.test.js index 1aa58ad..d2936fa 100644 --- a/src/delta/delta.test.js +++ b/src/delta/delta.test.js @@ -906,6 +906,93 @@ export const testDiffingCommonPreSuffix = () => { t.compare(d, delta.retain(3).delete(1)) } +/** @param {t.TestCase} _tc */ +export const testDiffSurrogates = _tc => { + // Exact operations matter: a JS round-trip can hide a retained high surrogate plus inserted low surrogate. + const cases = [ + ['Emoji probe ', '🚀', '🐈', ''], // shared high surrogate + ['x', '😀', '🨀', 'z'], // shared low surrogate + ['', '', '🐈', ''], ['abc', '', '🐈', ''], ['', '', '🐈', 'abc'], + ['🚀', '🚀', '🐈', '🚀'], ['é漢\r\n', '🚀', '🐈', '\u2028😀\u2029z'], + ['ab', 'c', 'X', '漢é'], + ['x', '\ud83d', '\ud83e', 'z'], ['x', '\udc08', '\ude80', 'z'], + ['', '\ud83d', '�', ''], ['', '\udc08', '�', ''], + ['', '\ud83d', '🐈', ''], ['', '\udc08', '🐈', ''] + ] + for (const [prefix, a, b, suffix] of cases) { + for (const [from, to] of [[a, b], [b, a]]) { + const start = delta.insert(prefix + from + suffix) + const end = delta.insert(prefix + to + suffix) + const expected = delta.retain(prefix.length).delete(from.length).insert(to).done() + t.compare(delta.diff(start, end), expected) + for (let i = 0; i <= start.childCnt + 1; i++) { + t.compare(delta.diff(start, end, { hint: position.create([i]) }), expected) + } + t.compare(delta.clone(start).apply(expected), end) + } + } + t.compare(delta.diff(delta.insert('🚀🚀🚀'), delta.insert('🚀🚀'), { hint: position.create([2]) }), delta.retain(2).delete(2)) + const a = delta.create('p').insert('x🚀z') + const b = delta.create('p').insert('x🐈z') + const change = delta.create('p').retain(1).delete(2).insert('🐈') + t.compare(delta.diff(delta.insert([a]), delta.insert([b]), { hint: position.create([0, 2]) }), delta.modify(change)) + t.compare(delta.diff(delta.setAttr('body', a), delta.setAttr('body', b), { hint: position.create(['body', 2]) }), delta.modifyAttr('body', change)) +} + +/** @param {t.TestCase} _tc */ +export const testDiffSurrogateTextOps = _tc => { + const a = delta.insert('prefix ', { bold: true }, { insert: ['alice'] }).insert('🚀', { italic: true }, { insert: ['alice'] }).insert(' suffix') + const b = delta.insert('prefix ', { bold: true }, { insert: ['alice'] }).insert('🐈', { bold: true }, { insert: ['bob'] }).insert(' suffix') + // Include pre-existing style/attribution boundaries between surrogate units. + const cases = [ + [a, b, delta.retain(7).delete(2).insert('🐈', { bold: true }, { insert: ['bob'] })], + [b, a, delta.retain(7).delete(2).insert('🚀', { italic: true }, { insert: ['alice'] })], + [delta.insert('x\ud83d').insert('\ude80', { bold: true }), delta.insert('x\ud83d').insert('\udc08', { bold: true }), delta.retain(1).delete(2).insert('\ud83d').insert('\udc08', { bold: true })], + [delta.insert('\ud83d', { bold: true }).insert('\ude00z'), delta.insert('\ud83e', { bold: true }).insert('\ude00z'), delta.delete_(2).insert('\ud83e', { bold: true }).insert('\ude00')], + [delta.insert('x\ud83d', undefined, { insert: ['a'] }).insert('\ude80', undefined, { insert: ['b'] }), delta.insert('x🐈', undefined, { insert: ['b'] }), delta.retain(1, undefined, { insert: ['b'] }).delete(2).insert('🐈', undefined, { insert: ['b'] })] + ] + for (const [start, end, expected] of cases) { + t.compare(delta.diff(start, end), expected) + t.compare(delta.clone(start).apply(expected), end) + t.compare(delta.clone(end).apply(delta.diff(end, start)), start) + } + const restyled = delta.insert('prefix ').insert('🚀', { bold: true }, { insert: ['bob'] }).insert(' suffix') + const change = delta.diff(a, restyled) + t.assert([...change.children].every(op => delta.$retainOp.check(op))) + t.compare(delta.clone(a).apply(change), restyled) +} + +/** @param {t.TestCase} tc */ +export const testRepeatDiffSurrogateText = tc => { + const chars = ['a', 'b', 'é', '漢', ' ', '\n', '\r', '\u2028', '\u2029', '🚀', '🐈', '😀', '🨀', '👩', '\u200d', '💻', '🏽', '️'] + const state = () => { + const parts = Array.from({ length: prng.int32(tc.prng, 0, 30) }, () => prng.oneOf(tc.prng, chars)) + const d = delta.create() + parts.forEach(c => d.insert(c, prng.bool(tc.prng) ? { bold: true } : undefined, prng.bool(tc.prng) ? { insert: ['alice'] } : { insert: ['bob'] })) + return { d, text: parts.join('') } + } + const a = state() + const b = state() + const change = delta.diff(a.d, b.d, { hint: position.create([prng.int32(tc.prng, 0, a.text.length + 1)]) }) + t.compare(delta.clone(a.d).apply(change), b.d) + let ai = 0 + let bi = 0 + for (const op of change.children) { + if (delta.$retainOp.check(op)) { + ai += op.retain + bi += op.retain + } else if (delta.$deleteOp.check(op)) { + ai += op.delete + } else if (delta.$textOp.check(op)) { + t.assert(!/[\uD800-\uDFFF]/u.test(op.insert), 'insert contains whole code points') + bi += op.length + } else { + t.fail('expected text-only diff') + } + t.assert((a.text.codePointAt(ai - 1) ?? 0) <= 0xffff && (b.text.codePointAt(bi - 1) ?? 0) <= 0xffff, 'operation does not split a surrogate pair') + } +} + export const testSlice = () => { const d1 = delta.slice(delta.insert('abcde'), 1, 3) t.assert(d1.equals(delta.insert('bc')))