diff --git a/lib/__tests__/jsonpath.test.ts b/lib/__tests__/jsonpath.test.ts index b5d4497..88cf9e6 100644 --- a/lib/__tests__/jsonpath.test.ts +++ b/lib/__tests__/jsonpath.test.ts @@ -2,6 +2,7 @@ import { describe, it, expect } from "vitest"; import { LosslessNumber, stringify } from "lossless-json"; import { runJsonPath } from "../jsonpath"; import { SAMPLE_JSON } from "../json-utils"; +import { buildJsonTreePath } from "../json-tree-path"; describe("runJsonPath", () => { it.each(["", " \t\n "])("rejects blank JSON %j", (raw) => { @@ -96,3 +97,197 @@ describe("runJsonPath", () => { }); }); }); + +describe("runJsonPath quoted names (RFC 9535 string literals)", () => { + const found = (raw: string, path: string) => stringify(runJsonPath(raw, path).matches); + const AWKWARD = '{"q\\"uote": 5, "back\\\\slash": 6, "br[ack]et": 9, "it\'s": 10, "tab\\there": 11}'; + + it.each([ + ['$["q\\"uote"]', "[5]"], + ['$["back\\\\slash"]', "[6]"], + ['$["br[ack]et"]', "[9]"], + ['$["it\'s"]', "[10]"], + ['$["tab\\there"]', "[11]"], + ["$['it\\'s']", "[10]"], + ])("finds the value for %s", (path, expected) => { + expect(found(AWKWARD, path)).toBe(expected); + }); + + it("keeps the paths that already worked working", () => { + const raw = '{"a b": 1, "": 2, "q\\"uote": 5}'; + expect(found(raw, '$["a b"]')).toBe("[1]"); + expect(found(raw, "$['a b']")).toBe("[1]"); + expect(found(raw, '$[""]')).toBe("[2]"); + expect(found(raw, "$['q\"uote']")).toBe("[5]"); + }); + + it("reads every escape of a string literal", () => { + const raw = JSON.stringify({ "a/b": 1, "a\bb": 2, "a\fb": 3, "a\nb": 4, "a\rb": 5, "a\tb": 6, "a\\b": 7, "é": 8, "😀": 9 }); + expect(found(raw, '$["a\\/b"]')).toBe("[1]"); + expect(found(raw, '$["a\\bb"]')).toBe("[2]"); + expect(found(raw, '$["a\\fb"]')).toBe("[3]"); + expect(found(raw, '$["a\\nb"]')).toBe("[4]"); + expect(found(raw, '$["a\\rb"]')).toBe("[5]"); + expect(found(raw, '$["a\\tb"]')).toBe("[6]"); + expect(found(raw, '$["a\\\\b"]')).toBe("[7]"); + expect(found(raw, '$["\\u00e9"]')).toBe("[8]"); + expect(found(raw, '$["\\u00E9"]')).toBe("[8]"); + expect(found(raw, '$["\\ud83d\\ude00"]')).toBe("[9]"); + }); + + it("takes [, ], . and , inside the quotes as characters", () => { + const raw = '{"a[0]": 1, "a]": 2, "a.b": 3, "a,b": 4, "a][b": 5}'; + expect(found(raw, '$["a[0]"]')).toBe("[1]"); + expect(found(raw, '$["a]"]')).toBe("[2]"); + expect(found(raw, '$["a.b"]')).toBe("[3]"); + expect(found(raw, "$['a,b']")).toBe("[4]"); + expect(found(raw, '$["a][b"]')).toBe("[5]"); + }); + + it("matches a name that looks like an operator as a name, not as the operator", () => { + const raw = '{"*": 1, "..": 2, "^": 3, "~": 4, "$": 5, "@": 6, "1:2": 7, "?(@.x)": 8, "(1)": 9, "a": 10}'; + for (const [name, value] of [["*", 1], ["..", 2], ["^", 3], ["~", 4], ["$", 5], ["@", 6], ["1:2", 7], ["?(@.x)", 8], ["(1)", 9]] as const) { + expect(found(raw, `$[${JSON.stringify(name)}]`)).toBe(`[${value}]`); + expect(found(raw, `$['${name}']`)).toBe(`[${value}]`); + } + }); + + it("finds the value for the path buildJsonTreePath writes, whatever the key", () => { + const keys = [ + '', 'a b', 'q"uote', "it's", 'a\'b"c', "back\\slash", "\\", '\\"', "br[ack]et", "a]", "[0]", "']", '"]', + "dot.key", "til~de", "sl/ash", "a~1b", "a~0b", "~0", "~1", "a,b", ";", "%", "$ref", "$", "@", "*", "..", "^", "~", "?(@.x)", "(1)", "1:2", "-1", "0", + "tab\there", "new\nline", "cr\rhere", "\u0000", "\u0008", "\u001f", "\u007f", "\u2028", "\ud800", "\udc00", + "é", "日本語", "😀", "a\u{10ffff}", "__proto__", "constructor", "length", + ]; + const raw = JSON.stringify(Object.fromEntries(keys.map((key, i) => [key, i])), null, 1); + // JSON.stringify drops __proto__ as an own key when built from an object, so write it by hand. + const body = "{" + keys.map((key, i) => `${JSON.stringify(key)}: ${i}`).join(", ") + "}"; + for (const text of [body, raw]) { + keys.forEach((key, i) => { + const path = buildJsonTreePath([{ type: "property", key }]); + const result = runJsonPath(text, path); + expect({ key, path, matches: stringify(result.matches) }).toEqual({ key, path, matches: `[${i}]` }); + }); + } + }); + + it("finds a nested value through awkward keys and array indexes", () => { + const raw = '{"it\'s": [{"a.b": {"c]": 1}}, {"a.b": {"c]": 2}}]}'; + const path = buildJsonTreePath([ + { type: "property", key: "it's" }, + { type: "index", index: 1 }, + { type: "property", key: "a.b" }, + { type: "property", key: "c]" }, + ]); + expect(found(raw, path)).toBe("[2]"); + }); + + it("finds the name through a wildcard, a recursive descent and a filter", () => { + const raw = '[{"it\'s": 1, "x": 0}, {"it\'s": 2, "x": 0}, {"deep": {"it\'s": 3}}]'; + expect(found(raw, '$[*]["it\'s"]')).toBe("[1,2]"); + expect(found(raw, '$..["it\'s"]')).toBe("[1,2,3]"); + expect(found(raw, '$[?(@.x==0)]["it\'s"]')).toBe("[1,2]"); + }); + + it("returns a property name as the name, and keeps the other lookups free of the lookup alias", () => { + const raw = '{"it\'s": 10, "a": 1}'; + expect(runJsonPath(raw, '$["it\'s"]~').matches).toEqual(["it's"]); + expect(runJsonPath(raw, "$.*~").matches).toEqual(["it's", "a"]); + expect(found(raw, "$.*")).toBe("[10,1]"); + expect(found(raw, "$..*")).toBe("[10,1]"); + expect(stringify(runJsonPath(raw, '$["it\'s"]').matches)).toBe("[10]"); + }); + + it("names the matched key in the result path", () => { + expect(runJsonPath('{"it\'s": 10}', '$["it\'s"]').paths).toEqual(["$['it's']"]); + expect(runJsonPath('{"q\\"uote": {"a": 1}}', '$["q\\"uote"].a').paths).toEqual(["$['q\"uote']['a']"]); + }); + + it("keeps large numbers exact when they are found by a quoted name", () => { + expect(found('{"it\'s": 12345678901234567890}', '$["it\'s"]')).toBe("[12345678901234567890]"); + }); + + it("does not change the data it searches: a hidden lookup never shows up as a key", () => { + const raw = '{"it\'s": {"it\'s": 1}, "b": 2}'; + expect(found(raw, '$["it\'s"]')).toBe('[{"it\'s":1}]'); + expect(found(raw, '$["it\'s"]["it\'s"]')).toBe("[1]"); + expect(runJsonPath(raw, "$..*~").matches).toEqual(["it's", "b", "it's"]); + }); + + it("reads the blanks RFC 9535 allows inside the brackets, for plain and awkward names alike", () => { + const raw = '{"a": 1, "it\'s": 2}'; + expect(found(raw, '$[ "a" ]')).toBe("[1]"); + expect(found(raw, "$[ 'a']")).toBe("[1]"); + expect(found(raw, '$["a" ]')).toBe("[1]"); + expect(found(raw, '$[ "it\'s" ]')).toBe("[2]"); + }); + + it("reads an escape in a name that is otherwise plain", () => { + expect(found('{"a b": 1}', '$["a\\u0020b"]')).toBe("[1]"); + expect(found('{"a\\nb": 1}', '$["a\\nb"]')).toBe("[1]"); + }); + + it("follows two different awkward names in one path, and the same one twice", () => { + const raw = '{"a.b": {"c]": {"a.b": 3}}}'; + expect(found(raw, '$["a.b"]["c]"]["a.b"]')).toBe("[3]"); + expect(found(raw, "$['a.b']['c]']")).toBe('[{"a.b":3}]'); + }); + + it("is not fooled by a key that looks like the lookup it adds", () => { + // Whatever private-use character the lookup is built from, a document that already uses one still works, + // whether the character is written as itself or as an escape in either case. + for (const code of ["e000", "e001", "e002"]) { + const char = String.fromCharCode(parseInt(code, 16)); + const key = `${char}0${char}`; + for (const written of [key, `\\u${code}0\\u${code}`, `\\u${code.toUpperCase()}0\\u${code.toUpperCase()}`]) { + const raw = `{"it's": 1, "${written}": 2}`; + expect(found(raw, '$["it\'s"]')).toBe("[1]"); + expect(found(raw, `$["${key}"]`)).toBe("[2]"); + // The parent, listed after the lookup ran, still has both keys, with their own values. + expect(runJsonPath(raw, '$["it\'s"]^.*~').matches).toEqual(["it's", key]); + expect(found(raw, '$["it\'s"]^.*')).toBe("[1,2]"); + } + } + }); + + it("does not let the expression mention the lookup either", () => { + const token = String.fromCharCode(0xe000) + "0" + String.fromCharCode(0xe000); + // A filter that asks for a property named like the first lookup must not find the hidden one. + expect(found('[{"it\'s": 1}]', `$[?(@['${token}'])]["it's"]`)).toBe("[]"); + expect(found('[{"it\'s": 1}]', `$[0]["it's"]`)).toBe("[1]"); + expect(found('{"it\'s": {"\\ue000": 5}}', `$["it's"]["${String.fromCharCode(0xe000)}"]`)).toBe("[5]"); + }); + + it("returns nothing, without failing, when the data is not an object", () => { + expect(runJsonPath("5", '$["it\'s"]')).toEqual({ ok: true, matches: [], paths: [] }); + expect(found('[{"it\'s": 1}, [{"it\'s": 2}]]', '$[0]["it\'s"]')).toBe("[1]"); + expect(found('[{"it\'s": 1}, [{"it\'s": 2}]]', '$[1][0]["it\'s"]')).toBe("[2]"); + }); + + it("keeps reading a literal with an escape RFC 9535 does not define the way it always was, as the literal characters", () => { + // \q is not an escape, and neither is \' in double quotes or \" in single quotes, so these selectors are not + // rewritten and keep their old result: the key that has the backslash in it, or nothing, never the key + // without the backslash. + const raw = JSON.stringify({ a: 1, "a\\qb": 2, "it\\'s": 3, "it's": 4, 'q\\"t': 5 }); + expect(found(raw, '$["a\\qb"]')).toBe("[2]"); + expect(found(raw, '$["a"]')).toBe("[1]"); + expect(found(raw, '$["it\\\'s"]')).toBe("[]"); + expect(found(raw, "$['it\\'s']")).toBe("[4]"); + expect(found(raw, "$['q\\\"t']")).toBe("[5]"); + }); + + it("does not shorten a bracket that holds more than one name", () => { + // jsonpath-plus finds nothing for a quoted union; it must not be turned into its first name. + const raw = '{"a": 1, "b": 2}'; + expect(found(raw, "$['a','b']")).toBe("[]"); + expect(found(raw, '$["a","b"]')).toBe("[]"); + }); + + it("lists the keys of an object that has a lookup without the lookup", () => { + const raw = '{"a.b": {"a.b": 1, "c": 2}}'; + expect(found(raw, '$["a.b"].*')).toBe("[1,2]"); + expect(runJsonPath(raw, '$["a.b"].*').paths).toEqual(["$['a.b']['a.b']", "$['a.b']['c']"]); + expect(runJsonPath(raw, '$["a.b"].*~').matches).toEqual(["a.b", "c"]); + expect(found(raw, '$["a.b"]..*')).toBe("[1,2]"); + }); +}); diff --git a/lib/json-tree-path.ts b/lib/json-tree-path.ts index 5f13dc4..9a76392 100644 --- a/lib/json-tree-path.ts +++ b/lib/json-tree-path.ts @@ -11,7 +11,8 @@ export function buildJsonTreePath(segments: JsonPathSegment[]): string { return `${path}[${segment.index}]`; } - if (SIMPLE_PROPERTY.test(segment.key)) { + // A bare `$` after a dot is the root, not a property, so a key called `$` is quoted like the other special names. + if (SIMPLE_PROPERTY.test(segment.key) && segment.key !== "$") { return `${path}.${segment.key}`; } diff --git a/lib/jsonpath.ts b/lib/jsonpath.ts index b3eb412..0ba4b6b 100644 --- a/lib/jsonpath.ts +++ b/lib/jsonpath.ts @@ -19,6 +19,144 @@ function atPointer(tree: unknown, pointer: string): unknown { ); } +// ---- Quoted names ---------------------------------------------------------------------------- +// jsonpath-plus splits an expression with regular expressions, so a quoted name that holds a quote, +// a backslash, a bracket or a dot, or that reads like an operator (`*`, `..`, `^`, `~`, `$`, `a,b`, +// `1:2`, `?(...)`), is cut up or run as the operator. Names it already reads correctly are left +// exactly as they are. Any other quoted name is read the way RFC 9535 defines a string literal and +// replaced by a harmless alias that is also added, hidden, to the parsed data, so the library +// finds the real key through it. + +// Names the library reads correctly when they are written without an escape. +const PLAIN_NAME = /^[A-Za-z0-9_ -]*$/; + +const ESCAPES: Record = { b: "\b", f: "\f", n: "\n", r: "\r", t: "\t", "\\": "\\", "/": "/" }; + +// A string literal that starts at `start` (a quote): its value and the index after its closing quote. +// An escape RFC 9535 does not define, or a literal that never closes, is not read (null). A lone surrogate +// written as \uD800 is accepted, because that is how JSON.stringify writes a key that has one. +function readStringLiteral(expr: string, start: number): { value: string; end: number } | null { + const quote = expr[start]; + let value = ""; + for (let i = start + 1; i < expr.length; i++) { + const ch = expr[i]; + if (ch === quote) return { value, end: i + 1 }; + if (ch !== "\\") { + value += ch; + continue; + } + const next = expr[++i]; + if (next === quote || (next !== undefined && Object.hasOwn(ESCAPES, next))) { + value += next === quote ? next : ESCAPES[next]; + } else if (next === "u" && /^[0-9A-Fa-f]{4}$/.test(expr.slice(i + 1, i + 5))) { + value += String.fromCharCode(parseInt(expr.slice(i + 1, i + 5), 16)); + i += 4; + } else { + return null; + } + } + return null; +} + +const isBlank = (ch: string | undefined) => ch === " " || ch === "\t" || ch === "\n" || ch === "\r"; +const skipBlanks = (text: string, from: number) => { + while (isBlank(text[from])) from++; + return from; +}; + +// The index after the `]` that closes the bracket opened at `start`, skipping over strings and nested brackets. +function bracketEnd(expr: string, start: number): number { + let depth = 0; + for (let i = start; i < expr.length; i++) { + const ch = expr[i]; + if (ch === '"' || ch === "'") { + for (i++; i < expr.length && expr[i] !== ch; i++) if (expr[i] === "\\") i++; + } else if (ch === "[") { + depth++; + } else if (ch === "]" && --depth === 0) { + return i + 1; + } + } + return expr.length; +} + +interface PreparedPath { + expr: string; + /** Alias token for each quoted name that needs one, in the order they were met. */ + aliases: { token: string; name: string }[]; +} + +// Rewrites the bracket selectors that hold exactly one quoted name; every other bracket is copied as written. +function prepareExpression(expr: string, sentinel: string): PreparedPath { + const aliases: PreparedPath["aliases"] = []; + let out = ""; + let i = 0; + while (i < expr.length) { + if (expr[i] !== "[") { + out += expr[i++]; + continue; + } + const end = bracketEnd(expr, i); + const literalStart = skipBlanks(expr, i + 1); + const quote = expr[literalStart]; + const literal = quote === '"' || quote === "'" ? readStringLiteral(expr, literalStart) : null; + if (!literal || skipBlanks(expr, literal.end) !== end - 1) { + out += expr.slice(i, end); + i = end; + continue; + } + if (PLAIN_NAME.test(literal.value)) { + // Nothing to protect, unless an escape or a space around the literal has to be read first. + const written = expr.slice(literalStart, literal.end); + const asWritten = expr.slice(i + 1, end - 1) === written && !written.includes("\\"); + out += asWritten ? expr.slice(i, end) : `['${literal.value}']`; + } else { + const token = `${sentinel}${aliases.length}${sentinel}`; + aliases.push({ token, name: literal.value }); + out += `['${token}']`; + } + i = end; + } + return { expr: out, aliases }; +} + +// Adds each alias to every object that has the key it stands for. It is not enumerable, so it never shows up +// as a key (`*`, `..`, `~`, filters, `Object.keys`), and the library still finds it by name. +function addAliases(root: unknown, aliases: PreparedPath["aliases"]): void { + const stack: unknown[] = [root]; + while (stack.length) { + const node = stack.pop(); + if (node === null || typeof node !== "object") continue; + if (!Array.isArray(node)) { + for (const { token, name } of aliases) { + if (Object.hasOwn(node, name)) { + Object.defineProperty(node, token, { + value: (node as Record)[name], + enumerable: false, + configurable: true, + writable: true, + }); + } + } + } + for (const child of Object.values(node)) stack.push(child); + } +} + +// A private-use character that is in neither the expression nor the JSON text, so an alias can never be +// mistaken for a key or a value (a private-use character in the data is either written as itself or as \uXXXX). +function pickSentinel(raw: string, path: string): string | null { + const lower = raw.toLowerCase(); + for (let code = 0xe000; code <= 0xe0ff; code++) { + const ch = String.fromCharCode(code); + const escaped = `\\u${code.toString(16)}`; + if (!raw.includes(ch) && !path.includes(ch) && !lower.includes(escaped)) return ch; + } + return null; +} + +const escapePointerToken = (name: string) => name.replace(/~/g, "~0").replace(/\//g, "~1"); + export function runJsonPath(raw: string, path: string): JsonPathResult { if (!raw.trim()) return { ok: false, matches: [], paths: [], error: "Paste JSON first" }; if (!path.trim()) return { ok: false, matches: [], paths: [], error: "Enter a JSONPath expression" }; @@ -33,8 +171,16 @@ export function runJsonPath(raw: string, path: string): JsonPathResult { return { ok: false, matches: [], paths: [], error: `Invalid JSON: ${e instanceof Error ? e.message : "parse error"}` }; } try { + const sentinel = pickSentinel(raw, path); + const prepared = sentinel ? prepareExpression(path, sentinel) : { expr: path, aliases: [] }; + if (prepared.aliases.length) addAliases(parsed, prepared.aliases); + // Where a result names an alias, put the real key back. + const names = new Map(prepared.aliases.map((a) => [a.token, a.name])); + const aliasToken = sentinel ? new RegExp(`${sentinel}\\d+${sentinel}`, "g") : null; + const unalias = (text: string, escape = (name: string) => name) => + aliasToken && names.size ? text.replace(aliasToken, (token) => escape(names.get(token) ?? token)) : text; const result = JSONPath({ - path, + path: prepared.expr, json: parsed as object, resultType: "all", wrap: true, @@ -42,8 +188,10 @@ export function runJsonPath(raw: string, path: string): JsonPathResult { return { ok: true, // A string match keeps its own value: for a property name (`$.*~`) the pointer is that property's value. - matches: result.map((r) => (typeof r.value === "string" ? r.value : atPointer(lossless, r.pointer))), - paths: result.map((r) => r.path), + matches: result.map((r) => + typeof r.value === "string" ? unalias(r.value) : atPointer(lossless, unalias(r.pointer, escapePointerToken)), + ), + paths: result.map((r) => unalias(r.path)), }; } catch (e) { return {