fix(json-formatter): find JSONPath quoted names with escapes, brackets or the other quote

lib/__tests__/jsonpath.test.ts

@@ -2,6 +2,7 @@ import { describe, it, expect } from "vitest";
 import { LosslessNumber, stringify } from "lossless-json";
 import { runJsonPath } from "../jsonpath";
 import { SAMPLE_JSON } from "../json-utils";
+import { buildJsonTreePath } from "../json-tree-path";
 
 describe("runJsonPath", () => {
   it.each(["", " \t\n "])("rejects blank JSON %j", (raw) => {
@@ -96,3 +97,197 @@ describe("runJsonPath", () => {
     });
   });
 });
+
+describe("runJsonPath quoted names (RFC 9535 string literals)", () => {
+  const found = (raw: string, path: string) => stringify(runJsonPath(raw, path).matches);
+  const AWKWARD = '{"q\\"uote": 5, "back\\\\slash": 6, "br[ack]et": 9, "it\'s": 10, "tab\\there": 11}';
+
+  it.each([
+    ['$["q\\"uote"]', "[5]"],
+    ['$["back\\\\slash"]', "[6]"],
+    ['$["br[ack]et"]', "[9]"],
+    ['$["it\'s"]', "[10]"],
+    ['$["tab\\there"]', "[11]"],
+    ["$['it\\'s']", "[10]"],
+  ])("finds the value for %s", (path, expected) => {
+    expect(found(AWKWARD, path)).toBe(expected);
+  });
+
+  it("keeps the paths that already worked working", () => {
+    const raw = '{"a b": 1, "": 2, "q\\"uote": 5}';
+    expect(found(raw, '$["a b"]')).toBe("[1]");
+    expect(found(raw, "$['a b']")).toBe("[1]");
+    expect(found(raw, '$[""]')).toBe("[2]");
+    expect(found(raw, "$['q\"uote']")).toBe("[5]");
+  });
+
+  it("reads every escape of a string literal", () => {
+    const raw = JSON.stringify({ "a/b": 1, "a\bb": 2, "a\fb": 3, "a\nb": 4, "a\rb": 5, "a\tb": 6, "a\\b": 7, "é": 8, "😀": 9 });
+    expect(found(raw, '$["a\\/b"]')).toBe("[1]");
+    expect(found(raw, '$["a\\bb"]')).toBe("[2]");
+    expect(found(raw, '$["a\\fb"]')).toBe("[3]");
+    expect(found(raw, '$["a\\nb"]')).toBe("[4]");
+    expect(found(raw, '$["a\\rb"]')).toBe("[5]");
+    expect(found(raw, '$["a\\tb"]')).toBe("[6]");
+    expect(found(raw, '$["a\\\\b"]')).toBe("[7]");
+    expect(found(raw, '$["\\u00e9"]')).toBe("[8]");
+    expect(found(raw, '$["\\u00E9"]')).toBe("[8]");
+    expect(found(raw, '$["\\ud83d\\ude00"]')).toBe("[9]");
+  });
+
+  it("takes [, ], . and , inside the quotes as characters", () => {
+    const raw = '{"a[0]": 1, "a]": 2, "a.b": 3, "a,b": 4, "a][b": 5}';
+    expect(found(raw, '$["a[0]"]')).toBe("[1]");
+    expect(found(raw, '$["a]"]')).toBe("[2]");
+    expect(found(raw, '$["a.b"]')).toBe("[3]");
+    expect(found(raw, "$['a,b']")).toBe("[4]");
+    expect(found(raw, '$["a][b"]')).toBe("[5]");
+  });
+
+  it("matches a name that looks like an operator as a name, not as the operator", () => {
+    const raw = '{"*": 1, "..": 2, "^": 3, "~": 4, "$": 5, "@": 6, "1:2": 7, "?(@.x)": 8, "(1)": 9, "a": 10}';
+    for (const [name, value] of [["*", 1], ["..", 2], ["^", 3], ["~", 4], ["$", 5], ["@", 6], ["1:2", 7], ["?(@.x)", 8], ["(1)", 9]] as const) {
+      expect(found(raw, `$[${JSON.stringify(name)}]`)).toBe(`[${value}]`);
+      expect(found(raw, `$['${name}']`)).toBe(`[${value}]`);
+    }
+  });
+
+  it("finds the value for the path buildJsonTreePath writes, whatever the key", () => {
+    const keys = [
+      '', 'a b', 'q"uote', "it's", 'a\'b"c', "back\\slash", "\\", '\\"', "br[ack]et", "a]", "[0]", "']", '"]',
+      "dot.key", "til~de", "sl/ash", "a~1b", "a~0b", "~0", "~1", "a,b", ";", "%", "$ref", "$", "@", "*", "..", "^", "~", "?(@.x)", "(1)", "1:2", "-1", "0",
+      "tab\there", "new\nline", "cr\rhere", "\u0000", "\u0008", "\u001f", "\u007f", "\u2028", "\ud800", "\udc00",
+      "é", "日本語", "😀", "a\u{10ffff}", "__proto__", "constructor", "length",
+    ];
+    const raw = JSON.stringify(Object.fromEntries(keys.map((key, i) => [key, i])), null, 1);
+    // JSON.stringify drops __proto__ as an own key when built from an object, so write it by hand.
+    const body = "{" + keys.map((key, i) => `${JSON.stringify(key)}: ${i}`).join(", ") + "}";
+    for (const text of [body, raw]) {
+      keys.forEach((key, i) => {
+        const path = buildJsonTreePath([{ type: "property", key }]);
+        const result = runJsonPath(text, path);
+        expect({ key, path, matches: stringify(result.matches) }).toEqual({ key, path, matches: `[${i}]` });
+      });
+    }
+  });
+
+  it("finds a nested value through awkward keys and array indexes", () => {
+    const raw = '{"it\'s": [{"a.b": {"c]": 1}}, {"a.b": {"c]": 2}}]}';
+    const path = buildJsonTreePath([
+      { type: "property", key: "it's" },
+      { type: "index", index: 1 },
+      { type: "property", key: "a.b" },
+      { type: "property", key: "c]" },
+    ]);
+    expect(found(raw, path)).toBe("[2]");
+  });
+
+  it("finds the name through a wildcard, a recursive descent and a filter", () => {
+    const raw = '[{"it\'s": 1, "x": 0}, {"it\'s": 2, "x": 0}, {"deep": {"it\'s": 3}}]';
+    expect(found(raw, '$[*]["it\'s"]')).toBe("[1,2]");
+    expect(found(raw, '$..["it\'s"]')).toBe("[1,2,3]");
+    expect(found(raw, '$[?(@.x==0)]["it\'s"]')).toBe("[1,2]");
+  });
+
+  it("returns a property name as the name, and keeps the other lookups free of the lookup alias", () => {
+    const raw = '{"it\'s": 10, "a": 1}';
+    expect(runJsonPath(raw, '$["it\'s"]~').matches).toEqual(["it's"]);
+    expect(runJsonPath(raw, "$.*~").matches).toEqual(["it's", "a"]);
+    expect(found(raw, "$.*")).toBe("[10,1]");
+    expect(found(raw, "$..*")).toBe("[10,1]");
+    expect(stringify(runJsonPath(raw, '$["it\'s"]').matches)).toBe("[10]");
+  });
+
+  it("names the matched key in the result path", () => {
+    expect(runJsonPath('{"it\'s": 10}', '$["it\'s"]').paths).toEqual(["$['it's']"]);
+    expect(runJsonPath('{"q\\"uote": {"a": 1}}', '$["q\\"uote"].a').paths).toEqual(["$['q\"uote']['a']"]);
+  });
+
+  it("keeps large numbers exact when they are found by a quoted name", () => {
+    expect(found('{"it\'s": 12345678901234567890}', '$["it\'s"]')).toBe("[12345678901234567890]");
+  });
+
+  it("does not change the data it searches: a hidden lookup never shows up as a key", () => {
+    const raw = '{"it\'s": {"it\'s": 1}, "b": 2}';
+    expect(found(raw, '$["it\'s"]')).toBe('[{"it\'s":1}]');
+    expect(found(raw, '$["it\'s"]["it\'s"]')).toBe("[1]");
+    expect(runJsonPath(raw, "$..*~").matches).toEqual(["it's", "b", "it's"]);
+  });
+
+  it("reads the blanks RFC 9535 allows inside the brackets, for plain and awkward names alike", () => {
+    const raw = '{"a": 1, "it\'s": 2}';
+    expect(found(raw, '$[ "a" ]')).toBe("[1]");
+    expect(found(raw, "$[ 'a']")).toBe("[1]");
+    expect(found(raw, '$["a" ]')).toBe("[1]");
+    expect(found(raw, '$[ "it\'s" ]')).toBe("[2]");
+  });
+
+  it("reads an escape in a name that is otherwise plain", () => {
+    expect(found('{"a b": 1}', '$["a\\u0020b"]')).toBe("[1]");
+    expect(found('{"a\\nb": 1}', '$["a\\nb"]')).toBe("[1]");
+  });
+
+  it("follows two different awkward names in one path, and the same one twice", () => {
+    const raw = '{"a.b": {"c]": {"a.b": 3}}}';
+    expect(found(raw, '$["a.b"]["c]"]["a.b"]')).toBe("[3]");
+    expect(found(raw, "$['a.b']['c]']")).toBe('[{"a.b":3}]');
+  });
+
+  it("is not fooled by a key that looks like the lookup it adds", () => {
+    // Whatever private-use character the lookup is built from, a document that already uses one still works,
+    // whether the character is written as itself or as an escape in either case.
+    for (const code of ["e000", "e001", "e002"]) {
+      const char = String.fromCharCode(parseInt(code, 16));
+      const key = `${char}0${char}`;
+      for (const written of [key, `\\u${code}0\\u${code}`, `\\u${code.toUpperCase()}0\\u${code.toUpperCase()}`]) {
+        const raw = `{"it's": 1, "${written}": 2}`;
+        expect(found(raw, '$["it\'s"]')).toBe("[1]");
+        expect(found(raw, `$["${key}"]`)).toBe("[2]");
+        // The parent, listed after the lookup ran, still has both keys, with their own values.
+        expect(runJsonPath(raw, '$["it\'s"]^.*~').matches).toEqual(["it's", key]);
+        expect(found(raw, '$["it\'s"]^.*')).toBe("[1,2]");
+      }
+    }
+  });
+
+  it("does not let the expression mention the lookup either", () => {
+    const token = String.fromCharCode(0xe000) + "0" + String.fromCharCode(0xe000);
+    // A filter that asks for a property named like the first lookup must not find the hidden one.
+    expect(found('[{"it\'s": 1}]', `$[?(@['${token}'])]["it's"]`)).toBe("[]");
+    expect(found('[{"it\'s": 1}]', `$[0]["it's"]`)).toBe("[1]");
+    expect(found('{"it\'s": {"\\ue000": 5}}', `$["it's"]["${String.fromCharCode(0xe000)}"]`)).toBe("[5]");
+  });
+
+  it("returns nothing, without failing, when the data is not an object", () => {
+    expect(runJsonPath("5", '$["it\'s"]')).toEqual({ ok: true, matches: [], paths: [] });
+    expect(found('[{"it\'s": 1}, [{"it\'s": 2}]]', '$[0]["it\'s"]')).toBe("[1]");
+    expect(found('[{"it\'s": 1}, [{"it\'s": 2}]]', '$[1][0]["it\'s"]')).toBe("[2]");
+  });
+
+  it("keeps reading a literal with an escape RFC 9535 does not define the way it always was, as the literal characters", () => {
+    // \q is not an escape, and neither is \' in double quotes or \" in single quotes, so these selectors are not
+    // rewritten and keep their old result: the key that has the backslash in it, or nothing, never the key
+    // without the backslash.
+    const raw = JSON.stringify({ a: 1, "a\\qb": 2, "it\\'s": 3, "it's": 4, 'q\\"t': 5 });
+    expect(found(raw, '$["a\\qb"]')).toBe("[2]");
+    expect(found(raw, '$["a"]')).toBe("[1]");
+    expect(found(raw, '$["it\\\'s"]')).toBe("[]");
+    expect(found(raw, "$['it\\'s']")).toBe("[4]");
+    expect(found(raw, "$['q\\\"t']")).toBe("[5]");
+  });
+
+  it("does not shorten a bracket that holds more than one name", () => {
+    // jsonpath-plus finds nothing for a quoted union; it must not be turned into its first name.
+    const raw = '{"a": 1, "b": 2}';
+    expect(found(raw, "$['a','b']")).toBe("[]");
+    expect(found(raw, '$["a","b"]')).toBe("[]");
+  });
+
+  it("lists the keys of an object that has a lookup without the lookup", () => {
+    const raw = '{"a.b": {"a.b": 1, "c": 2}}';
+    expect(found(raw, '$["a.b"].*')).toBe("[1,2]");
+    expect(runJsonPath(raw, '$["a.b"].*').paths).toEqual(["$['a.b']['a.b']", "$['a.b']['c']"]);
+    expect(runJsonPath(raw, '$["a.b"].*~').matches).toEqual(["a.b", "c"]);
+    expect(found(raw, '$["a.b"]..*')).toBe("[1,2]");
+  });
+});

lib/json-tree-path.ts

@@ -11,7 +11,8 @@ export function buildJsonTreePath(segments: JsonPathSegment[]): string {
       return `${path}[${segment.index}]`;
     }
 
-    if (SIMPLE_PROPERTY.test(segment.key)) {
+    // A bare `$` after a dot is the root, not a property, so a key called `$` is quoted like the other special names.
+    if (SIMPLE_PROPERTY.test(segment.key) && segment.key !== "$") {
       return `${path}.${segment.key}`;
     }
 

lib/jsonpath.ts

@@ -19,6 +19,144 @@ function atPointer(tree: unknown, pointer: string): unknown {
     );
 }
 
+// ---- Quoted names ----------------------------------------------------------------------------
+// jsonpath-plus splits an expression with regular expressions, so a quoted name that holds a quote,
+// a backslash, a bracket or a dot, or that reads like an operator (`*`, `..`, `^`, `~`, `$`, `a,b`,
+// `1:2`, `?(...)`), is cut up or run as the operator. Names it already reads correctly are left
+// exactly as they are. Any other quoted name is read the way RFC 9535 defines a string literal and
+// replaced by a harmless alias that is also added, hidden, to the parsed data, so the library
+// finds the real key through it.
+
+// Names the library reads correctly when they are written without an escape.
+const PLAIN_NAME = /^[A-Za-z0-9_ -]*$/;
+
+const ESCAPES: Record<string, string> = { b: "\b", f: "\f", n: "\n", r: "\r", t: "\t", "\\": "\\", "/": "/" };
+
+// A string literal that starts at `start` (a quote): its value and the index after its closing quote.
+// An escape RFC 9535 does not define, or a literal that never closes, is not read (null). A lone surrogate
+// written as \uD800 is accepted, because that is how JSON.stringify writes a key that has one.
+function readStringLiteral(expr: string, start: number): { value: string; end: number } | null {
+  const quote = expr[start];
+  let value = "";
+  for (let i = start + 1; i < expr.length; i++) {
+    const ch = expr[i];
+    if (ch === quote) return { value, end: i + 1 };
+    if (ch !== "\\") {
+      value += ch;
+      continue;
+    }
+    const next = expr[++i];
+    if (next === quote || (next !== undefined && Object.hasOwn(ESCAPES, next))) {
+      value += next === quote ? next : ESCAPES[next];
+    } else if (next === "u" && /^[0-9A-Fa-f]{4}$/.test(expr.slice(i + 1, i + 5))) {
+      value += String.fromCharCode(parseInt(expr.slice(i + 1, i + 5), 16));
+      i += 4;
+    } else {
+      return null;
+    }
+  }
+  return null;
+}
+
+const isBlank = (ch: string | undefined) => ch === " " || ch === "\t" || ch === "\n" || ch === "\r";
+const skipBlanks = (text: string, from: number) => {
+  while (isBlank(text[from])) from++;
+  return from;
+};
+
+// The index after the `]` that closes the bracket opened at `start`, skipping over strings and nested brackets.
+function bracketEnd(expr: string, start: number): number {
+  let depth = 0;
+  for (let i = start; i < expr.length; i++) {
+    const ch = expr[i];
+    if (ch === '"' || ch === "'") {
+      for (i++; i < expr.length && expr[i] !== ch; i++) if (expr[i] === "\\") i++;
+    } else if (ch === "[") {
+      depth++;
+    } else if (ch === "]" && --depth === 0) {
+      return i + 1;
+    }
+  }
+  return expr.length;
+}
+
+interface PreparedPath {
+  expr: string;
+  /** Alias token for each quoted name that needs one, in the order they were met. */
+  aliases: { token: string; name: string }[];
+}
+
+// Rewrites the bracket selectors that hold exactly one quoted name; every other bracket is copied as written.
+function prepareExpression(expr: string, sentinel: string): PreparedPath {
+  const aliases: PreparedPath["aliases"] = [];
+  let out = "";
+  let i = 0;
+  while (i < expr.length) {
+    if (expr[i] !== "[") {
+      out += expr[i++];
+      continue;
+    }
+    const end = bracketEnd(expr, i);
+    const literalStart = skipBlanks(expr, i + 1);
+    const quote = expr[literalStart];
+    const literal = quote === '"' || quote === "'" ? readStringLiteral(expr, literalStart) : null;
+    if (!literal || skipBlanks(expr, literal.end) !== end - 1) {
+      out += expr.slice(i, end);
+      i = end;
+      continue;
+    }
+    if (PLAIN_NAME.test(literal.value)) {
+      // Nothing to protect, unless an escape or a space around the literal has to be read first.
+      const written = expr.slice(literalStart, literal.end);
+      const asWritten = expr.slice(i + 1, end - 1) === written && !written.includes("\\");
+      out += asWritten ? expr.slice(i, end) : `['${literal.value}']`;
+    } else {
+      const token = `${sentinel}${aliases.length}${sentinel}`;
+      aliases.push({ token, name: literal.value });
+      out += `['${token}']`;
+    }
+    i = end;
+  }
+  return { expr: out, aliases };
+}
+
+// Adds each alias to every object that has the key it stands for. It is not enumerable, so it never shows up
+// as a key (`*`, `..`, `~`, filters, `Object.keys`), and the library still finds it by name.
+function addAliases(root: unknown, aliases: PreparedPath["aliases"]): void {
+  const stack: unknown[] = [root];
+  while (stack.length) {
+    const node = stack.pop();
+    if (node === null || typeof node !== "object") continue;
+    if (!Array.isArray(node)) {
+      for (const { token, name } of aliases) {
+        if (Object.hasOwn(node, name)) {
+          Object.defineProperty(node, token, {
+            value: (node as Record<string, unknown>)[name],
+            enumerable: false,
+            configurable: true,
+            writable: true,
+          });
+        }
+      }
+    }
+    for (const child of Object.values(node)) stack.push(child);
+  }
+}
+
+// A private-use character that is in neither the expression nor the JSON text, so an alias can never be
+// mistaken for a key or a value (a private-use character in the data is either written as itself or as \uXXXX).
+function pickSentinel(raw: string, path: string): string | null {
+  const lower = raw.toLowerCase();
+  for (let code = 0xe000; code <= 0xe0ff; code++) {
+    const ch = String.fromCharCode(code);
+    const escaped = `\\u${code.toString(16)}`;
+    if (!raw.includes(ch) && !path.includes(ch) && !lower.includes(escaped)) return ch;
+  }
+  return null;
+}
+
+const escapePointerToken = (name: string) => name.replace(/~/g, "~0").replace(/\//g, "~1");
+
 export function runJsonPath(raw: string, path: string): JsonPathResult {
   if (!raw.trim()) return { ok: false, matches: [], paths: [], error: "Paste JSON first" };
   if (!path.trim()) return { ok: false, matches: [], paths: [], error: "Enter a JSONPath expression" };
@@ -33,17 +171,27 @@ export function runJsonPath(raw: string, path: string): JsonPathResult {
     return { ok: false, matches: [], paths: [], error: `Invalid JSON: ${e instanceof Error ? e.message : "parse error"}` };
   }
   try {
+    const sentinel = pickSentinel(raw, path);
+    const prepared = sentinel ? prepareExpression(path, sentinel) : { expr: path, aliases: [] };
+    if (prepared.aliases.length) addAliases(parsed, prepared.aliases);
+    // Where a result names an alias, put the real key back.
+    const names = new Map(prepared.aliases.map((a) => [a.token, a.name]));
+    const aliasToken = sentinel ? new RegExp(`${sentinel}\\d+${sentinel}`, "g") : null;
+    const unalias = (text: string, escape = (name: string) => name) =>
+      aliasToken && names.size ? text.replace(aliasToken, (token) => escape(names.get(token) ?? token)) : text;
     const result = JSONPath({
-      path,
+      path: prepared.expr,
       json: parsed as object,
       resultType: "all",
       wrap: true,
     }) as { value: unknown; path: string; pointer: string }[];
     return {
       ok: true,
       // A string match keeps its own value: for a property name (`$.*~`) the pointer is that property's value.
-      matches: result.map((r) => (typeof r.value === "string" ? r.value : atPointer(lossless, r.pointer))),
-      paths: result.map((r) => r.path),
+      matches: result.map((r) =>
+        typeof r.value === "string" ? unalias(r.value) : atPointer(lossless, unalias(r.pointer, escapePointerToken)),
+      ),
+      paths: result.map((r) => unalias(r.path)),
     };
   } catch (e) {
     return {