diff --git a/src/worker/index.ts b/src/worker/index.ts index 121b32f..07a96fd 100644 --- a/src/worker/index.ts +++ b/src/worker/index.ts @@ -2,6 +2,68 @@ import type { WorkerAPI, WorkerMessageRequest, WorkerMessageResponse } from "$ut import { advancedQuery } from "./advancedQuery"; import { deserialize, simpleQuery, type UnicodeMappings } from "@emnudge/unicode-query"; +/** + * Expands First/Last range markers in UnicodeData.txt format. + * + * The Unicode data file uses range markers like: + * 3400;;Lo;0;L;;;;;N;;;;; + * 4DBF;;Lo;0;L;;;;;N;;;;; + * + * This function expands these to include all codepoints in the range. + */ +function expandUnicodeRanges(unicodeData: string): string { + const lines = unicodeData.split("\n"); + const result: string[] = []; + + for (let i = 0; i < lines.length; i++) { + const line = lines[i]; + if (!line.trim()) { + result.push(line); + continue; + } + + const parts = line.split(";"); + const label = parts[1] || ""; + + // Check if this is a "First" range marker + if (label.includes(", First>")) { + // Look for the corresponding "Last" marker on the next line + const nextLine = lines[i + 1]; + if (nextLine) { + const nextParts = nextLine.split(";"); + const nextLabel = nextParts[1] || ""; + + if (nextLabel.includes(", Last>")) { + const startCodepoint = parseInt(parts[0], 16); + const endCodepoint = parseInt(nextParts[0], 16); + + // Extract the base name (e.g., "CJK Ideograph Extension A" from "") + const baseName = label.replace(/^$/, ""); + + // Generate entries for all codepoints in the range + for (let cp = startCodepoint; cp <= endCodepoint; cp++) { + const cpHex = cp.toString(16).toUpperCase().padStart(4, "0"); + // Create a new entry with the base name and codepoint + const newParts = [...parts]; + newParts[0] = cpHex; + newParts[1] = `<${baseName}-${cpHex}>`; + result.push(newParts.join(";")); + } + + // Skip the "Last" line since we've processed the range + i++; + continue; + } + } + } + + // Not a range marker, keep the line as-is + result.push(line); + } + + return result.join("\n"); +} + let unicodeMappingsCache: UnicodeMappings | null = null; async function init() { if (!unicodeMappingsCache) { @@ -13,7 +75,7 @@ async function init() { unicodeMappingsCache = deserialize({ blocks, - unicodeData, + unicodeData: expandUnicodeRanges(unicodeData), symbolHtmlNames, }); }