Skip to content

fix(main): optimize order search with indexed candidate retrieval #1204

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
khaira777 merged 3 commits into from
Oct 17, 2026
Merged

fix(main): optimize order search with indexed candidate retrieval #1204

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
254c85e
perf(orders): optimize order search with indexed candidate retrieval …
khaira777 Oct 17, 2026
722409c
no-mistakes(review): use codepoint length for trigram search eligibility
khaira777 Oct 17, 2026
bd8a8e2
no-mistakes(document): format changed search test files and verify docs
khaira777 Oct 17, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
92 changes: 91 additions & 1 deletion main/db.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1323,6 +1323,19 @@ function recoverInterruptedDatabaseReplacement(dbPath: string, backupDir: string
}
}

export function registerOrderSearchFunctions(dbInstance: Database.Database): void {
try {
dbInstance.function('orders_search_contains', (value: string | null, query: string) =>
typeof value === 'string' && value.toLowerCase().includes(query.toLowerCase()) ? 1 : 0,
);
dbInstance.function('orders_search_digits_contains', (value: string | null, digits: string) =>
typeof value === 'string' && value.replace(/\D/g, '').includes(digits) ? 1 : 0,
);
} catch {
// Function already registered or connection closed
}
}

export function initDatabase(
recoverInterruptedReplacement = true,
allowDuringShutdown = false,
Expand Down Expand Up @@ -1350,6 +1363,7 @@ export function initDatabase(
console.log(`[DB] Opening database at: ${dbPath}`);
dbHealthError = null;
db = new Database(dbPath);
registerOrderSearchFunctions(db);
db.pragma('journal_mode = WAL');
db.pragma('synchronous = NORMAL');
db.pragma('busy_timeout = 5000');
Expand Down Expand Up @@ -2409,7 +2423,11 @@ export function getTables(dbInstance: Database.Database): string[] {
const tables = dbInstance
.prepare(`
SELECT name FROM sqlite_master WHERE type='table'
AND name NOT LIKE 'sqlite_%' AND name <> '_flo_meta'
AND (sql NOT LIKE 'CREATE VIRTUAL TABLE%' OR sql IS NULL)
AND name NOT LIKE 'sqlite_%'
AND name NOT LIKE '%_fts_%'
AND name NOT LIKE '%_fts'
AND name <> '_flo_meta'
`)
.all() as { name: string }[];
return tables.map((t) => t.name);
Expand Down Expand Up @@ -4093,6 +4111,7 @@ function dataOnlyRestore(
ON CONFLICT(key) DO UPDATE SET value = excluded.value, updated_at = excluded.updated_at
`)
.run(now());
rebuildSearchIndex(currentDb);
const inventoryValidationError = validateInventoryLedgerDatabase(currentDb);
if (inventoryValidationError) {
throw new Error(`Restore would violate the inventory ledger: ${inventoryValidationError}`);
Expand Down Expand Up @@ -4247,9 +4266,31 @@ export function getCurrentSchemaVersion(): number {
return (db ?? getDatabase()).pragma('user_version', { simple: true }) as number;
}

export function rebuildSearchIndex(targetDb?: Database.Database): void {
const target = targetDb || db;
if (!target) return;
try {
const hasOrdersFts = target
.prepare("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'orders_search_fts'")
.get();
if (hasOrdersFts) {
target.exec("INSERT INTO orders_search_fts(orders_search_fts) VALUES('rebuild')");
}
const hasCustomersFts = target
.prepare("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'customers_search_fts'")
.get();
if (hasCustomersFts) {
target.exec("INSERT INTO customers_search_fts(customers_search_fts) VALUES('rebuild')");
}
} catch (err) {
console.warn('[DB] rebuildSearchIndex error:', err);
}
}

/** Builds in-memory database using migration pipeline as ideal schema reference. */
export function buildIdealSchemaDb(): Database.Database {
const idealDb = new Database(':memory:');
registerOrderSearchFunctions(idealDb);
idealDb.pragma('foreign_keys = OFF'); // Off during migrations
const previousDb = db;
db = idealDb;
Expand Down Expand Up @@ -7348,6 +7389,55 @@ export const MIGRATIONS: { version: number; name: string; up: () => void }[] = [
`);
},
},
{
version: 115,
name: 'add_order_and_customer_search_fts',
up: () => {
db.exec(`
CREATE VIRTUAL TABLE IF NOT EXISTS orders_search_fts USING fts5(
order_number,
content='orders',
content_rowid='id',
tokenize='trigram'
);
CREATE TRIGGER IF NOT EXISTS trg_orders_search_ai AFTER INSERT ON orders BEGIN
INSERT INTO orders_search_fts(rowid, order_number) VALUES (new.id, new.order_number);
END;
CREATE TRIGGER IF NOT EXISTS trg_orders_search_ad AFTER DELETE ON orders BEGIN
INSERT INTO orders_search_fts(orders_search_fts, rowid, order_number) VALUES('delete', old.id, old.order_number);
END;
CREATE TRIGGER IF NOT EXISTS trg_orders_search_au AFTER UPDATE OF order_number ON orders BEGIN
INSERT INTO orders_search_fts(orders_search_fts, rowid, order_number) VALUES('delete', old.id, old.order_number);
INSERT INTO orders_search_fts(rowid, order_number) VALUES (new.id, new.order_number);
END;

CREATE VIRTUAL TABLE IF NOT EXISTS customers_search_fts USING fts5(
name,
phone,
phone_digits,
content='customers',
content_rowid='rowid',
tokenize='trigram'
);
CREATE TRIGGER IF NOT EXISTS trg_customers_search_ai AFTER INSERT ON customers BEGIN
INSERT INTO customers_search_fts(rowid, name, phone, phone_digits) VALUES (new.rowid, new.name, new.phone, new.phone_digits);
END;
CREATE TRIGGER IF NOT EXISTS trg_customers_search_ad AFTER DELETE ON customers BEGIN
INSERT INTO customers_search_fts(customers_search_fts, rowid, name, phone, phone_digits) VALUES('delete', old.rowid, old.name, old.phone, old.phone_digits);
END;
CREATE TRIGGER IF NOT EXISTS trg_customers_search_au AFTER UPDATE OF name, phone, phone_digits ON customers BEGIN
INSERT INTO customers_search_fts(customers_search_fts, rowid, name, phone, phone_digits) VALUES('delete', old.rowid, old.name, old.phone, old.phone_digits);
INSERT INTO customers_search_fts(rowid, name, phone, phone_digits) VALUES (new.rowid, new.name, new.phone, new.phone_digits);
END;

INSERT INTO orders_search_fts(rowid, order_number)
SELECT id, order_number FROM orders;

INSERT INTO customers_search_fts(rowid, name, phone, phone_digits)
SELECT rowid, name, phone, phone_digits FROM customers;
`);
},
},
];

function syncBackupBeforeMigration(fromVersion: number, toVersion: number): void {
Expand Down
81 changes: 70 additions & 11 deletions main/routes/orders.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ import {
utcTodayDate,
recordOrderAudit,
isKdsEnabled,
registerOrderSearchFunctions,
} from '../db';
import { asyncHandler } from '../middleware/async-handler';
import {
Expand Down Expand Up @@ -70,6 +71,25 @@ import { isOperationReference, serializeOperationFailure } from '../lib/operatio

type PinApproverCandidate = { id: string; role: string; pin_hash: string };

const registeredSearchDatabases = new WeakSet<object>();

function ensureOrderSearchFunctions(dbInstance: any): void {
if (registeredSearchDatabases.has(dbInstance)) return;
registerOrderSearchFunctions(dbInstance);
registeredSearchDatabases.add(dbInstance);
}

function isSearchIndexAvailable(dbInstance: any): boolean {
try {
const row = dbInstance
.prepare("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'orders_search_fts'")
.get();
return Boolean(row);
} catch {
return false;
}
}

const router = Router();
const ORDER_WRITE_RATE_LIMIT_ERROR = {
error: 'Too many requests, please try again later.',
Expand Down Expand Up @@ -638,28 +658,67 @@ router.get(
}
const search = typeof req.query.search === 'string' ? req.query.search.trim() : '';
if (search) {
db.function('orders_search_contains', (value: string | null, query: string) =>
typeof value === 'string' && value.toLowerCase().includes(query.toLowerCase()) ? 1 : 0,
);
db.function('orders_search_digits_contains', (value: string | null, digits: string) =>
typeof value === 'string' && value.replace(/\D/g, '').includes(digits) ? 1 : 0,
);
ensureOrderSearchFunctions(db);
const phoneDigits = search.replace(/\D/g, '');
const isPhoneSearch = Boolean(phoneDigits && !/\p{L}/u.test(search));
const customerSearch = ['orders_search_contains(search_customer.name, ?)'];
if (phoneDigits && !/\p{L}/u.test(search)) {
if (isPhoneSearch) {
customerSearch.push('orders_search_digits_contains(search_customer.phone_digits, ?)');
customerSearch.push('orders_search_digits_contains(search_customer.phone, ?)');
}
wheres.push(`(
const udfClause = `(
orders_search_contains(orders.order_number, ?)
OR EXISTS (
SELECT 1 FROM customers AS search_customer
WHERE search_customer.id = orders.customer_id
AND (${customerSearch.join(' OR ')})
)
)`);
params.push(search, search);
if (phoneDigits && !/\p{L}/u.test(search)) params.push(phoneDigits, phoneDigits);
)`;

const isEligible = [...search].length >= 3 && (!isPhoneSearch || phoneDigits.length >= 3);
const hasFts = isEligible && isSearchIndexAvailable(db);

if (hasFts) {
let candidateClause: string;
const candidateParams: any[] = [];

if (isPhoneSearch) {
// For phone searches, customer phone strings may contain varied formatting
// (e.g. "+1/555 ext 0199"). Querying customers table with digit UDF efficiently
// identifies matching customer IDs across the small customers table, avoiding a
// full orders scan while preserving full parity across phone formats.
const ftsSearch = '"' + phoneDigits.replace(/"/g, '""') + '"';
candidateClause = `(
orders.id IN (SELECT rowid FROM orders_search_fts WHERE orders_search_fts MATCH ?)
OR orders.customer_id IN (
SELECT id FROM customers
WHERE (phone_digits IS NOT NULL AND orders_search_digits_contains(phone_digits, ?))
OR (phone IS NOT NULL AND orders_search_digits_contains(phone, ?))
OR orders_search_contains(name, ?)
)
)`;
candidateParams.push(ftsSearch, phoneDigits, phoneDigits, search);
} else {
const ftsSearch = '"' + search.replace(/"/g, '""') + '"';
candidateClause = `(
orders.id IN (SELECT rowid FROM orders_search_fts WHERE orders_search_fts MATCH ?)
OR orders.customer_id IN (
SELECT id FROM customers WHERE rowid IN (
SELECT rowid FROM customers_search_fts WHERE customers_search_fts MATCH ?
)
)
)`;
candidateParams.push(ftsSearch, ftsSearch);
}

wheres.push(`(${candidateClause} AND ${udfClause})`);
params.push(...candidateParams, search, search);
if (isPhoneSearch) params.push(phoneDigits, phoneDigits);
} else {
wheres.push(udfClause);
params.push(search, search);
if (isPhoneSearch) params.push(phoneDigits, phoneDigits);
}
}
// Filter by UTC day or date range across indexed created_at column.
if (req.query.today && req.query.today !== '0' && req.query.today !== 'false') {
Expand Down
3 changes: 2 additions & 1 deletion package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -252,7 +252,8 @@
"test:rtl-dashboard-pos-common": "ts-node --transpile-only -P tests/tsconfig.json tests/rtl-dashboard-pos-common.test.ts",
"test:rtl-kds-server-whatsapp": "ts-node --transpile-only -P tests/tsconfig.json tests/rtl-kds-server-whatsapp.test.ts",
"test:phone": "ts-node --transpile-only -P tests/tsconfig.json tests/phone.test.ts",
"test:orders-search": "ts-node --transpile-only -P tests/tsconfig.json tests/orders-search.test.ts && node tests/run-electron-node-test.cjs tests/orders-search-api.test.ts",
"test:orders-search": "ts-node --transpile-only -P tests/tsconfig.json tests/orders-search.test.ts && node tests/run-electron-node-test.cjs tests/orders-search-api.test.ts && npm run test:orders-search-performance",
"test:orders-search-performance": "node tests/run-electron-node-test.cjs tests/orders-search-performance.test.ts",
"test:country-localization": "ts-node --transpile-only -P tests/tsconfig.json tests/country-localization.test.ts",
"test:currency": "ts-node --transpile-only -P tests/tsconfig.json tests/currency.test.ts && ts-node --transpile-only -P tests/tsconfig.json tests/country-profile.test.ts && ts-node --transpile-only -P tests/tsconfig.json tests/locale-iran.test.ts && ts-node --transpile-only -P tests/tsconfig.json tests/regional-snapshot.test.ts && node tests/run-electron-node-test.cjs tests/currency-split-checks.test.ts && node tests/run-electron-node-test.cjs tests/currency-payments-refunds-minor-units.test.ts && node tests/run-electron-node-test.cjs tests/currency-reset.test.ts",
"test:currency-split": "node tests/run-electron-node-test.cjs tests/currency-split-checks.test.ts",
Expand Down
Loading
Loading