Skip to content

fix(main): chunk order and addon queries to prevent exceeding SQLite bind limit #1224

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
khaira777 wants to merge 1 commit into
base: main
Choose a base branch
from
Open

fix(main): chunk order and addon queries to prevent exceeding SQLite bind limit #1224

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
87 changes: 51 additions & 36 deletions main/db.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8720,53 +8720,68 @@ export function attachEffectiveAddons<T extends { id: number }>(
})[];

const ids = items.map((item) => item.id);
const placeholders = ids.map(() => '?').join(',');
const rows = dbInstance
.prepare(`SELECT * FROM order_item_addons WHERE order_item_id IN (${placeholders}) ORDER BY id`)
.all(...ids) as {
order_item_id: number;
addon_id: string | null;
addon_name: string;
price: number;
quantity: number;
}[];
const CHUNK_SIZE = 400;

const byItem = new Map<
number,
{ id: string | null; name: string; price: number; quantity: number }[]
>();
for (const row of rows) {
const list = byItem.get(row.order_item_id) || [];
list.push({ id: row.addon_id, name: row.addon_name, price: row.price, quantity: row.quantity });
byItem.set(row.order_item_id, list);
for (let offset = 0; offset < ids.length; offset += CHUNK_SIZE) {
const chunk = ids.slice(offset, offset + CHUNK_SIZE);
const placeholders = chunk.map(() => '?').join(',');
const rows = dbInstance
.prepare(
`SELECT * FROM order_item_addons WHERE order_item_id IN (${placeholders}) ORDER BY id`,
)
.all(...chunk) as {
order_item_id: number;
addon_id: string | null;
addon_name: string;
price: number;
quantity: number;
}[];
for (const row of rows) {
const list = byItem.get(row.order_item_id) || [];
list.push({
id: row.addon_id,
name: row.addon_name,
price: row.price,
quantity: row.quantity,
});
byItem.set(row.order_item_id, list);
}
}

const editVersionByItem = new Map<number, number>();
const supersededItemIds = new Set<number>();
try {
const auditRows = dbInstance
.prepare(
`SELECT id, order_item_id, action, details_json FROM order_audit_log WHERE order_item_id IN (${placeholders}) ORDER BY id`,
)
.all(...ids) as {
id: number;
order_item_id: number | string | null;
action: string;
details_json: string | null;
}[];
for (const audit of auditRows) {
const itemId = Number(audit.order_item_id);
if (!Number.isFinite(itemId)) continue;
const prev = editVersionByItem.get(itemId) ?? 0;
if (audit.id > prev) editVersionByItem.set(itemId, audit.id);
if (audit.action === 'item_edited' && audit.details_json) {
try {
const details = JSON.parse(audit.details_json) as { superseded?: unknown } | null;
if (details?.superseded === true) {
supersededItemIds.add(itemId);
for (let offset = 0; offset < ids.length; offset += CHUNK_SIZE) {
const chunk = ids.slice(offset, offset + CHUNK_SIZE);
const placeholders = chunk.map(() => '?').join(',');
const auditRows = dbInstance
.prepare(
`SELECT id, order_item_id, action, details_json FROM order_audit_log WHERE order_item_id IN (${placeholders}) ORDER BY id`,
)
.all(...chunk) as {
id: number;
order_item_id: number | string | null;
action: string;
details_json: string | null;
}[];
for (const audit of auditRows) {
const itemId = Number(audit.order_item_id);
if (!Number.isFinite(itemId)) continue;
const prev = editVersionByItem.get(itemId) ?? 0;
if (audit.id > prev) editVersionByItem.set(itemId, audit.id);
if (audit.action === 'item_edited' && audit.details_json) {
try {
const details = JSON.parse(audit.details_json) as { superseded?: unknown } | null;
if (details?.superseded === true) {
supersededItemIds.add(itemId);
}
} catch {
// Ignore malformed audit details_json.
}
} catch {
// Ignore malformed audit details_json.
}
}
}
Expand Down
44 changes: 31 additions & 13 deletions main/services/cloud-sync.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3648,11 +3648,19 @@ export class CloudSyncService {

const db = getDatabase();
const orderIds = orders.map((o) => o.id);
const orderPlaceholders = orderIds.map(() => '?').join(',');
const CHUNK_SIZE = 400;

const rawItems: OrderItemSnapshotSource[] = [];
for (let offset = 0; offset < orderIds.length; offset += CHUNK_SIZE) {
const chunk = orderIds.slice(offset, offset + CHUNK_SIZE);
const placeholders = chunk.map(() => '?').join(',');
rawItems.push(
...(db
.prepare(`SELECT * FROM order_items WHERE order_id IN (${placeholders}) ORDER BY id ASC`)
.all(...chunk) as OrderItemSnapshotSource[]),
);
}

const rawItems = db
.prepare(`SELECT * FROM order_items WHERE order_id IN (${orderPlaceholders}) ORDER BY id ASC`)
.all(...orderIds) as OrderItemSnapshotSource[];
const parsedItems = attachEffectiveAddons(db, rawItems.map(parseItemJson));
const itemsByOrderId = new Map<string, unknown[]>();
for (const item of parsedItems) {
Expand All @@ -3670,18 +3678,28 @@ export class CloudSyncService {
);
const tableMap = new Map<string, TableSnapshotSource>();
if (tableIds.length > 0) {
const tablePlaceholders = tableIds.map(() => '?').join(',');
const tableRows = db
.prepare(`SELECT * FROM tables WHERE id IN (${tablePlaceholders})`)
.all(...tableIds) as TableSnapshotSource[];
for (const table of tableRows) {
tableMap.set(String(table.id), table);
for (let offset = 0; offset < tableIds.length; offset += CHUNK_SIZE) {
const chunk = tableIds.slice(offset, offset + CHUNK_SIZE);
const placeholders = chunk.map(() => '?').join(',');
const tableRows = db
.prepare(`SELECT * FROM tables WHERE id IN (${placeholders})`)
.all(...chunk) as TableSnapshotSource[];
for (const table of tableRows) {
tableMap.set(String(table.id), table);
}
}
}

const billRows = db
.prepare(`SELECT * FROM bills WHERE order_id IN (${orderPlaceholders}) ORDER BY id ASC`)
.all(...orderIds) as BillSnapshotSource[];
const billRows: BillSnapshotSource[] = [];
for (let offset = 0; offset < orderIds.length; offset += CHUNK_SIZE) {
const chunk = orderIds.slice(offset, offset + CHUNK_SIZE);
const placeholders = chunk.map(() => '?').join(',');
billRows.push(
...(db
.prepare(`SELECT * FROM bills WHERE order_id IN (${placeholders}) ORDER BY id ASC`)
.all(...chunk) as BillSnapshotSource[]),
);
}
const billsByOrderId = new Map<string, BillSnapshotSource[]>();
for (const bill of billRows) {
const key = String(bill.order_id);
Expand Down
78 changes: 78 additions & 0 deletions tests/cloud-live-orders-batching.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -527,6 +527,84 @@ async function run() {
console.log(' ✓ status filtering restricts returned orders to requested statuses');
}

// ------------------------------------------------------------------------
// Test 6: SQLite bind limit protection with large order/item counts (Issue #1223)
// ------------------------------------------------------------------------
{
console.log('Testing large order/item batching exceeding SQLite bind limit (Issue #1223)...');
db.prepare('DELETE FROM order_item_addons').run();
db.prepare('DELETE FROM order_audit_log').run();
db.prepare('DELETE FROM order_items').run();
db.prepare('DELETE FROM bills').run();
db.prepare('DELETE FROM orders').run();

const insertOrder = db.prepare(
`INSERT INTO orders (order_number, table_id, type, status, subtotal, total, created_at, updated_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?)`,
);
const insertItem = db.prepare(
`INSERT INTO order_items (order_id, product_id, product_name, quantity, unit_price, subtotal, total, created_at, updated_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,
);
const insertAddon = db.prepare(
`INSERT INTO order_item_addons (order_item_id, addon_id, addon_name, price, quantity, created_at)
VALUES (?, ?, ?, ?, ?, ?)`,
);

// Seed 164 orders with 200 items each = 32,800 items (exceeds SQLite 32,766 max bind limit)
const numOrders = 164;
const itemsPerOrder = 200;
const nowIso = now();

db.transaction(() => {
for (let i = 0; i < numOrders; i++) {
const ordId = insertOrder.run(
`ORD-SCALE-${String(i).padStart(4, '0')}`,
null,
'takeaway',
'pending',
100.0,
100.0,
nowIso,
nowIso,
).lastInsertRowid;

for (let j = 0; j < itemsPerOrder; j++) {
const itmId = insertItem.run(
ordId,
`prod-${j}`,
`Item ${j}`,
1,
0.5,
0.5,
0.5,
nowIso,
nowIso,
).lastInsertRowid;

if (j === 0) {
insertAddon.run(itmId, null, 'Extra Sauce', 0.25, 1, nowIso);
}
}
}
})();

const totalItems = db.prepare('SELECT COUNT(*) as count FROM order_items').get() as {
count: number;
};
assertEqual(totalItems.count, 32800, 'Seeded exactly 32,800 order items');

// Call liveOrders - must succeed without throwing "too many SQL variables"
const result = (service as any).liveOrders();
assertEqual(result.orders.length, 164, 'Successfully returned all 164 orders');
const firstOrder = result.orders[0];
assertEqual(firstOrder.items.length, 200, 'First order has 200 items');
assertEqual(firstOrder.items[0].addons.length, 1, 'First item has addon attached');
assertEqual(firstOrder.items[0].addons[0].name, 'Extra Sauce', 'Addon name matches');

console.log(' ✓ 164 orders with 32,800 items handled cleanly without exceeding bind limits');
}

console.log('='.repeat(60));
console.log('✅ All cloud live-orders batching tests passed successfully!');
} finally {
Expand Down
Loading