From b4ea8d15007e8823d380dec0536431e73779f48a Mon Sep 17 00:00:00 2001 From: Muse Date: Tue, 6 Oct 2026 04:47:30 +0000 Subject: [PATCH 1/2] fix(expenses): dedupe double-posted receipts, Personal/Tech sides - sync: collapse same-receipt double-posts (vendor/amount/day/label fingerprint) and skip suppressed idempotency keys, so one receipt is always one net expense row going forward - schema: suppressed_expenses table for ledger rows posted in error - categories: Personal and Tech top-level sides; all business stays under Tech with the existing subcategories (AI, Cloud/Hosting, Domains, Software, Services, Other); Personal empty for now - dashboard: filter bar leads with All / Personal / Tech; title Expenses - 26 unit tests green --- workers/expenses-site/schema.sql | 10 +++ .../src/__tests__/categories.test.mjs | 24 +++++- .../src/__tests__/render.test.mjs | 14 +++- .../expenses-site/src/__tests__/sync.test.mjs | 82 ++++++++++++++++++- workers/expenses-site/src/categories.mjs | 22 ++++- workers/expenses-site/src/index.mjs | 39 +++++---- workers/expenses-site/src/render.mjs | 63 +++++++++----- workers/expenses-site/src/sync.mjs | 73 ++++++++++++++++- 8 files changed, 278 insertions(+), 49 deletions(-) diff --git a/workers/expenses-site/schema.sql b/workers/expenses-site/schema.sql index 0ceebf9c..5b086e0d 100644 --- a/workers/expenses-site/schema.sql +++ b/workers/expenses-site/schema.sql @@ -26,3 +26,13 @@ CREATE TABLE IF NOT EXISTS sync_state ( key TEXT PRIMARY KEY, value TEXT NOT NULL ); + +-- Idempotency keys that must never render on the dashboard, even though they +-- still exist upstream. Used for ledger rows that were posted in error +-- (double-posted receipts, superseded gross/discount/correction rows) and +-- later corrected at the display layer. Sync skips these keys on every run. +CREATE TABLE IF NOT EXISTS suppressed_expenses ( + idempotency_key TEXT PRIMARY KEY, + reason TEXT NOT NULL, + suppressed_at TEXT NOT NULL +); diff --git a/workers/expenses-site/src/__tests__/categories.test.mjs b/workers/expenses-site/src/__tests__/categories.test.mjs index 6098dc33..8028ab73 100644 --- a/workers/expenses-site/src/__tests__/categories.test.mjs +++ b/workers/expenses-site/src/__tests__/categories.test.mjs @@ -1,5 +1,12 @@ import { describe, expect, it } from "vitest"; -import { CATEGORIES, categorizeExpense, categoryLabel } from "../categories.mjs"; +import { + CATEGORIES, + SIDES, + categorizeExpense, + categoryLabel, + sideForCategory, + sideLabel, +} from "../categories.mjs"; describe("categorizeExpense", () => { it("maps AI vendors to tech-ai", () => { @@ -36,8 +43,21 @@ describe("categorizeExpense", () => { }); it("exposes stable Title Case labels", () => { - expect(categoryLabel("tech-ai")).toBe("Tech / AI"); + expect(categoryLabel("tech-ai")).toBe("AI"); expect(categoryLabel("bogus")).toBe("Other"); expect(Object.keys(CATEGORIES)).toContain("tech-ai"); }); + + it("puts everything business under the Tech side", () => { + expect(sideForCategory("tech-ai")).toBe("tech"); + expect(sideForCategory("cloud-hosting")).toBe("tech"); + expect(sideForCategory("domains")).toBe("tech"); + expect(sideForCategory("software")).toBe("tech"); + expect(sideForCategory("services")).toBe("tech"); + expect(sideForCategory("other")).toBe("tech"); + expect(sideForCategory("personal")).toBe("personal"); + expect(sideLabel("personal")).toBe("Personal"); + expect(sideLabel("tech")).toBe("Tech"); + expect(Object.keys(SIDES)).toEqual(["personal", "tech"]); + }); }); diff --git a/workers/expenses-site/src/__tests__/render.test.mjs b/workers/expenses-site/src/__tests__/render.test.mjs index 9ab5aef1..a51e038a 100644 --- a/workers/expenses-site/src/__tests__/render.test.mjs +++ b/workers/expenses-site/src/__tests__/render.test.mjs @@ -25,16 +25,22 @@ const rows = [ describe("renderDashboard", () => { it("renders a Title Case heading and the 6-month window note", () => { const html = renderDashboard(rows, "2026-10-05T00:00:00Z", 42); - expect(html).toContain("

Business Expenses

"); + expect(html).toContain("

Expenses

"); expect(html).toContain("Last 6 months, rolling."); expect(html).toContain("Showing 2 of 42 recorded expenses."); }); - it("shows All and Tech / AI filters prominently, plus present categories", () => { + it("leads the filter bar with All, Personal, Tech, then subcategories", () => { const html = renderDashboard(rows, "2026-10-05T00:00:00Z", 2); expect(html).toContain('data-filter="all"'); - expect(html).toContain("Tech / AI"); + expect(html).toContain('data-filter="side:personal"'); + expect(html).toContain('data-filter="side:tech"'); + expect(html).toContain(">Personal"); + expect(html).toContain(">Tech"); + expect(html).toContain(">AI"); expect(html).toContain("Cloud / Hosting"); + // Rows carry their side for client-side filtering. + expect(html).toContain('data-side="tech"'); }); it("renders vendor, formatted amount, and category pill per row", () => { @@ -42,7 +48,7 @@ describe("renderDashboard", () => { expect(html).toContain("Anthropic"); expect(html).toContain("$213.20"); expect(html).toContain("Sep 30, 2026"); - expect(html).toContain("Tech / AI"); + expect(html).toContain(">AI"); }); it("escapes hostile vendor text", () => { diff --git a/workers/expenses-site/src/__tests__/sync.test.mjs b/workers/expenses-site/src/__tests__/sync.test.mjs index 7101d20c..54a60396 100644 --- a/workers/expenses-site/src/__tests__/sync.test.mjs +++ b/workers/expenses-site/src/__tests__/sync.test.mjs @@ -1,11 +1,24 @@ import { describe, expect, it, vi } from "vitest"; -import { buildUpsertStatements, mapUpstreamExpense, syncExpenses } from "../sync.mjs"; +import { + buildUpsertStatements, + dedupeUpstreamRows, + expenseFingerprint, + loadSuppressedKeys, + mapUpstreamExpense, + syncExpenses, +} from "../sync.mjs"; -function fakeDb() { +function fakeDb(suppressed = []) { return { prepare: vi.fn((sql) => ({ sql, bind: vi.fn((...args) => ({ sql, args, run: async () => ({}) })), + all: async () => ({ + results: sql.includes("suppressed_expenses") + ? suppressed.map((idempotency_key) => ({ idempotency_key })) + : [], + }), + run: async () => ({}), })), batch: vi.fn(async (stmts) => stmts), }; @@ -53,6 +66,36 @@ describe("buildUpsertStatements", () => { }); }); +describe("dedupeUpstreamRows", () => { + const oneTime = { ...upstream, idempotencyKey: "k-one", kind: "one_time" }; + const usageDup = { ...upstream, idempotencyKey: "k-use", kind: "usage" }; + + it("collapses same-receipt double-posts to one row", () => { + const rows = dedupeUpstreamRows([oneTime, usageDup]); + expect(rows).toHaveLength(1); + expect(rows[0].kind).toBe("one_time"); + }); + + it("keeps genuinely different charges apart", () => { + const other = { ...upstream, idempotencyKey: "k-other", label: "Other invoice" }; + expect(dedupeUpstreamRows([oneTime, other])).toHaveLength(2); + const otherAmount = { ...upstream, idempotencyKey: "k-amt", amountUsd: 99.99 }; + expect(dedupeUpstreamRows([oneTime, otherAmount])).toHaveLength(2); + }); + + it("builds a vendor/amount/day/label fingerprint", () => { + expect(expenseFingerprint(oneTime)).toBe(expenseFingerprint(usageDup)); + }); +}); + +describe("loadSuppressedKeys", () => { + it("reads the suppression table", async () => { + const keys = await loadSuppressedKeys(fakeDb(["k-bad"])); + expect(keys.has("k-bad")).toBe(true); + expect(keys.has("k-good")).toBe(false); + }); +}); + describe("syncExpenses", () => { it("pages the upstream ledger and batches into D1", async () => { const db = fakeDb(); @@ -76,6 +119,8 @@ describe("syncExpenses", () => { }); expect(result.upserted).toBe(1); expect(result.pages).toBe(2); + expect(result.suppressedCount).toBe(0); + expect(result.dedupedCount).toBe(0); expect(db.batch).toHaveBeenCalledTimes(1); // Second page requested with the cursor. expect(String(fetchMock.mock.calls[1][0])).toContain("cursor=cursor-1"); @@ -88,6 +133,39 @@ describe("syncExpenses", () => { } }); + it("skips suppressed keys and collapses double-posts", async () => { + const db = fakeDb(["k-bad"]); + const bad = { ...upstream, idempotencyKey: "k-bad", label: "Bad row" }; + const oneTime = { ...upstream, idempotencyKey: "k-one", kind: "one_time" }; + const usageDupe = { ...upstream, idempotencyKey: "k-dupe", kind: "usage" }; + const solo = { ...upstream, idempotencyKey: "k-solo", kind: "one_time", label: "Solo" }; + const fetchMock = vi.fn().mockResolvedValue({ + ok: true, + json: async () => ({ + expenses: [bad, oneTime, usageDupe, solo], + hasMore: false, + nextCursor: null, + }), + }); + vi.stubGlobal("fetch", fetchMock); + try { + const result = await syncExpenses({ + UPSTREAM_URL: "https://usage.jays.services", + USAGE_READ_TOKEN: "x", + EXPENSES_DB: db, + }); + expect(result.suppressedCount).toBe(1); + expect(result.dedupedCount).toBe(1); + // k-bad suppressed, k-dupe collapsed into k-one: k-one + k-solo upsert. + expect(result.upserted).toBe(2); + const bound = db.batch.mock.calls[0][0]; + expect(bound).toHaveLength(2); + expect(bound.map((s) => s.args[0]).sort()).toEqual(["k-one", "k-solo"]); + } finally { + vi.unstubAllGlobals(); + } + }); + it("throws when the upstream read fails", async () => { vi.stubGlobal( "fetch", diff --git a/workers/expenses-site/src/categories.mjs b/workers/expenses-site/src/categories.mjs index 4a00fc2d..4c8cb636 100644 --- a/workers/expenses-site/src/categories.mjs +++ b/workers/expenses-site/src/categories.mjs @@ -1,6 +1,10 @@ /** * Expense categorization for the expenses dashboard. * + * Two top-level sides: Personal and Tech. Everything business goes under + * Tech, keeping the subcategories below. Personal is empty for now — no + * personal transaction source is wired up yet. + * * Maps a vendor/label pair to a stable category slug. The slug is stored in * D1 at sync time so a future edit UI can override it without a migration. * Keep slugs stable: the dashboard filter bar and the JSON API both expose @@ -11,8 +15,14 @@ * where regex word-boundary anchoring reads as a URL-host pattern). */ +export const SIDES = { + personal: "Personal", + tech: "Tech", +}; + export const CATEGORIES = { - "tech-ai": "Tech / AI", + personal: "Personal", + "tech-ai": "AI", "cloud-hosting": "Cloud / Hosting", "domains": "Domains", "software": "Software", @@ -20,6 +30,16 @@ export const CATEGORIES = { "other": "Other", }; +/** The top-level side a category belongs to. Everything except Personal is Tech. */ +export function sideForCategory(slug) { + return slug === "personal" ? "personal" : "tech"; +} + +/** Human label for a side slug (Title Case for headings). */ +export function sideLabel(slug) { + return SIDES[slug] ?? SIDES.tech; +} + // [slug, regexes] — first match wins. Tested against vendor + label joined. const RULES = [ [ diff --git a/workers/expenses-site/src/index.mjs b/workers/expenses-site/src/index.mjs index d0496fa2..4fec992e 100644 --- a/workers/expenses-site/src/index.mjs +++ b/workers/expenses-site/src/index.mjs @@ -13,27 +13,29 @@ */ import { syncExpenses } from "./sync.mjs"; import { formatAmount, formatDate, renderDashboard } from "./render.mjs"; +import { sideForCategory } from "./categories.mjs"; const SIX_MONTHS_SQL = "date('now', '-6 months')"; +// Side is derived from category (personal -> Personal, everything else -> +// Tech), so the SQL mirrors sideForCategory without a schema change. +const SIDE_SQL = "CASE WHEN category = 'personal' THEN 'personal' ELSE 'tech' END"; -async function readDashboardRows(db, category) { - const where = - category && category !== "all" - ? "WHERE occurred_at >= " + SIX_MONTHS_SQL + " AND category = ?" - : "WHERE occurred_at >= " + SIX_MONTHS_SQL; - const stmt = - category && category !== "all" - ? db.prepare( - `SELECT idempotency_key, vendor, amount_usd, occurred_at, kind, - label, notes, confidence, category, calendar_sort - FROM expenses ${where} ORDER BY occurred_at DESC LIMIT 1000` - ).bind(category) - : db.prepare( - `SELECT idempotency_key, vendor, amount_usd, occurred_at, kind, - label, notes, confidence, category, calendar_sort - FROM expenses ${where} ORDER BY occurred_at DESC LIMIT 1000` - ); - const result = await stmt.all(); +async function readDashboardRows(db, filter) { + let where = "WHERE occurred_at >= " + SIX_MONTHS_SQL; + let params = []; + if (filter === "personal" || filter === "tech") { + where += " AND " + SIDE_SQL + " = ?"; + params = [filter]; + } else if (filter && filter !== "all") { + where += " AND category = ?"; + params = [filter]; + } + const stmt = db.prepare( + `SELECT idempotency_key, vendor, amount_usd, occurred_at, kind, + label, notes, confidence, category, calendar_sort + FROM expenses ${where} ORDER BY occurred_at DESC LIMIT 1000` + ); + const result = params.length > 0 ? await stmt.bind(...params).all() : await stmt.all(); return result.results ?? []; } @@ -111,6 +113,7 @@ export default { kind: e.kind, label: e.label, category: e.category, + side: sideForCategory(e.category), })), asOf, totalCount, diff --git a/workers/expenses-site/src/render.mjs b/workers/expenses-site/src/render.mjs index a26992de..1005af62 100644 --- a/workers/expenses-site/src/render.mjs +++ b/workers/expenses-site/src/render.mjs @@ -6,7 +6,7 @@ * sentences. The table body is rendered server-side from D1 rows; a small * inline script handles the category filter and column sorting client-side. */ -import { CATEGORIES, categoryLabel } from "./categories.mjs"; +import { CATEGORIES, SIDES, categoryLabel, sideForCategory, sideLabel } from "./categories.mjs"; /** Escape text for HTML. */ export function esc(value) { @@ -45,13 +45,15 @@ export function formatAmount(usd) { export function renderDashboard(expenses, asOf, totalCount) { const present = []; for (const slug of Object.keys(CATEGORIES)) { + // "personal" is covered by the side filter, not the subcategory list. + if (slug === "personal") continue; if (slug === "other" || expenses.some((e) => e.category === slug)) { present.push(slug); } } - // "Other" last; "Tech / AI" right after "All". + // Subcategories after the two sides; "Other" last. present.sort((a, b) => { - const rank = (s) => (s === "tech-ai" ? 0 : s === "other" ? 99 : 50); + const rank = (s) => (s === "tech-ai" ? 10 : s === "other" ? 99 : 50); return rank(a) - rank(b); }); @@ -59,11 +61,11 @@ export function renderDashboard(expenses, asOf, totalCount) { const rows = expenses .map( - (e) => ` + (e) => ` ${esc(formatDate(e.occurred_at))} ${esc(e.vendor)} ${esc(e.label || "—")} @@ -76,14 +78,26 @@ export function renderDashboard(expenses, asOf, totalCount) { ) .join("\n"); - const filters = ["all", ...present] - .map((slug) => { - const label = slug === "all" ? "All" : categoryLabel(slug); - const active = slug === "all" ? " aria-current=\"true\"" : ""; - const cls = slug === "all" || slug === "tech-ai" ? "filter primary" : "filter"; - return ` `; + // Filter bar: All, then the two sides, then the subcategories present. + const sideFilters = Object.keys(SIDES).map((slug) => ({ + value: "side:" + slug, + label: sideLabel(slug), + primary: true, + })); + const filters = ["all", ...sideFilters.map((f) => f.value), ...present] + .map((value) => { + const isAll = value === "all"; + const isSide = value.startsWith("side:"); + const label = isAll + ? "All" + : isSide + ? sideLabel(value.slice(5)) + : categoryLabel(value); + const active = isAll ? ' aria-current="true"' : ""; + const primary = isAll || isSide; + return ` `; }) .join("\n"); @@ -92,7 +106,7 @@ export function renderDashboard(expenses, asOf, totalCount) { -Business Expenses +Expenses