Listen on every Ask AI answer; one shared module for the AI dialogs (v0.26.0)

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@profullstack/readm3",
-  "version": "0.25.2",
+  "version": "0.26.0",
   "description": "Markdown editing and sharing for people and teams. Web/PWA, CLI, API and MCP with version history, view/edit links, and a terminal reader.",
   "type": "module",
   "license": "MIT",

server/billing.ts

@@ -295,13 +295,7 @@ export class Billing {
     if (!exempt && !free && this.balance(user.id) < estimate) {
       throw new HttpError(402, `Today's ${this.freeQuestionsPerDay} free questions are used. Not enough balance for another (about $${(estimate / 1e6).toFixed(4)}). Top up to continue, or come back tomorrow.`);
     }
-    if (this.running.has(user.id)) throw new HttpError(429, "An answer is already being written for this account.");
-    this.running.add(user.id);
-    let result: Awaited<ReturnType<Summarize>>;
-    try { result = await this.summarize(ASK_SYSTEM, text); }
-    catch { throw new HttpError(502, "The answer could not be written. You were not charged; try again."); }
-    finally { this.running.delete(user.id); }
-    if (result.refused || !result.text) throw new HttpError(422, "The model declined to answer this. You were not charged.");
+    const result = await this.call(user, ASK_SYSTEM, text, "answer");
     const model = result.model ?? this.model;
     const cost = costMicros(model, result.usage);
     const charged = free || exempt ? 0 : priceMicros(cost, arm);
@@ -310,7 +304,32 @@ export class Billing {
       id(), user.id, "charge", -charged, `${free ? "free-question" : "question"}:${id()}`, free || exempt ? null : arm,
       JSON.stringify({ kind: "question", files: files.length, model, costMicros: cost, input: result.usage.input_tokens, output: result.usage.output_tokens }), now(),
     );
-    return { answer: result.text, model, chargedMicros: charged, free, exempt, freeQuestions: this.freeQuestions(user.id), balanceMicros: this.balance(user.id) };
+    // Kept like a summary, so Listen reads it aloud through the same route (summaries/<id>/speech).
+    const summaryId = this.keep(user.id, createHash("sha256").update(`question\0${id()}`).digest("hex"), "question", files.length === 1 ? files[0]!.path : `${files.length} files`, result.text, model);
+    return { answer: result.text, summaryId, listen: !!this.speak, model, chargedMicros: charged, free, exempt, freeQuestions: this.freeQuestions(user.id), balanceMicros: this.balance(user.id) };
+  }
+
+  /** One model call at a time per account (two cannot spend one balance); a failure or a refusal is never charged. */
+  private async call(user: User, system: string, text: string, what: "summary" | "answer", attachment?: Attachment, refusal = "this") {
+    if (this.running.has(user.id)) throw new HttpError(429, `A${what === "answer" ? "n" : ""} ${what} is already being written for this account.`);
+    this.running.add(user.id);
+    let result: Awaited<ReturnType<Summarize>>;
+    try { result = await this.summarize!(system, text, attachment); }
+    catch { throw new HttpError(502, `The ${what} could not be written. You were not charged; try again.`); }
+    finally { this.running.delete(user.id); }
+    // A declined request is not the customer's to pay for.
+    if (result.refused || !result.text) throw new HttpError(422, `The model declined to ${what === "answer" ? "answer" : "summarize"} ${refusal}. You were not charged.`);
+    return result;
+  }
+
+  /** Keeps a summary or an answer for the account; its id is what Listen reads aloud. */
+  private keep(userId: string, keyHash: string, kind: string, path: string, text: string, model: string): string {
+    const summaryId = id();
+    this.store.run(
+      "INSERT OR REPLACE INTO summary_cache (id,userId,keyHash,kind,path,summary,model,createdAt) VALUES (?,?,?,?,?,?,?,?)",
+      summaryId, userId, keyHash, kind, path, text, model, now(),
+    );
+    return summaryId;
   }
 
   /** This week's free document summaries: how many are left and when the count starts again. */
@@ -454,14 +473,7 @@ export class Billing {
       const used = freeAllowed ? `This week's ${this.freePerWeek} free summaries are used. ` : "";
       throw new HttpError(402, `${used}Not enough balance for a summary (about $${(estimate / 1e6).toFixed(4)}). Top up to continue.`);
     }
-    if (this.running.has(user.id)) throw new HttpError(429, "A summary is already being written for this account.");
-    this.running.add(user.id);
-    let result: Awaited<ReturnType<Summarize>>;
-    try { result = await this.summarize!(system, text, attachment); }
-    catch { throw new HttpError(502, "The summary could not be written. You were not charged; try again."); }
-    finally { this.running.delete(user.id); }
-    // A declined request is not the customer's to pay for.
-    if (result.refused || !result.text) throw new HttpError(422, `The model declined to summarize this ${kind}. You were not charged.`);
+    const result = await this.call(user, system, text, "summary", attachment, `this ${kind}`);
     const model = result.model ?? this.model;
     const cost = costMicros(model, result.usage);
     const charged = free || exempt ? 0 : priceMicros(cost, arm);
@@ -471,11 +483,7 @@ export class Billing {
       id(), user.id, "charge", -charged, `${free ? "free-summary" : "summary"}:${id()}`, free ? null : arm,
       JSON.stringify({ kind, path, model, costMicros: cost, input: result.usage.input_tokens, output: result.usage.output_tokens }), now(),
     );
-    const summaryId = id();
-    this.store.run(
-      "INSERT OR REPLACE INTO summary_cache (id,userId,keyHash,kind,path,summary,model,createdAt) VALUES (?,?,?,?,?,?,?,?)",
-      summaryId, user.id, keyHash, kind, path, result.text, model, now(),
-    );
+    const summaryId = this.keep(user.id, keyHash, kind, path, result.text, model);
     return { summary: result.text, summaryId, cached: false, listen: !!this.speak, chargedMicros: charged, costMicros: cost, arm, model, free, freeSummaries: this.free(user.id), balanceMicros: this.balance(user.id) };
   }
 

server/test/billing.test.ts

@@ -197,6 +197,19 @@ describe("kept summaries and Listen", () => {
     expect(calls).toBe(3);
   });
 
+  test("an Ask AI answer is kept like a summary, so Listen reads it aloud", async () => {
+    let spoken = "";
+    const f = await listening(async (text) => { spoken = text; return new Uint8Array([7]); });
+    const asked = await (await f.post("ask", { question: "When?", files: [{ path: "plan.md", text: "Ships Friday." }] })).json() as any;
+    expect(asked).toMatchObject({ listen: true, free: true });
+    expect(asked.summaryId).toBeString();
+    f.billing.credit(f.user.id, 1_000_000, "test:l", "test");
+    const heard = await f.post(`summaries/${asked.summaryId}/speech`);
+    expect(heard.status).toBe(200);
+    expect([...new Uint8Array(await heard.arrayBuffer())]).toEqual([7]);
+    expect(spoken).toContain("Ships Friday");
+  });
+
   test("Listen reads a kept summary aloud once, charged at the arm's price, then replays free", async () => {
     const spoken: string[] = [];
     const f = await listening(async (text) => { spoken.push(text); return new Uint8Array([0xff, 0xfb, 1, 2]); });

site/ai-dialog.ts

@@ -0,0 +1,90 @@
+import { ApiError } from "./cloud.ts";
+import { dollars, el } from "./history-panel.ts";
+
+/**
+ * What the AI dialogs share (Summarize in site/summary-panel.ts, Ask AI in
+ * site/ask-panel.ts): the dialog shell, the sign-in line, the Top up button, and Listen.
+ *
+ * Listen reads a kept summary or answer aloud when the server has a voice (ElevenLabs):
+ * the first listen is paid from the balance (admins never pay), then the audio is kept
+ * on the account and replays free; this session also keeps it, so a replay makes no
+ * request at all.
+ */
+const heard = new Map<string, string>();
+
+/** A kept summary or answer read aloud: an MP3 from the server, or the reason it could not be had. */
+async function speech(summaryId: string): Promise<{ url: string; chargedMicros: number }> {
+  const known = heard.get(summaryId);
+  if (known) return { url: known, chargedMicros: 0 };
+  const response = await fetch(`/api/v1/summaries/${encodeURIComponent(summaryId)}/speech`, { method: "POST", credentials: "same-origin", cache: "no-store" });
+  if (!response.ok) {
+    const data = await response.json().catch(() => ({})) as { error?: string };
+    throw new ApiError(data.error || `Listening failed (${response.status})`, response.status);
+  }
+  const url = URL.createObjectURL(await response.blob());
+  heard.set(summaryId, url);
+  return { url, chargedMicros: Number(response.headers.get("x-charged-micros") ?? 0) };
+}
+
+/** A modal with a title, a close button and a body; it removes itself when closed. */
+export function aiDialog(className: string, id: string, title: string, closeLabel: string) {
+  const box = document.createElement("dialog");
+  box.className = `history-dialog summary-dialog ${className}`;
+  box.setAttribute("aria-labelledby", id);
+  box.innerHTML = `<div class="dialog-header"><h2 id="${id}"></h2><button type="button"></button></div>`;
+  box.querySelector("h2")!.textContent = title;
+  const close = box.querySelector<HTMLButtonElement>(".dialog-header button")!;
+  close.textContent = "×";
+  close.setAttribute("aria-label", closeLabel);
+  close.onclick = () => box.close();
+  box.addEventListener("click", (event) => { if (event.target === box) box.close(); });
+  box.addEventListener("close", () => box.remove());
+  document.body.append(box);
+  box.showModal();
+  return box;
+}
+
+/** "Sign in (free) to …", returning here afterwards. */
+export function signInLine(what: string, next = location.pathname + location.search): HTMLElement {
+  const link = el("a", undefined, "Sign in (free)");
+  link.href = `/account?next=${encodeURIComponent(next)}`;
+  const line = el("p");
+  line.append(link, ` ${what}`);
+  return line;
+}
+
+/** Top up: closes the dialog and opens Add funds. */
+export function topUpButton(box: HTMLDialogElement, topUp: () => void): HTMLButtonElement {
+  const top = el("button", "ghost", "Top up");
+  top.type = "button";
+  top.onclick = () => { box.close(); topUp(); };
+  return top;
+}
+
+/** The Listen button for a kept summary or answer; it becomes an audio player once played. */
+export function listenControl(summaryId: string, label: string, box: HTMLDialogElement, topUp: () => void): HTMLElement {
+  const player = el("div", "summary-listen");
+  const listen = el("button", "ghost", heard.has(summaryId) ? "Play" : "Listen");
+  listen.type = "button";
+  listen.setAttribute("aria-label", label);
+  const status = el("small");
+  listen.onclick = async () => {
+    listen.disabled = true;
+    status.textContent = heard.has(summaryId) ? "" : "Preparing audio…";
+    try {
+      const audio = await speech(summaryId);
+      const element = document.createElement("audio");
+      element.controls = true;
+      element.autoplay = true;
+      element.src = audio.url;
+      status.textContent = audio.chargedMicros ? `Listen · ${dollars(audio.chargedMicros)} · replays are free` : "";
+      player.replaceChildren(element, status);
+    } catch (error) {
+      listen.disabled = false;
+      status.textContent = error instanceof Error ? error.message : "Listening failed.";
+      if (error instanceof ApiError && error.status === 402) player.append(" ", topUpButton(box, topUp));
+    }
+  };
+  player.append(listen, " ", status);
+  return player;
+}

site/ask-panel.ts

@@ -1,5 +1,6 @@
 import { ApiError, request } from "./cloud.ts";
 import { dollars, el } from "./history-panel.ts";
+import { aiDialog, listenControl, signInLine, topUpButton } from "./ai-dialog.ts";
 import { binaryType } from "../src/code.ts";
 import type { Document } from "./workspace.ts";
 
@@ -8,10 +9,11 @@ import type { Document } from "./workspace.ts";
  * one, a folder it is in, or the whole workspace. Only text goes (a PDF, an image or a
  * file stored apart is left out and counted), up to 400 KB of it per question. Signed in,
  * the first 10 questions each day are free (server/billing.ts FREE_QUESTIONS_PER_DAY),
- * then each is paid from the prepaid balance; admins are never charged.
+ * then each is paid from the prepaid balance; admins are never charged. Every answer can
+ * be read aloud with Listen, like a summary (site/ai-dialog.ts).
  */
 interface FreeQuestions { perDay: number; left: number; resetsAt: string; }
-interface Answered { answer: string; chargedMicros: number; free: boolean; exempt?: boolean; freeQuestions: FreeQuestions; balanceMicros: number; }
+interface Answered { answer: string; summaryId?: string; listen?: boolean; chargedMicros: number; free: boolean; exempt?: boolean; freeQuestions: FreeQuestions; balanceMicros: number; }
 type Turn = { question: string; answer: string };
 
 const MAX_TEXT = 400 * 1024;
@@ -43,25 +45,17 @@ const freeLine = (free: FreeQuestions) => free.left > 0
   : `Today's ${free.perDay} free questions are used; more at midnight UTC`;
 
 export async function showAsk(documents: () => Document[], active: string, topUp: () => void): Promise<void> {
-  const box = document.createElement("dialog");
-  box.className = "history-dialog summary-dialog ask-dialog";
-  box.setAttribute("aria-labelledby", "ask-title");
-  box.innerHTML = `<div class="dialog-header"><h2 id="ask-title">Ask AI about your files</h2><button type="button" aria-label="Close Ask AI">×</button></div>
-<label class="ask-scope">Ask about <select id="ask-scope"></select></label>
+  const box = aiDialog("ask-dialog", "ask-title", "Ask AI about your files", "Close Ask AI");
+  box.insertAdjacentHTML("beforeend", `<label class="ask-scope">Ask about <select id="ask-scope"></select></label>
 <div class="summary-body ask-log" aria-live="polite"></div>
 <form class="ask-form"><textarea id="ask-question" rows="3" placeholder="What does this say about…? Where is… set? Summarize the decisions in this folder." aria-label="Your question"></textarea><button type="submit" class="primary">Ask</button></form>
-<p class="history-account"></p>`;
+<p class="history-account"></p>`);
   const log = box.querySelector<HTMLElement>(".ask-log")!;
   const account = box.querySelector<HTMLElement>(".history-account")!;
   const select = box.querySelector<HTMLSelectElement>("#ask-scope")!;
   const form = box.querySelector<HTMLFormElement>(".ask-form")!;
   const input = box.querySelector<HTMLTextAreaElement>("#ask-question")!;
   const submit = form.querySelector<HTMLButtonElement>("button")!;
-  box.querySelector<HTMLButtonElement>(".dialog-header button")!.onclick = () => box.close();
-  box.addEventListener("click", (event) => { if (event.target === box) box.close(); });
-  box.addEventListener("close", () => box.remove());
-  document.body.append(box);
-  box.showModal();
 
   const scopes: { label: string; scope: { file?: string; folder?: string } }[] = [
     { label: `This file · ${active.split("/").at(-1)}`, scope: { file: active } },
@@ -70,13 +64,21 @@ export async function showAsk(documents: () => Document[], active: string, topUp
   ];
   for (const [i, { label }] of scopes.entries()) select.append(new Option(label, String(i)));
   // Each scope keeps its own conversation; changing scope starts a new one.
-  const turns = new Map<number, Turn[]>();
+  const turns = new Map<number, (Turn & { summaryId?: string })[]>();
+  let listen = false;
+  /** One answer on screen, with Listen when the server has a voice. */
+  const answerBlock = (answer: string, summaryId?: string) => {
+    const block = el("div", "ask-answer");
+    block.append(el("p", "summary-text", answer));
+    if (listen && summaryId) block.append(listenControl(summaryId, "Listen to this answer", box, topUp));
+    return block;
+  };
   const say = (message: string) => log.replaceChildren(el("p", "history-empty", message));
   const describe = () => {
     const { files, skipped } = gather(documents(), scopes[Number(select.value)]!.scope);
     const past = turns.get(Number(select.value)) ?? [];
     log.replaceChildren();
-    for (const turn of past) log.append(el("p", "ask-question", turn.question), el("p", "summary-text", turn.answer));
+    for (const turn of past) log.append(el("p", "ask-question", turn.question), answerBlock(turn.answer, turn.summaryId));
     if (!past.length) say(files.length
       ? `${files.length.toLocaleString()} file${files.length === 1 ? "" : "s"} will be read${skipped ? ` (${skipped.toLocaleString()} left out: not text, too large, or past 400 KB)` : ""}. Ask anything about ${files.length === 1 ? "it" : "them"}.`
       : "There is no text to ask about here. Pick a folder or the whole workspace.");
@@ -88,11 +90,7 @@ export async function showAsk(documents: () => Document[], active: string, topUp
   let billing: { enabled?: boolean; freeQuestions?: FreeQuestions; exempt?: boolean; balanceMicros: number } | null = null;
   try { billing = await request("billing"); } catch { billing = null; }
   if (!billing) {
-    const link = el("a", undefined, "Sign in (free)");
-    link.href = `/account?next=${encodeURIComponent("/talk")}`;
-    const line = el("p");
-    line.append(link, " to ask AI about your files: 10 questions a day free, then pay as you go.");
-    log.replaceChildren(line);
+    log.replaceChildren(signInLine("to ask AI about your files: 10 questions a day free, then pay as you go.", "/talk"));
     form.hidden = true;
     return;
   }
@@ -115,20 +113,16 @@ export async function showAsk(documents: () => Document[], active: string, topUp
     input.value = "";
     submit.disabled = true;
     try {
-      const result = await request<Answered>("ask", "POST", { question, files, skipped, history: past });
-      pending.replaceWith(el("p", "summary-text", result.answer));
-      turns.set(index, [...past, { question, answer: result.answer }]);
+      const result = await request<Answered>("ask", "POST", { question, files, skipped, history: past.map(({ question, answer }) => ({ question, answer })) });
+      listen = !!result.listen;
+      pending.replaceWith(answerBlock(result.answer, result.summaryId));
+      turns.set(index, [...past, { question, answer: result.answer, summaryId: result.summaryId }]);
       const cost = result.exempt ? "Admin, no charge" : result.free ? "Free" : dollars(result.chargedMicros);
       account.textContent = result.exempt ? cost : `${cost} · ${freeLine(result.freeQuestions)} · balance ${dollars(result.balanceMicros)}`;
     } catch (error) {
       pending.textContent = error instanceof Error ? error.message : "The question failed.";
       input.value = question;
-      if (error instanceof ApiError && error.status === 402) {
-        const top = el("button", "ghost", "Top up");
-        top.type = "button";
-        top.onclick = () => { box.close(); topUp(); };
-        account.replaceChildren(top);
-      }
+      if (error instanceof ApiError && error.status === 402) account.replaceChildren(topUpButton(box, topUp));
     } finally {
       submit.disabled = false;
       log.scrollTop = log.scrollHeight;