Release 1.8.0: offline reading, automatic sync, and account improvements

AGENTS.md

@@ -13,6 +13,9 @@ requires the exact Expo SDK 57 documentation before writing mobile code.
 - Deliver assigned work in coherent chunks, with one PR per chunk.
 - Give every small, meaningful change its own commit. Commit incrementally;
   do not wait until the end and put the entire task into one large commit.
+- Prefer a higher number of focused commits per PR. Separate independently
+  reviewable behavior, UI, and documentation changes instead of bundling them;
+  keep regression tests with the behavior they verify.
 - Keep as many meaningful, atomic commits as the chunk naturally produces in
   its PR. There is no numeric maximum or minimum. Do not split a coherent
   change into broken fragments or make empty commits to inflate the count.

README.md

@@ -34,6 +34,7 @@ link's build current automatically.
 - [Architecture](docs/ARCHITECTURE.md) — how the pieces fit, selection, reminders, security
 - [Roadmap](docs/ROADMAP.md) — shipped phases and what's next
 - [Backend](backend/README.md) — API endpoints, auth, running locally
+- [Email templates](docs/EMAIL_TEMPLATES.md) — branded Supabase emails and installation
 - [Deployment](mobile/DEPLOYMENT.md) — EAS builds, OTA updates, release runbook
 
 ## Contributing

backend/.env.example

@@ -13,6 +13,11 @@ DATABASE_URL=postgresql://postgres.PROJECT:PASSWORD@aws-0-REGION.pooler.supabase
 # Session-mode pooler (port 5432). Used for migrations and any DDL.
 DIRECT_URL=postgresql://postgres.PROJECT:PASSWORD@aws-0-REGION.pooler.supabase.com:5432/postgres
 
+# Best-effort API connection warm-up. Zero interval disables it.
+# The timeout covers pool checkout, reconnect, and the SELECT 1 probe.
+DB_KEEPALIVE_INTERVAL_SECONDS=30
+DB_KEEPALIVE_TIMEOUT_SECONDS=5
+
 # Settings → API
 SUPABASE_URL=https://PROJECT.supabase.co
 # Public anon key (Settings → API). Safe to expose — it's already in the mobile
@@ -34,11 +39,12 @@ GEMINI_MODEL=gemini-3.1-flash-lite
 GENERATION_ENABLED=false
 # The worker tops each topic up to this many published concepts.
 MIN_POOL_PER_TOPIC=25
-# Hard ceiling so a retry loop cannot burn the daily quota.
+# Shared daily reservation ceiling across API prefetch and workers (Pacific day).
+# Use the same cap on all services; zero blocks new generation calls.
 GENERATION_DAILY_CALL_CAP=200
-# Seconds between worker calls; the free tier allows ~10 requests a minute.
+# Seconds between scheduled worker calls; provider limits vary by model/tier.
 GENERATION_PACE_SECONDS=6
-# Last-resort generation inside a request when a user's own pool is empty.
+# Background refill when a user is running low; never waits on the HTTP path.
 GENERATION_ON_DEMAND=true
 
 # --- HTTP -----------------------------------------------------------------

backend/README.md

@@ -29,6 +29,7 @@ backend/
 │   │   └── users.py         profile bootstrap (safety net for the DB trigger)
 │   └── api/v1/              health, topics, daily
 ├── migrations/          # plain SQL, applied in filename order
+├── email-templates/     # account email HTML installed manually in Supabase Auth
 ├── tests/               # 25 tests: token verification, selection, HTTP
 ├── Dockerfile           # what Railway builds
 └── .env.example         # copy to .env — never commit the filled copy
@@ -52,7 +53,9 @@ python3 -m venv .venv && .venv/bin/pip install -r requirements-dev.txt
 | GET | `/v1/topics` | yes | Active topics, concept counts, and whether you follow each. |
 | GET | `/v1/daily` | yes | Today's concept. Creates the assignment on first call, idempotent after. |
 | POST | `/v1/daily/complete` | yes | Mark today learned. Server sets the timestamp and the day it counts for. |
-| GET | `/v1/me/state` | yes | Everything the app renders: follows, history, likes, saves, streaks. One query. |
+| GET | `/v1/me/state?compact=true` | yes | Startup state with at most 50 learned/saved detail rows, full membership and totals, and today's lesson. |
+| GET | `/v1/me/history` | yes | Completed concepts, newest assigned day first; cursor pagination. |
+| GET | `/v1/me/saved` | yes | Saved metadata, newest save first; cursor pagination. |
 | GET | `/v1/me/stats` | yes | Streaks alone, for other consumers. |
 | PUT | `/v1/me/topics` | yes | Replace the followed set (whole-list semantics, so retries are safe). |
 | PATCH | `/v1/me` | yes | Display name and timezone. Unknown zones are rejected. |
@@ -66,6 +69,33 @@ python3 -m venv .venv && .venv/bin/pip install -r requirements-dev.txt
 been assigned every published concept — it never repeats one. Phase 6 hooks
 Gemini generation in at that point.
 
+## Startup and collection pagination
+
+Updated mobile clients send `compact=true` on `GET /v1/me/state`,
+`PUT /v1/me/topics`, and `PATCH /v1/me`. Each response embeds at most 50 learned
+and 50 saved detail records. Full `likes` and `bookmarks` slug arrays, streaks,
+and `stats.total_learned` remain authoritative. `learned_before_window` groups
+older completions by topic name; add those counts to the recent learned rows
+for category totals, including any optimistic offline completion.
+
+Continue from `history_next_cursor` or `saved_next_cursor` using the matching
+collection endpoint. Each returns `{items, next_cursor}`; a null cursor means
+there are no more rows. Both accept `limit` (default 50, range 1–100) and an
+optional `cursor`. History uses the last assigned date; Saved uses an opaque
+save-timestamp/UUID cursor so tied timestamps and deletions do not skip rows.
+Keep cursors unchanged and URL-encode them. All queries use the verified user.
+Pages are live reads: refresh startup state to see new saves/completions made
+above an existing cursor while paging.
+
+The limit applies before per-concept like-count enrichment. Bare membership
+arrays and aggregate calculations still grow with account activity. Older
+clients that omit `compact=true` keep the full legacy detail response during
+backend/OTA rollout; their startup cost is unchanged until updated. The mobile
+History screen still shows the last ten lessons; its full-history UI is separate.
+Saved loads older metadata only when opened, preserving search/category filters,
+and caches it for offline use. Downloaded lesson bodies also supply missing
+metadata offline, even if Saved was never opened before.
+
 ## Authentication
 
 The project signs tokens with **ES256**, so the API verifies them against the
@@ -77,6 +107,11 @@ confusion attacks, both of which are covered by tests.
 `user_id` is taken from the verified token's `sub` claim and from nowhere else.
 No endpoint accepts a user id as a parameter.
 
+Supabase Auth sends signup, recovery, and enabled security notifications using
+the project's configured sender. See [Email templates](../docs/EMAIL_TEMPLATES.md)
+for the three branded HTML files and manual installation steps. An app deployment
+does not publish these templates or change SMTP settings.
+
 ## Content generation
 
 Gemini writes lessons. It does **not** choose subjects.
@@ -86,6 +121,8 @@ curated backlog (150 titles)
         ↓
 worker: is a topic below MIN_POOL_PER_TOPIC published concepts?
         ↓
+claim a title + reserve one shared daily call, then commit
+        ↓
 Gemini writes {summary, example} for one backlogged title
         ↓
 validate — length bounds, no boilerplate opener, no code fence,
@@ -117,17 +154,60 @@ It is safe to run concurrently: backlog items are claimed with
 | Control | Effect |
 |---|---|
 | `GENERATION_ENABLED` | Master switch. Nothing calls Gemini when false. |
-| `GENERATION_DAILY_CALL_CAP` | Hard ceiling per run, so a retry loop cannot burn the quota. |
+| `GENERATION_DAILY_CALL_CAP` | Shared daily reservation ceiling across scheduled refill, on-demand prefetch, and catalog rewriting; zero stops new calls. |
 | `attempts < 3` | A title that keeps failing is retired rather than blocking the queue. |
 | Validation | Malformed output leaves the item pending; it never reaches a reader. |
-| `GENERATION_ON_DEMAND` | Last-resort in-request generation when one user's pool is dry. |
+| `GENERATION_ON_DEMAND` | Allows background refill when a user has few unread lessons. |
+
+### Shared generation budget
+
+`generation_daily_usage` stores committed call reservations, one row per Pacific
+calendar day. PostgreSQL computes the day in `America/Los_Angeles`, matching
+[Gemini's midnight Pacific RPD reset](https://ai.google.dev/gemini-api/docs/rate-limits),
+including daylight saving time. User assignment/streak timezones are unchanged.
+The atomic UPSERT prevents competing API/worker processes from spending the same
+last slot. Restarts and repeated job runs retain usage; a new day gets a new row.
+
+`generate_one` claims the backlog title and reserves quota in one transaction,
+then commits before calling Gemini. An exhausted budget rolls back the title
+claim and attempt. Once committed, failed responses, rate limits, cancellation,
+or a worker crash keep the reservation; uncertain provider calls must not be
+refunded. Gemini throttling still refunds the separate **backlog retry attempt**.
+Empty backlog does not consume quota. Database/reservation failures stop generation
+before the provider call. The manual rewrite worker uses the same ledger and
+respects `GENERATION_ENABLED`; unfinished lessons retain their old prompt version
+for a later run. Daily reading remains independent of generation.
+
+Set the **same `GENERATION_DAILY_CALL_CAP` on the API and every worker** sharing
+this database. Changing it does not erase existing usage. This is an application
+budget, not a provider quota lookup: other applications using the same Gemini
+project are outside this ledger, and provider rate/token limits still apply.
+
+**Before deploying this code:** apply
+[`0010_generation_daily_usage.sql`](migrations/0010_generation_daily_usage.sql)
+using the migration procedure in [RELEASING.md](../RELEASING.md). The production
+ledger remains unchanged until actual application is verified. Earlier application
+versions do not use this counter, and calls made before deployment cannot be
+reconstructed from it. Pause old generators during rollout; enable the new code
+at the next Pacific reset, or conservatively seed today's usage while generation
+is paused, to avoid granting another allowance in the middle of the day.
+
+Inspect reservations without changing them:
+
+```sql
+select budget_day, calls_used
+from public.generation_daily_usage
+order by budget_day desc
+limit 7;
+```
 
 ### Fallback ladder in `/v1/daily`
 
 1. An unseen concept in a followed topic.
-2. Failing that, generate one in the user's least-recently-seen followed topic.
-3. Failing that, widen to the whole catalog and flag `outside_followed_topics`.
-4. Failing that, return `409 catalog_exhausted`. A concept is never repeated.
+2. If that pool is dry, schedule a background refill and immediately widen to
+   the whole catalog, flagging `outside_followed_topics`.
+3. If nothing unseen remains, return `409 catalog_exhausted`. A concept is never
+   repeated. A low unread watermark can also schedule refill before exhaustion.
 
 ## Latency and database region
 
@@ -136,16 +216,48 @@ Measured against a Supabase project in `ap-northeast-1` from Europe, a single
 round trip is 160–1100 ms — so the code is written to minimise the *number* of
 statements rather than their complexity:
 
-- `/v1/me/state` is **one query**. It returns follows, history, likes, saves,
-  today's assignment, and streaks together, and bootstrapping only runs when
-  that query finds no profile.
+- The state aggregate is **one query**, returning follows, recent detail rows,
+  membership, assignment slug, and full streaks/totals. Bootstrapping only runs
+  when no profile exists. The `/v1/me/state` handler separately resolves today's
+  lesson, folding it into the same HTTP response.
 - Follow updates are one statement (a data-modifying CTE), not one per topic.
 - Connection pooling is on. Without it every request paid a fresh TCP + TLS +
   auth handshake to the database region, which cost seconds.
 
-The remaining latency is geography. **Deploy the API in the same region as the
-database** — on Railway, pick the region closest to your Supabase project — and
-these round trips drop to single-digit milliseconds.
+Geography and connection reuse both affect latency. **Deploy the API close to
+the database**, then compare fresh, immediately reused, and post-idle requests.
+Timing a slow query alone does not establish that a new connection was opened.
+
+The API runs a best-effort `SELECT 1` probe immediately on startup and then
+every `DB_KEEPALIVE_INTERVAL_SECONDS` (default 30 seconds, measured after each
+probe finishes). It borrows from the same pool as requests and promptly returns
+the connection, rolling back the implicit transaction. The pool reuses its most
+recently returned connection, so low traffic can use a warm slot while surplus
+idle slots expire. `pool_pre_ping` remains enabled for dead connections.
+
+`DB_KEEPALIVE_TIMEOUT_SECONDS` (default 5 seconds) bounds checkout, reconnect,
+and query together. Rollback/return has a separate budget of the same duration;
+failed cleanup invalidates the connection. A failed attempt logs only the
+exception type and retries after the interval; it does not block API startup.
+Shutdown cancels the task before disposing the engine. Set the interval to `0`
+to disable probes. Each API
+process runs its own task; importing the engine in cron workers starts no task.
+Pool size and overflow limits remain 5 each. A cold startup or a burst requiring
+additional connections can still pay connection setup time.
+
+Reproduce idle expiry without production services using:
+
+```bash
+DATABASE_URL=postgresql+asyncpg://postgres:postgres@127.0.0.1:55433/postgres \
+SUPABASE_URL=http://test.invalid SUPABASE_JWKS_URL=http://test.invalid/jwks \
+GENERATION_ENABLED=false GEMINI_API_KEY= \
+  .venv/bin/python -m pytest tests/test_db_keepalive_postgres.py -q -s
+```
+
+The PostgreSQL 16 tests set short idle expiry only on their own sessions and
+compare connection counts and request timings with warming disabled/enabled.
+They also check transaction cleanup and recovery from a terminated connection.
+These timings describe the local test environment, not deployed Supavisor latency.
 
 ## Connection strings
 

backend/app/api/v1/me.py

@@ -1,14 +1,18 @@
-from datetime import time
+from datetime import date, time
 
-from fastapi import APIRouter, Depends, HTTPException, status
+from fastapi import APIRouter, Depends, HTTPException, Query, status
 from sqlalchemy import ARRAY, Time, bindparam, text
 from sqlalchemy.ext.asyncio import AsyncSession
 
 from app.db.session import get_db
 from app.deps import CurrentUser, get_current_user
 from app.schemas.daily import ConceptOut, DailyOut
-from app.schemas.me import LearnedOut, ProfileIn, SavedConceptOut, StateOut, StreakOut, TopicsIn
+from app.schemas.me import (
+    HistoryPageOut, LearnedOut, ProfileIn, SavedConceptOut, SavedPageOut, StateOut,
+    StreakOut, TopicsIn,
+)
 from app.schemas.notifications import NotificationPrefs, PushTokenIn
+from app.services.collections import history_page, saved_page
 from app.services.interactions import set_followed_topics
 from app.services.selection import DailyResult, get_or_create_daily
 from app.services.state import load_state
@@ -59,12 +63,16 @@ def _to_state_out(state) -> StateOut:
             for s in state.saved
         ],
         stats=StreakOut(**vars(state.stats)),
+        learned_before_window=state.learned_before_window,
+        history_next_cursor=state.history_next_cursor,
+        saved_next_cursor=state.saved_next_cursor,
         assignment_slug=state.assignment_slug,
     )
 
 
 @router.get("/state", response_model=StateOut)
 async def get_state(
+    compact: bool = False,
     user: CurrentUser = Depends(get_current_user),
     db: AsyncSession = Depends(get_db),
 ) -> StateOut:
@@ -73,18 +81,41 @@ async def get_state(
     Bootstrapping only runs when the state query finds no profile, so the
     common path costs a single round trip.
     """
-    state = await load_state(db, user.id)
+    state = await load_state(db, user.id, compact=compact)
     if state is None:
         await ensure_bootstrapped(db, user.id, user.email)
         await db.commit()
-        state = await load_state(db, user.id)
+        state = await load_state(db, user.id, compact=compact)
     out = _to_state_out(state)
     # Fold today's concept in so the app needs one startup round trip (#102).
     # Same create-on-first-call behaviour as GET /v1/daily.
     out.daily = _daily_out_or_none(await get_or_create_daily(db, user.id))
     return out
 
 
+@router.get("/history", response_model=HistoryPageOut)
+async def get_history(
+    cursor: date | None = None,
+    limit: int = Query(default=50, ge=1, le=100),
+    user: CurrentUser = Depends(get_current_user),
+    db: AsyncSession = Depends(get_db),
+) -> HistoryPageOut:
+    return await history_page(db, user.id, cursor, limit)
+
+
+@router.get("/saved", response_model=SavedPageOut)
+async def get_saved(
+    cursor: str | None = Query(default=None, max_length=200),
+    limit: int = Query(default=50, ge=1, le=100),
+    user: CurrentUser = Depends(get_current_user),
+    db: AsyncSession = Depends(get_db),
+) -> SavedPageOut:
+    try:
+        return await saved_page(db, user.id, cursor, limit)
+    except ValueError as exc:
+        raise HTTPException(status.HTTP_400_BAD_REQUEST, detail="Invalid saved cursor") from exc
+
+
 @router.get("/stats", response_model=StreakOut)
 async def get_stats(
     user: CurrentUser = Depends(get_current_user),
@@ -96,11 +127,12 @@ async def get_stats(
 @router.put("/topics", response_model=StateOut)
 async def put_topics(
     body: TopicsIn,
+    compact: bool = False,
     user: CurrentUser = Depends(get_current_user),
     db: AsyncSession = Depends(get_db),
 ) -> StateOut:
     await set_followed_topics(db, user.id, body.topics)
-    return _to_state_out(await load_state(db, user.id))
+    return _to_state_out(await load_state(db, user.id, compact=compact))
 
 
 @router.post("/push-token", status_code=status.HTTP_204_NO_CONTENT)
@@ -217,6 +249,7 @@ async def put_notifications(
 @router.patch("", response_model=StateOut)
 async def patch_profile(
     body: ProfileIn,
+    compact: bool = False,
     user: CurrentUser = Depends(get_current_user),
     db: AsyncSession = Depends(get_db),
 ) -> StateOut:
@@ -244,4 +277,4 @@ async def patch_profile(
             {"n": body.display_name, "uid": user.id},
         )
     await db.commit()
-    return _to_state_out(await load_state(db, user.id))
+    return _to_state_out(await load_state(db, user.id, compact=compact))

backend/app/config.py

@@ -1,5 +1,6 @@
 from functools import lru_cache
 
+from pydantic import Field
 from pydantic_settings import BaseSettings, SettingsConfigDict
 
 
@@ -17,6 +18,10 @@ class Settings(BaseSettings):
     database_url: str
     direct_url: str | None = None
 
+    # Best-effort API pool warm-up; zero disables it. Workers do not start it.
+    db_keepalive_interval_seconds: float = Field(default=30, ge=0, allow_inf_nan=False)
+    db_keepalive_timeout_seconds: float = Field(default=5, gt=0, allow_inf_nan=False)
+
     supabase_url: str
     supabase_jwks_url: str
     # Present for legacy HS256 projects; this project signs with ES256 via JWKS.
@@ -32,10 +37,11 @@ class Settings(BaseSettings):
     gemini_model: str = "gemini-3.1-flash-lite"
     generation_enabled: bool = False
     min_pool_per_topic: int = 25
-    generation_daily_call_cap: int = 200
+    # Shared by all generation paths; zero prevents new reservations.
+    generation_daily_call_cap: int = Field(default=200, ge=0)
     # Seconds between worker calls; the free tier allows ~10 requests a minute.
     generation_pace_seconds: float = 6.0
-    # Last-resort generation inside a request, when a user's pool is empty.
+    # Schedule background refill when a user's unread pool is low.
     generation_on_demand: bool = True
 
     allowed_origins: str = "http://localhost:8081"