From 0e82f03cb88497e8964876ff0e2fe8c27091c71d Mon Sep 17 00:00:00 2001 From: Muawiya Amir Date: Sat, 12 Sep 2026 16:03:06 +0500 Subject: [PATCH 01/59] docs: record confirmed offline gaps for issue 133 --- docs/WORK_LOG.md | 39 +++++++++++++++++++++++++-------------- 1 file changed, 25 insertions(+), 14 deletions(-) diff --git a/docs/WORK_LOG.md b/docs/WORK_LOG.md index 2594d07..cd2ad42 100644 --- a/docs/WORK_LOG.md +++ b/docs/WORK_LOG.md @@ -7,20 +7,31 @@ claims as completed work. ## Current status -- Assigned scope: add the explicitly requested one-time What's New card for - `1.7.1` to the open release PR. -- Branch: `codex/1-7-1-whats-new`, based on `develop` at `0596f6d`. -- Version preparation [#178](https://github.com/Coding-Moves/one-concept/pull/178) - is merged. Release [#179](https://github.com/Coding-Moves/one-concept/pull/179) - is open; its migration check and version-preparation preview OTA passed. -- Card and standing-policy follow-up: - [#180](https://github.com/Coding-Moves/one-concept/pull/180), targeting `develop` - so it is included in release #179. Its PR records the final merge/check status. -- [PR #177](https://github.com/Coding-Moves/one-concept/pull/177) is merged; its - [preview OTA](https://github.com/Coding-Moves/one-concept/actions/runs/34677521802) - passed. Its completed browser checks also covered retry after reconnecting, - dark/reduced-motion mode, and friendly offline sign-in errors. -- Production merge/publication remains the owner's next step after release review. +- Assigned scope: only [#133](https://github.com/Coding-Moves/one-concept/issues/133). + Confirm the problems first, then open one PR into `develop` for offline reading, + personalization persistence, and automatic action synchronization. +- Branch: `codex/133-offline-reading-sync`, based on refreshed `origin/develop` + at `89a8fb8`. The starting tree matches the tested baseline. +- Release #179 and card follow-up #180 are merged; #181 synchronized `main` + back into `develop`. This task does not authorize another production release. + +## Offline reading and synchronization (#133) — 2026-09-12 + +- Before editing, exported the unchanged web app and exercised it in Chromium + with dummy authentication and intercepted API requests. No live account used. +- Confirmed: a custom saved concept opened online loses its full text after an + offline restart; a warmed topic catalog is unavailable after offline restart; + offline topic changes do not enter the persistent queue; restoring connectivity + alone leaves an offline like queued without sending a request. +- Passing controls: saved detail online, cached Today offline, and durable offline + like queuing. Preserve those existing behaviors and the offline banner. +- Planned atomic commits: persistent full-concept reading with account cleanup + and tests; cached topic catalog and queued follows with tests; automatic sync + triggers with tests; validation, codebase map, and PR handoff. +- Clarification pending: keep the current APK with sync while open/reopened, or + add native OS background scheduling (new APK, OS-controlled execution timing). +- Read the exact Expo SDK 57 documentation before mobile edits. Other issues, + including #182's password-visibility request, remain outside this PR. ## Working agreement — 2026-09-11 From 00852857c43b89dac09747f8f382758a57235308 Mon Sep 17 00:00:00 2001 From: Muawiya Amir Date: Sat, 12 Sep 2026 16:06:09 +0500 Subject: [PATCH 02/59] feat: persist full lessons for offline saved reading --- mobile/src/screens/ConceptDetailScreen.tsx | 7 +- mobile/src/services/accountCaches.ts | 2 + mobile/src/services/conceptApi.ts | 49 ++++++++++++- mobile/src/services/offlineCache.ts | 55 +++++++++++++++ .../src/services/remoteProgressRepository.ts | 19 +++++ mobile/tests/offlineCache.test.mjs | 69 +++++++++++++++++++ 6 files changed, 198 insertions(+), 3 deletions(-) create mode 100644 mobile/src/services/offlineCache.ts create mode 100644 mobile/tests/offlineCache.test.mjs diff --git a/mobile/src/screens/ConceptDetailScreen.tsx b/mobile/src/screens/ConceptDetailScreen.tsx index 1704044..352e461 100644 --- a/mobile/src/screens/ConceptDetailScreen.tsx +++ b/mobile/src/screens/ConceptDetailScreen.tsx @@ -31,7 +31,12 @@ export function ConceptDetailScreen() { useEffect(() => { let active = true; setStatus('loading'); - fetchConcept(conceptId) + fetchConcept(conceptId, (cached) => { + if (active) { + setConcept(cached); + setStatus('ready'); + } + }) .then((c) => { if (active) { setConcept(c); diff --git a/mobile/src/services/accountCaches.ts b/mobile/src/services/accountCaches.ts index 04dc6a4..c0a1534 100644 --- a/mobile/src/services/accountCaches.ts +++ b/mobile/src/services/accountCaches.ts @@ -8,6 +8,7 @@ */ import { clearDailyCache } from './dailyApi'; +import { conceptCache } from './conceptApi'; import { clearNotificationPrefsCache } from './notifications'; import { clearServerStateCache } from './remoteProgressRepository'; @@ -15,6 +16,7 @@ export async function clearAccountCaches(): Promise { await Promise.all([ clearServerStateCache(), clearDailyCache(), + conceptCache.clear(), clearNotificationPrefsCache(), ]); } diff --git a/mobile/src/services/conceptApi.ts b/mobile/src/services/conceptApi.ts index fdae75a..76186c3 100644 --- a/mobile/src/services/conceptApi.ts +++ b/mobile/src/services/conceptApi.ts @@ -1,5 +1,9 @@ -import { apiRequest } from '../api/client'; +import AsyncStorage from '@react-native-async-storage/async-storage'; +import { apiRequest, getConnectivity } from '../api/client'; import { Category, Concept } from '../types'; +import { OfflineCache } from './offlineCache'; + +export const conceptCache = new OfflineCache(AsyncStorage, 'one-concept/concepts/v1/'); /** Server shape from GET /v1/concepts/{slug} (matches the daily ConceptOut). */ interface ConceptResponse { @@ -18,7 +22,7 @@ interface ConceptResponse { * concept's local id (see dailyApi.toConcept), so History/Saved conceptIds pass * straight through here. */ -export async function fetchConcept(slug: string): Promise { +async function downloadConcept(slug: string): Promise { const c = await apiRequest(`/v1/concepts/${encodeURIComponent(slug)}`); return { id: c.slug, @@ -29,3 +33,44 @@ export async function fetchConcept(slug: string): Promise { likeCount: c.like_count ?? 0, }; } + +/** Paint cached text immediately, then refresh counts/content when reachable. */ +export async function fetchConcept( + slug: string, + onCached?: (concept: Concept) => void, +): Promise { + const epoch = conceptCache.epoch; + const cached = await conceptCache.get(slug, epoch); + if (cached) { + onCached?.(cached); + if (!getConnectivity()) return cached; + } + try { + const concept = await downloadConcept(slug); + await conceptCache.set(slug, concept, epoch).catch(() => {}); + return concept; + } catch (error) { + if (cached) return cached; + throw error; + } +} + +/** Download every missing saved lesson with bounded concurrency. Individual + * entries avoid one large AsyncStorage row; already-downloaded lessons stay. */ +export async function cacheSavedConcepts(slugs: string[], epoch = conceptCache.epoch): Promise { + const remaining = [...new Set(slugs)]; + let cursor = 0; + const worker = async () => { + while (cursor < remaining.length && epoch === conceptCache.epoch && getConnectivity()) { + const slug = remaining[cursor++]; + if (await conceptCache.get(slug, epoch)) continue; + if (epoch !== conceptCache.epoch) return; + try { + await conceptCache.set(slug, await downloadConcept(slug), epoch); + } catch { + // Missing downloads retry on the next successful state load/reconnect. + } + } + }; + await Promise.all(Array.from({ length: Math.min(3, remaining.length) }, worker)); +} diff --git a/mobile/src/services/offlineCache.ts b/mobile/src/services/offlineCache.ts new file mode 100644 index 0000000..3269950 --- /dev/null +++ b/mobile/src/services/offlineCache.ts @@ -0,0 +1,55 @@ +export interface CacheStorage { + getItem(key: string): Promise; + setItem(key: string, value: string): Promise; + getAllKeys(): Promise; + multiRemove(keys: string[]): Promise; +} + +/** Per-entry disk storage. Serial writes and a generation fence make sign-out + * win over pending reads, writes, and downloads from the previous account. */ +export class OfflineCache { + private storage: CacheStorage; + private prefix: string; + private writes: Promise = Promise.resolve(); + private generation = 0; + + constructor(storage: CacheStorage, prefix: string) { + this.storage = storage; + this.prefix = prefix; + } + + get epoch(): number { return this.generation; } + + async get(id: string, epoch = this.epoch): Promise { + await this.writes; + if (epoch !== this.epoch) return null; + try { + const raw = await this.storage.getItem(this.prefix + id); + return raw && epoch === this.epoch ? JSON.parse(raw) as T : null; + } catch { + return null; + } + } + + set(id: string, value: T, epoch = this.epoch): Promise { + const raw = JSON.stringify(value); + return this.write(async () => { + if (epoch === this.epoch) await this.storage.setItem(this.prefix + id, raw); + }); + } + + clear(): Promise { + this.generation += 1; + return this.write(async () => { + const keys = (await this.storage.getAllKeys()).filter(key => key.startsWith(this.prefix)); + if (keys.length) await this.storage.multiRemove(keys); + }); + } + + private write(operation: () => Promise): Promise { + const next = this.writes.then(operation); + // A disk failure must not prevent later writes or account cleanup. + this.writes = next.catch(() => {}); + return next; + } +} diff --git a/mobile/src/services/remoteProgressRepository.ts b/mobile/src/services/remoteProgressRepository.ts index f5340ae..f1bea56 100644 --- a/mobile/src/services/remoteProgressRepository.ts +++ b/mobile/src/services/remoteProgressRepository.ts @@ -2,6 +2,8 @@ import AsyncStorage from '@react-native-async-storage/async-storage'; import { ApiError, apiRequest } from '../api/client'; import { Category, DailyPayload, ProgressState } from '../types'; import { todayKey } from './dates'; +import { cacheSavedConcepts, conceptCache } from './conceptApi'; +import { toConcept } from './dailyApi'; import { clearQueue, dequeue, enqueue, keyOf, pending, QueuedMutation } from './mutationQueue'; import { ProgressRepository } from './progressRepository'; import { EMPTY_PROGRESS } from './storage'; @@ -94,6 +96,16 @@ export class RemoteProgressRepository implements ProgressRepository { } private async fromState(payload: StatePayload, epoch: number): Promise { + if (epoch !== this.epoch) return EMPTY_PROGRESS; + // Keep each day's full text for later History/Saved reading, and download + // saved bodies without holding up the initial screen. + const contentEpoch = conceptCache.epoch; + if (payload.daily) { + const concept = toConcept(payload.daily); + await conceptCache.set(concept.id, concept, contentEpoch).catch(() => {}); + } + if (epoch !== this.epoch) return EMPTY_PROGRESS; + void cacheSavedConcepts(payload.bookmarks, contentEpoch); return this.remember(toProgressState(payload), epoch); } @@ -107,10 +119,17 @@ export class RemoteProgressRepository implements ProgressRepository { async loadCached(): Promise { const epoch = this.epoch; + const contentEpoch = conceptCache.epoch; const raw = await AsyncStorage.getItem(CACHE_KEY).catch(() => null); if (!raw || epoch !== this.epoch) return null; try { this.cache = JSON.parse(raw) as ProgressState; + // Also upgrade an existing installation's cached Today while offline. + if (this.cache.serverDaily?.status === 'ok') { + const concept = toConcept(this.cache.serverDaily.payload); + await conceptCache.set(concept.id, concept, contentEpoch).catch(() => {}); + } + if (epoch !== this.epoch) return null; return this.cache; } catch { return null; diff --git a/mobile/tests/offlineCache.test.mjs b/mobile/tests/offlineCache.test.mjs new file mode 100644 index 0000000..1447745 --- /dev/null +++ b/mobile/tests/offlineCache.test.mjs @@ -0,0 +1,69 @@ +import assert from 'node:assert/strict'; +import { test } from 'node:test'; +import { OfflineCache } from '../src/services/offlineCache.ts'; + +function disk() { + const rows = new Map(); + return { rows, getItem: async k => rows.get(k) ?? null, + setItem: async (k, v) => { rows.set(k, v); }, + getAllKeys: async () => [...rows.keys()], + multiRemove: async keys => { keys.forEach(k => rows.delete(k)); } }; +} +const tick = () => new Promise(resolve => setImmediate(resolve)); +function deferred() { + let resolve; + return { promise: new Promise(r => { resolve = r; }), resolve: (...args) => resolve(...args) }; +} + +test('full lesson text survives a new cache instance and corrupt entries do not block other lessons', async () => { + const storage = disk(); + const cache = new OfflineCache(storage, 'lessons/'); + const lesson = { id: 'saved', summary: 'Full explanation', example: 'Concrete example' }; + await cache.set(lesson.id, lesson); + storage.rows.set('lessons/corrupt', '{'); + const reopened = new OfflineCache(storage, 'lessons/'); + assert.deepEqual(await reopened.get('saved'), lesson); + assert.equal(await reopened.get('corrupt'), null); +}); + +test('sign-out removes pending disk writes and rejects late downloads without deleting device preferences', async () => { + const storage = disk(); + const started = deferred(), finish = deferred(); + storage.setItem = async (k, v) => { started.resolve(); await finish.promise; storage.rows.set(k, v); }; + storage.rows.set('theme', 'dark'); + const cache = new OfflineCache(storage, 'lessons/'); + const oldAccount = cache.epoch; + const write = cache.set('saved', { summary:'old account' }, oldAccount); + await started.promise; + const clear = cache.clear(); + const lateDownload = cache.set('late', { summary:'late old account' }, oldAccount); + finish.resolve(); + await Promise.all([write, clear, lateDownload]); + assert.deepEqual([...storage.rows], [['theme', 'dark']]); + await cache.set('new', { summary:'new account' }); + assert.deepEqual(await cache.get('new'), { summary:'new account' }); +}); + +test('sign-out suppresses a disk read already in flight', async () => { + const storage = disk(), response = deferred(); + storage.getItem = () => response.promise; + const cache = new OfflineCache(storage, 'lessons/'); + const read = cache.get('old'); + await tick(); + await cache.clear(); + response.resolve(JSON.stringify({summary:'old account'})); + assert.equal(await read, null); +}); + +test('failed storage writes do not poison subsequent downloads or cleanup', async () => { + const storage = disk(); + const put = storage.setItem; + storage.setItem = async () => { throw new Error('disk full'); }; + const cache = new OfflineCache(storage, 'lessons/'); + await assert.rejects(cache.set('one', {summary:'one'})); + storage.setItem = put; + await cache.set('two', {summary:'two'}); + assert.deepEqual(await cache.get('two'), {summary:'two'}); + await cache.clear(); + assert.equal(await cache.get('two'), null); +}); From 41da1abb7bc8e5cea9cf8271f5a7415b9e753b99 Mon Sep 17 00:00:00 2001 From: Muawiya Amir Date: Sat, 12 Sep 2026 16:10:01 +0500 Subject: [PATCH 03/59] feat: cache topic catalog and queue offline follow changes --- mobile/src/context/ProgressContext.tsx | 9 ++- mobile/src/hooks/useTopics.ts | 31 ++------- mobile/src/services/accountCaches.ts | 2 + mobile/src/services/mutationQueue.ts | 7 +++ mobile/src/services/topicStore.ts | 87 ++++++++++++++++++++++++++ mobile/src/services/topicsApi.ts | 43 +++++++------ mobile/tests/topicStore.test.mjs | 77 +++++++++++++++++++++++ 7 files changed, 210 insertions(+), 46 deletions(-) create mode 100644 mobile/src/services/topicStore.ts create mode 100644 mobile/tests/topicStore.test.mjs diff --git a/mobile/src/context/ProgressContext.tsx b/mobile/src/context/ProgressContext.tsx index 8b806e9..74b783c 100644 --- a/mobile/src/context/ProgressContext.tsx +++ b/mobile/src/context/ProgressContext.tsx @@ -9,7 +9,7 @@ import { useState, } from 'react'; import { AppState } from 'react-native'; -import { subscribeConnectivity } from '../api/client'; +import { getConnectivity, subscribeConnectivity } from '../api/client'; import { CONCEPTS } from '../data/concepts'; import { Category, Concept, DailyOutcome, ProgressState } from '../types'; import { selectDailyConcept } from '../services/dailyConcept'; @@ -17,6 +17,8 @@ import { todayKey } from '../services/dates'; import { localProgressRepository } from '../services/localProgressRepository'; import { ProgressRepository } from '../services/progressRepository'; import { remoteProgressRepository } from '../services/remoteProgressRepository'; +import { subscribeQueue } from '../services/mutationQueue'; +import { fetchTopics } from '../services/topicsApi'; import { useAuth } from './AuthContext'; import { EMPTY_PROGRESS } from '../services/storage'; import { computeStreaks, StreakStats } from '../services/streak'; @@ -122,6 +124,7 @@ export function ProgressProvider({ children, repository: override }: Props) { try { const next = await repository.flushQueue?.(); if (active && next) setProgress(next); + if (active && next && getConnectivity()) await fetchTopics().catch(() => {}); } catch { // A flush failure just leaves items queued for the next trigger. } finally { @@ -132,6 +135,9 @@ export function ProgressProvider({ children, repository: override }: Props) { const unsubscribe = subscribeConnectivity((online) => { if (online) flush(); }); + const unsubscribeQueue = subscribeQueue(() => { + if (getConnectivity()) flush(); + }); const appState = AppState.addEventListener('change', (s) => { if (s === 'active') flush(); }); @@ -140,6 +146,7 @@ export function ProgressProvider({ children, repository: override }: Props) { return () => { active = false; unsubscribe(); + unsubscribeQueue(); appState.remove(); }; }, [repository]); diff --git a/mobile/src/hooks/useTopics.ts b/mobile/src/hooks/useTopics.ts index cc8d28f..3d5986c 100644 --- a/mobile/src/hooks/useTopics.ts +++ b/mobile/src/hooks/useTopics.ts @@ -1,6 +1,6 @@ -import { useCallback, useEffect, useState } from 'react'; +import { useCallback, useEffect, useState, useSyncExternalStore } from 'react'; import { useAuth } from '../context/AuthContext'; -import { fetchTopics, ServerTopic, setFollowedTopics } from '../services/topicsApi'; +import { fetchTopics, ServerTopic, topicStore } from '../services/topicsApi'; export interface Topics { loading: boolean; @@ -17,7 +17,7 @@ export function useTopics(): Topics { // Key on the user id, not the session object: supabase hands a fresh object // on every token refresh, which would otherwise refetch the list hourly. const userId = session?.user?.id ?? null; - const [topics, setTopics] = useState([]); + const topics = useSyncExternalStore(topicStore.subscribe, topicStore.getSnapshot); const [loading, setLoading] = useState(true); const [error, setError] = useState(false); const [attempt, setAttempt] = useState(0); @@ -25,7 +25,6 @@ export function useTopics(): Topics { useEffect(() => { if (!userId) { - setTopics([]); setLoading(false); return; } @@ -33,9 +32,6 @@ export function useTopics(): Topics { setLoading(true); setError(false); fetchTopics() - .then((list) => { - if (!cancelled) setTopics(list); - }) .catch(() => { if (!cancelled) setError(true); }) @@ -49,25 +45,10 @@ export function useTopics(): Topics { const toggle = useCallback( (slug: string) => { - // Compute the next list synchronously from current state — never read a - // value assigned inside a setState updater, which React may not have run - // yet (that would PUT an empty list and unfollow everything). - const next = topics.map((t) => - t.slug === slug ? { ...t, following: !t.following } : t - ); - setTopics(next); - - const followed = next.filter((t) => t.following).map((t) => t.slug); - // Fire-and-forget; on failure reload the server's truth so the pill can - // never lie about what was actually saved. - setFollowedTopics(followed).catch(() => { - fetchTopics() - .then(setTopics) - .catch(() => {}); - }); + topicStore.toggle(slug).catch(() => setError(true)); }, - [topics] + [] ); - return { loading, error, retry, topics, toggle }; + return { loading: loading && topics.length === 0, error, retry, topics, toggle }; } diff --git a/mobile/src/services/accountCaches.ts b/mobile/src/services/accountCaches.ts index c0a1534..96f7279 100644 --- a/mobile/src/services/accountCaches.ts +++ b/mobile/src/services/accountCaches.ts @@ -11,12 +11,14 @@ import { clearDailyCache } from './dailyApi'; import { conceptCache } from './conceptApi'; import { clearNotificationPrefsCache } from './notifications'; import { clearServerStateCache } from './remoteProgressRepository'; +import { clearTopicsCache } from './topicsApi'; export async function clearAccountCaches(): Promise { await Promise.all([ clearServerStateCache(), clearDailyCache(), conceptCache.clear(), + clearTopicsCache(), clearNotificationPrefsCache(), ]); } diff --git a/mobile/src/services/mutationQueue.ts b/mobile/src/services/mutationQueue.ts index d2a1c8d..cceaf1e 100644 --- a/mobile/src/services/mutationQueue.ts +++ b/mobile/src/services/mutationQueue.ts @@ -11,6 +11,12 @@ import AsyncStorage from '@react-native-async-storage/async-storage'; * only the final set. */ const QUEUE_KEY = 'one-concept/mutation-queue/v1'; +const listeners = new Set<() => void>(); + +export function subscribeQueue(listener: () => void): () => void { + listeners.add(listener); + return () => { listeners.delete(listener); }; +} export type QueuedMutation = | { kind: 'like'; slug: string; desired: boolean } @@ -57,6 +63,7 @@ export async function enqueue(m: QueuedMutation): Promise { const q = await ensureLoaded(); q[keyOf(m)] = m; await persist(); + listeners.forEach(listener => listener()); } /** diff --git a/mobile/src/services/topicStore.ts b/mobile/src/services/topicStore.ts new file mode 100644 index 0000000..e8557e5 --- /dev/null +++ b/mobile/src/services/topicStore.ts @@ -0,0 +1,87 @@ +export interface ServerTopic { + slug: string; + name: string; + conceptCount: number; + following: boolean; +} + +interface Dependencies { + read: () => Promise; + write: (topics: ServerTopic[]) => Promise; + fetch: () => Promise; + pending: () => Promise; + enqueue: (slugs: string[]) => Promise; +} + +/** Shared catalog and optimistic follow state for Personalization and Stats. */ +export class TopicStore { + private deps: Dependencies; + private topics: ServerTopic[] = []; + private revision = 0; + private epoch = 0; + private writes: Promise = Promise.resolve(); + private listeners = new Set<() => void>(); + + constructor(deps: Dependencies) { this.deps = deps; } + + getSnapshot = (): ServerTopic[] => this.topics; + subscribe = (listener: () => void): (() => void) => { + this.listeners.add(listener); + return () => { this.listeners.delete(listener); }; + }; + + private publish(topics: ServerTopic[]): void { + this.topics = topics; + this.listeners.forEach(listener => listener()); + } + + async load(): Promise { + const revision = ++this.revision; + const cached = await this.deps.read(); + const overlay = (rows: ServerTopic[], slugs?: string[]) => slugs + ? rows.map(topic => ({ ...topic, following: slugs.includes(topic.slug) })) + : rows; + const queued = await this.deps.pending(); + if (revision !== this.revision) return this.topics; + if (cached && this.topics.length === 0) this.publish(overlay(cached, queued)); + try { + const rows = await this.deps.fetch(); + const pending = await this.deps.pending(); + if (revision !== this.revision) return this.topics; + this.publish(overlay(rows, pending)); + await this.deps.write(this.topics); + return this.topics; + } catch (error) { + if (revision !== this.revision || this.topics.length) return this.topics; + throw error; + } + } + + toggle(slug: string): Promise { + if (!this.topics.some(topic => topic.slug === slug)) return Promise.resolve(); + const before = this.topics; + const next = before.map(topic => topic.slug === slug + ? { ...topic, following: !topic.following } : topic); + const revision = ++this.revision; + const epoch = this.epoch; + this.publish(next); + const write = this.writes.then(async () => { + if (epoch !== this.epoch) return; + // All follows use the durable queue, even online. A single replay path + // preserves order when connectivity changes during rapid taps. + await this.deps.enqueue(next.filter(topic => topic.following).map(topic => topic.slug)); + if (epoch === this.epoch) await this.deps.write(next); + }); + this.writes = write.catch(() => {}); + return write.catch(error => { + if (revision === this.revision) this.publish(before); + throw error; + }); + } + + reset(): void { + this.epoch += 1; + this.revision += 1; + this.publish([]); + } +} diff --git a/mobile/src/services/topicsApi.ts b/mobile/src/services/topicsApi.ts index 07c5f36..796d8b1 100644 --- a/mobile/src/services/topicsApi.ts +++ b/mobile/src/services/topicsApi.ts @@ -5,14 +5,14 @@ * slug space so a topic the app has never heard of is never dropped. */ +import AsyncStorage from '@react-native-async-storage/async-storage'; import { apiRequest } from '../api/client'; +import { enqueue, pending } from './mutationQueue'; +import { OfflineCache } from './offlineCache'; +import { TopicStore, ServerTopic } from './topicStore'; -export interface ServerTopic { - slug: string; - name: string; - conceptCount: number; - following: boolean; -} +export type { ServerTopic } from './topicStore'; +const cache = new OfflineCache(AsyncStorage, 'one-concept/topics/v1/'); interface TopicPayload { slug: string; @@ -21,19 +21,22 @@ interface TopicPayload { following: boolean; } -export async function fetchTopics(): Promise { - const rows = await apiRequest('/v1/topics'); - return rows.map((r) => ({ - slug: r.slug, - name: r.name, - conceptCount: r.concept_count, - following: r.following, - })); -} +export const topicStore = new TopicStore({ + read: () => cache.get('catalog'), + write: topics => cache.set('catalog', topics), + fetch: async () => { + const rows = await apiRequest('/v1/topics'); + return rows.map(r => ({ + slug: r.slug, name: r.name, conceptCount: r.concept_count, following: r.following, + })); + }, + pending: async () => (await pending()).find(m => m.kind === 'topics')?.slugs, + enqueue: slugs => enqueue({ kind: 'topics', slugs }), +}); + +export const fetchTopics = () => topicStore.load(); -/** Replace the followed set. Whole-list semantics: the caller sends every - * slug it wants followed, so nothing it omits by accident survives — which - * is exactly why the caller must pass the full server list, not a subset. */ -export async function setFollowedTopics(slugs: string[]): Promise { - await apiRequest('/v1/me/topics', { method: 'PUT', body: { topics: slugs } }); +export async function clearTopicsCache(): Promise { + topicStore.reset(); + await cache.clear(); } diff --git a/mobile/tests/topicStore.test.mjs b/mobile/tests/topicStore.test.mjs new file mode 100644 index 0000000..4ae543a --- /dev/null +++ b/mobile/tests/topicStore.test.mjs @@ -0,0 +1,77 @@ +import assert from 'node:assert/strict'; +import { test } from 'node:test'; +import { TopicStore } from '../src/services/topicStore.ts'; + +const topics = [ + {slug:'computer-science',name:'Computer Science',conceptCount:25,following:true}, + {slug:'new-server-topic',name:'New server topic',conceptCount:12,following:false}, +]; +function setup() { + let cached = null, queued; + const calls = []; + const deps = { read: async () => cached, write: async rows => { cached=rows; }, + fetch: async () => topics, pending: async () => queued, + enqueue: async slugs => { queued=slugs; calls.push(slugs); } }; + return { deps, calls, store: new TopicStore(deps) }; +} +function deferred() { + let resolve; + return { promise: new Promise(r => { resolve=r; }), resolve: (...args) => resolve(...args) }; +} +const tick = () => new Promise(resolve => setImmediate(resolve)); + +test('offline dynamic topic follows survive restart and override stale server state until synced', async () => { + const {store, deps, calls} = setup(); + await store.load(); + deps.fetch = async () => { throw new Error('offline'); }; + await store.toggle('new-server-topic'); + assert.deepEqual(calls, [['computer-science','new-server-topic']]); + const reopened = new TopicStore(deps); + await reopened.load(); + assert.equal(reopened.getSnapshot()[1].following, true); + deps.fetch = async () => topics; + await reopened.load(); + assert.equal(reopened.getSnapshot()[1].following, true); +}); + +test('rapid toggles preserve the complete followed set and stale catalog loads cannot undo them', async () => { + const {store, deps, calls} = setup(); + await store.load(); + const response = deferred(); + deps.fetch = () => response.promise; + const loading = store.load(); + await tick(); + await Promise.all([store.toggle('new-server-topic'),store.toggle('computer-science')]); + response.resolve(topics); + await loading; + assert.deepEqual(calls.at(-1), ['new-server-topic']); + assert.deepEqual(store.getSnapshot().map(t=>t.following), [false,true]); +}); + +test('cached catalog paints before a stalled fetch completes', async () => { + const {store, deps} = setup(); + await store.load(); + const response = deferred(); + deps.fetch = () => response.promise; + const reopened = new TopicStore(deps); + const loading = reopened.load(); + await tick(); + assert.deepEqual(reopened.getSnapshot(), topics); + response.resolve(topics); + await loading; +}); + +test('account reset suppresses late topic fetches and queued follow writes', async () => { + const {store, deps, calls} = setup(); + await store.load(); + const response = deferred(); + deps.fetch = () => response.promise; + const loading = store.load(); + await tick(); + const write = store.toggle('new-server-topic'); + store.reset(); + response.resolve(topics); + await Promise.all([loading,write]); + assert.deepEqual(store.getSnapshot(), []); + assert.deepEqual(calls, []); +}); From f8d1ca65901125a240442b6adb21a1019ed769c7 Mon Sep 17 00:00:00 2001 From: Muawiya Amir Date: Sat, 12 Sep 2026 16:14:35 +0500 Subject: [PATCH 04/59] fix: serialize durable offline action persistence --- mobile/src/services/mutationOutbox.ts | 90 +++++++++++++++++++++++ mobile/src/services/mutationQueue.ts | 101 +++----------------------- mobile/tests/mutationOutbox.test.mjs | 45 ++++++++++++ 3 files changed, 146 insertions(+), 90 deletions(-) create mode 100644 mobile/src/services/mutationOutbox.ts create mode 100644 mobile/tests/mutationOutbox.test.mjs diff --git a/mobile/src/services/mutationOutbox.ts b/mobile/src/services/mutationOutbox.ts new file mode 100644 index 0000000..4e952b9 --- /dev/null +++ b/mobile/src/services/mutationOutbox.ts @@ -0,0 +1,90 @@ +interface Storage { + getItem(key: string): Promise; + setItem(key: string, value: string): Promise; + removeItem(key: string): Promise; +} + +export type QueuedMutation = + | { kind: 'like'; slug: string; desired: boolean } + | { kind: 'save'; slug: string; desired: boolean } + | { kind: 'topics'; slugs: string[] } + // The date it was completed: /v1/daily/complete only completes "today", so a + // 'learn' queued on a previous day must be dropped, not replayed (#133). + | { kind: 'learn'; date: string }; + +/** Stable coalescing key — one pending intent per (kind, target). */ +export function keyOf(m: QueuedMutation): string { + switch (m.kind) { + case 'like': + return `like:${m.slug}`; + case 'save': + return `save:${m.slug}`; + case 'topics': + return 'topics'; + case 'learn': + return 'learn'; + } +} + +/** Serialize disk operations so replay acknowledgements cannot race a new + * intent or resurrect data after sign-out. Keeps the existing disk format. */ +export class MutationOutbox { + private storage: Storage; + private key: string; + private epoch = 0; + private tail: Promise = Promise.resolve(); + private listeners = new Set<() => void>(); + + constructor(storage: Storage, key: string) { this.storage = storage; this.key = key; } + + subscribe = (listener: () => void): (() => void) => { + this.listeners.add(listener); + return () => { this.listeners.delete(listener); }; + }; + + private async read(): Promise> { + const raw = await this.storage.getItem(this.key); + try { return raw ? JSON.parse(raw) : {}; } catch { return {}; } + } + + private serial(operation: () => Promise): Promise { + const next = this.tail.then(operation); + this.tail = next.catch(() => {}); + return next; + } + + enqueue(mutation: QueuedMutation): Promise { + const epoch = this.epoch; + return this.serial(async () => { + const map = await this.read(); + if (epoch !== this.epoch) return; + map[keyOf(mutation)] = mutation; + await this.storage.setItem(this.key, JSON.stringify(map)); + if (epoch === this.epoch) this.listeners.forEach(listener => listener()); + }); + } + + dequeue(key: string, expected?: QueuedMutation): Promise { + const epoch = this.epoch; + return this.serial(async () => { + const map = await this.read(); + if (epoch !== this.epoch || !(key in map)) return; + if (expected && JSON.stringify(map[key]) !== JSON.stringify(expected)) return; + delete map[key]; + await this.storage.setItem(this.key, JSON.stringify(map)); + }); + } + + pending(): Promise { + const epoch = this.epoch; + return this.serial(async () => { + const map = await this.read(); + return epoch === this.epoch ? Object.values(map) : []; + }); + } + + clear(): Promise { + this.epoch += 1; + return this.serial(() => this.storage.removeItem(this.key)); + } +} diff --git a/mobile/src/services/mutationQueue.ts b/mobile/src/services/mutationQueue.ts index cceaf1e..ac7004b 100644 --- a/mobile/src/services/mutationQueue.ts +++ b/mobile/src/services/mutationQueue.ts @@ -1,91 +1,12 @@ import AsyncStorage from '@react-native-async-storage/async-storage'; - -/** - * Durable outbox for mutations made while offline (issue #133). - * - * Entries are COALESCED by a stable key, keeping only the latest intent — the - * server operations are all idempotent or whole-list (like/save via PUT/DELETE, - * topics via a whole-list PUT, daily-complete is idempotent), so replaying the - * final desired state is correct and order across different keys doesn't matter. - * A like→unlike→like offline collapses to a single "like"; two topic edits keep - * only the final set. - */ -const QUEUE_KEY = 'one-concept/mutation-queue/v1'; -const listeners = new Set<() => void>(); - -export function subscribeQueue(listener: () => void): () => void { - listeners.add(listener); - return () => { listeners.delete(listener); }; -} - -export type QueuedMutation = - | { kind: 'like'; slug: string; desired: boolean } - | { kind: 'save'; slug: string; desired: boolean } - | { kind: 'topics'; slugs: string[] } - // The date it was completed: /v1/daily/complete only completes "today", so a - // 'learn' queued on a previous day must be dropped, not replayed (#133). - | { kind: 'learn'; date: string }; - -/** Stable coalescing key — one pending intent per (kind, target). */ -export function keyOf(m: QueuedMutation): string { - switch (m.kind) { - case 'like': - return `like:${m.slug}`; - case 'save': - return `save:${m.slug}`; - case 'topics': - return 'topics'; - case 'learn': - return 'learn'; - } -} - -// In-memory mirror of the persisted map, lazily loaded once. -let map: Record | null = null; - -async function ensureLoaded(): Promise> { - if (map) return map; - try { - const raw = await AsyncStorage.getItem(QUEUE_KEY); - map = raw ? (JSON.parse(raw) as Record) : {}; - } catch { - map = {}; - } - return map; -} - -async function persist(): Promise { - if (map) await AsyncStorage.setItem(QUEUE_KEY, JSON.stringify(map)).catch(() => {}); -} - -/** Add or replace the intent for its key (latest wins). */ -export async function enqueue(m: QueuedMutation): Promise { - const q = await ensureLoaded(); - q[keyOf(m)] = m; - await persist(); - listeners.forEach(listener => listener()); -} - -/** - * Remove the intent at a key. If `expected` is given, only remove it when the - * stored intent still equals it — so a flush that replayed an old intent can't - * clobber a newer one enqueued for the same key mid-flush (#133). - */ -export async function dequeue(key: string, expected?: QueuedMutation): Promise { - const q = await ensureLoaded(); - if (!(key in q)) return; - if (expected && JSON.stringify(q[key]) !== JSON.stringify(expected)) return; - delete q[key]; - await persist(); -} - -/** All pending intents (order across keys is not significant). */ -export async function pending(): Promise { - return Object.values(await ensureLoaded()); -} - -/** Drop everything — used on sign-out so one account's queue can't leak. */ -export async function clearQueue(): Promise { - map = {}; - await AsyncStorage.removeItem(QUEUE_KEY).catch(() => {}); -} +import { MutationOutbox, QueuedMutation } from './mutationOutbox'; +export { keyOf } from './mutationOutbox'; +export type { QueuedMutation } from './mutationOutbox'; + +// Preserve existing queues when an installed app receives the update. +const queue = new MutationOutbox(AsyncStorage, 'one-concept/mutation-queue/v1'); +export const subscribeQueue = queue.subscribe; +export const enqueue = (mutation: QueuedMutation) => queue.enqueue(mutation); +export const dequeue = (key: string, expected?: QueuedMutation) => queue.dequeue(key, expected); +export const pending = () => queue.pending(); +export const clearQueue = () => queue.clear(); diff --git a/mobile/tests/mutationOutbox.test.mjs b/mobile/tests/mutationOutbox.test.mjs new file mode 100644 index 0000000..6ade89f --- /dev/null +++ b/mobile/tests/mutationOutbox.test.mjs @@ -0,0 +1,45 @@ +import assert from 'node:assert/strict'; +import { test } from 'node:test'; +import { MutationOutbox } from '../src/services/mutationOutbox.ts'; + +function disk() { + let raw=null; + return {getItem:async()=>raw,setItem:async(_k,v)=>{raw=v;},removeItem:async()=>{raw=null;}}; +} +const like={kind:'like',slug:'saved',desired:true}; +const topics={kind:'topics',slugs:['new-server-topic']}; +test('concurrent offline actions survive restart and keep only the latest intent for each key', async () => { + const storage=disk(), queue=new MutationOutbox(storage,'queue'); + await Promise.all([queue.enqueue(like),queue.enqueue(topics),queue.enqueue({...like,desired:false})]); + assert.deepEqual(await new MutationOutbox(storage,'queue').pending(),[{...like,desired:false},topics]); +}); +test('an old acknowledgement cannot remove a newer same-key offline action', async () => { + const queue=new MutationOutbox(disk(),'queue'); + await queue.enqueue(like); + const [sent]=await queue.pending(); + await Promise.all([queue.enqueue({...like,desired:false}),queue.dequeue('like:saved',sent)]); + assert.deepEqual(await queue.pending(),[{...like,desired:false}]); +}); +test('sign-out clears a disk write already in flight before another account can read the queue', async () => { + const storage=disk(), put=storage.setItem; + let release, started; + const writing=new Promise(r=>{started=r;}); + const finish=new Promise(r=>{release=r;}); + storage.setItem=async(k,v)=>{started();await finish;await put(k,v);}; + const queue=new MutationOutbox(storage,'queue'); + const write=queue.enqueue(like); + await writing; + const clear=queue.clear(); + release(); + await Promise.all([write,clear]); + assert.deepEqual(await new MutationOutbox(storage,'queue').pending(),[]); +}); +test('storage failures are reported and do not prevent a later retry', async () => { + const storage=disk(), put=storage.setItem; + storage.setItem=async()=>{throw new Error('disk full');}; + const queue=new MutationOutbox(storage,'queue'); + await assert.rejects(queue.enqueue(like),/disk full/); + storage.setItem=put; + await queue.enqueue(like); + assert.deepEqual(await queue.pending(),[like]); +}); From 1a82c2e28d00c245c088aeaa2d4b6a2d8d3144e5 Mon Sep 17 00:00:00 2001 From: Muawiya Amir Date: Sat, 12 Sep 2026 16:15:57 +0500 Subject: [PATCH 05/59] feat: retry offline synchronization while the app is active --- mobile/src/context/ProgressContext.tsx | 133 +++++++++++++++---------- mobile/src/services/syncLoop.ts | 54 ++++++++++ mobile/tests/syncLoop.test.mjs | 39 ++++++++ 3 files changed, 173 insertions(+), 53 deletions(-) create mode 100644 mobile/src/services/syncLoop.ts create mode 100644 mobile/tests/syncLoop.test.mjs diff --git a/mobile/src/context/ProgressContext.tsx b/mobile/src/context/ProgressContext.tsx index 74b783c..51de35a 100644 --- a/mobile/src/context/ProgressContext.tsx +++ b/mobile/src/context/ProgressContext.tsx @@ -8,8 +8,8 @@ import { useRef, useState, } from 'react'; -import { AppState } from 'react-native'; -import { getConnectivity, subscribeConnectivity } from '../api/client'; +import { AppState, Platform } from 'react-native'; +import { getConnectivity, isApiConfigured, setConnectivity, subscribeConnectivity } from '../api/client'; import { CONCEPTS } from '../data/concepts'; import { Category, Concept, DailyOutcome, ProgressState } from '../types'; import { selectDailyConcept } from '../services/dailyConcept'; @@ -17,7 +17,8 @@ import { todayKey } from '../services/dates'; import { localProgressRepository } from '../services/localProgressRepository'; import { ProgressRepository } from '../services/progressRepository'; import { remoteProgressRepository } from '../services/remoteProgressRepository'; -import { subscribeQueue } from '../services/mutationQueue'; +import { pending as queuedMutations, subscribeQueue } from '../services/mutationQueue'; +import { createSyncLoop } from '../services/syncLoop'; import { fetchTopics } from '../services/topicsApi'; import { useAuth } from './AuthContext'; import { EMPTY_PROGRESS } from '../services/storage'; @@ -75,15 +76,24 @@ export function ProgressProvider({ children, repository: override }: Props) { override ?? (session ? remoteProgressRepository : localProgressRepository); const today = todayKey(); + const userId = session?.user.id; + const chain = useRef>(Promise.resolve()); + const pending = useRef(0); + const accountEpoch = useRef(0); + const confirmed = useRef(null); useEffect(() => { let cancelled = false; - (async () => { - setLoading(true); + accountEpoch.current += 1; + pending.current = 0; + confirmed.current = null; + setLoading(true); + setProgress(EMPTY_PROGRESS); + chain.current = chain.current.then(async () => { + if (cancelled) return; // The repository just swapped (sign-in or sign-out). The old account's // state must not stay on screen while the new source loads — wiping the // caches below is not enough when the leak lives in React state. - setProgress(EMPTY_PROGRESS); // Paint from the last known state immediately — on a slow connection // the difference between this and waiting on the network is the whole @@ -103,53 +113,16 @@ export function ProgressProvider({ children, repository: override }: Props) { const next = picked ? await repository.setAssignment(picked.id, today) : stored; if (cancelled) return; - setProgress(next); + confirmed.current = next; + if (pending.current === 0) setProgress(next); setLoading(false); - })(); + if (userId && getConnectivity()) void fetchTopics().catch(() => {}); + }).catch(() => { if (!cancelled) setLoading(false); }); return () => { cancelled = true; + accountEpoch.current += 1; }; - }, [repository, today]); - - // Drain the offline mutation queue when connectivity returns or the app comes - // back to the foreground, then apply the server-reconciled state (issue #133). - // Serialised via `flushing` so overlapping triggers don't double-replay. - useEffect(() => { - if (!repository.flushQueue) return; - let active = true; - let flushing = false; - const flush = async () => { - if (flushing || !active) return; - flushing = true; - try { - const next = await repository.flushQueue?.(); - if (active && next) setProgress(next); - if (active && next && getConnectivity()) await fetchTopics().catch(() => {}); - } catch { - // A flush failure just leaves items queued for the next trigger. - } finally { - flushing = false; - } - }; - - const unsubscribe = subscribeConnectivity((online) => { - if (online) flush(); - }); - const unsubscribeQueue = subscribeQueue(() => { - if (getConnectivity()) flush(); - }); - const appState = AppState.addEventListener('change', (s) => { - if (s === 'active') flush(); - }); - flush(); // catch up on anything left from a previous session - - return () => { - active = false; - unsubscribe(); - unsubscribeQueue(); - appState.remove(); - }; - }, [repository]); + }, [repository, today, userId]); // The day's assignment is pinned once made, even if the concept's topic is // unfollowed later that day — topic changes apply from the next assignment. @@ -184,22 +157,25 @@ export function ProgressProvider({ children, repository: override }: Props) { // momentarily wipe a later tap's optimistic change (a flicker). On failure we // undo just this change functionally, on top of the latest state, so a // concurrent change is never lost (the clobber #39 originally fixed). - const chain = useRef>(Promise.resolve()); - const pending = useRef(0); const apply = useCallback( ( optimistic: ((prev: ProgressState) => ProgressState) | null, - run: () => Promise, + run: () => Promise, undo?: (prev: ProgressState) => ProgressState ) => { + const epoch = accountEpoch.current; if (optimistic) setProgress(optimistic); pending.current += 1; chain.current = chain.current.then(async () => { + if (epoch !== accountEpoch.current) return; try { const next = await run(); + if (epoch !== accountEpoch.current) return; pending.current -= 1; - if (pending.current === 0) setProgress(next); + if (next) confirmed.current = next; + if (pending.current === 0 && confirmed.current) setProgress(confirmed.current); } catch { + if (epoch !== accountEpoch.current) return; pending.current -= 1; if (undo) setProgress(undo); } @@ -209,6 +185,57 @@ export function ProgressProvider({ children, repository: override }: Props) { [] ); + // Queue replay shares the same chain as taps and initial loading. A later + // optimistic action cannot be overwritten by a reconnect's server snapshot. + useEffect(() => { + if (!repository.flushQueue || !isApiConfigured()) return; + let active = true; + const loop = createSyncLoop(async () => { + let retry = true; + await apply(null, async () => { + if (!active) return null; + const entries = await queuedMutations(); + let next: ProgressState | null = null; + if (entries.length) next = await repository.flushQueue!(); + else if (!getConnectivity()) next = await repository.load(); + if (!active) return null; + if (next && getConnectivity()) await fetchTopics().catch(() => {}); + retry = !getConnectivity() || (await queuedMutations()).length > 0; + return next; + }); + return retry; + }, AppState.currentState !== 'background' && AppState.currentState !== 'inactive'); + const unsubscribe = subscribeConnectivity(online => { + if (online) loop.wake(); + else loop.retry(); + }); + const unsubscribeQueue = subscribeQueue(() => { + if (getConnectivity()) loop.wake(); + else loop.retry(); + }); + const appState = AppState.addEventListener('change', state => loop.setActive(state === 'active')); + // Browsers have an immediate reconnect event. Native JS retries pending + // work with backoff because the current APK has no connectivity module. + const reconnect = () => loop.wake(); + const disconnect = () => setConnectivity(false); + if (Platform.OS === 'web') { + window.addEventListener('online', reconnect); + window.addEventListener('offline', disconnect); + } + loop.wake(); + return () => { + active = false; + loop.stop(); + unsubscribe(); + unsubscribeQueue(); + appState.remove(); + if (Platform.OS === 'web') { + window.removeEventListener('online', reconnect); + window.removeEventListener('offline', disconnect); + } + }; + }, [apply, repository, userId]); + // Retry shares the mutation chain, so a refresh cannot overwrite a later tap. const refresh = useCallback(() => apply(null, () => repository.load()), [apply, repository]); diff --git a/mobile/src/services/syncLoop.ts b/mobile/src/services/syncLoop.ts new file mode 100644 index 0000000..a6a2a6c --- /dev/null +++ b/mobile/src/services/syncLoop.ts @@ -0,0 +1,54 @@ +/** Retry only while work remains or the API is unreachable. No native module: + * timers pause in the background and resume immediately on foregrounding. */ +export function createSyncLoop(run: () => Promise, initiallyActive = true) { + let active = initiallyActive; + let stopped = false; + let running = false; + let wakeAgain = false; + let delay = 5000; + let timer: ReturnType | undefined; + + const cancel = () => { + if (timer !== undefined) clearTimeout(timer); + timer = undefined; + }; + const schedule = (ms: number) => { + if (!active || stopped) return; + cancel(); + timer = setTimeout(async () => { + timer = undefined; + running = true; + let retry = true; + try { retry = await run(); } catch { /* retry after transient failure */ } + finally { + running = false; + if (wakeAgain) { + wakeAgain = false; + schedule(0); + } else if (retry) { + schedule(delay); + delay = Math.min(delay * 2, 30000); + } else { + delay = 5000; + } + } + }, ms); + }; + const wake = () => { + if (!active || stopped) return; + delay = 5000; + if (running) wakeAgain = true; + else schedule(0); + }; + + return { + wake, + retry: () => { if (!running && timer === undefined) schedule(delay); }, + setActive: (next: boolean) => { + active = next; + if (active) wake(); + else { cancel(); wakeAgain = false; } + }, + stop: () => { stopped = true; cancel(); wakeAgain = false; }, + }; +} diff --git a/mobile/tests/syncLoop.test.mjs b/mobile/tests/syncLoop.test.mjs new file mode 100644 index 0000000..055448e --- /dev/null +++ b/mobile/tests/syncLoop.test.mjs @@ -0,0 +1,39 @@ +import assert from 'node:assert/strict'; +import { test } from 'node:test'; +import { createSyncLoop } from '../src/services/syncLoop.ts'; + +const settle = async () => { for (let i=0;i<5;i++) await Promise.resolve(); }; +test('retries without navigation, backs off, and stops polling after the queue drains', async t => { + t.mock.timers.enable({apis:['setTimeout']}); + let calls=0, pending=true; + const loop=createSyncLoop(async()=>{ calls++; return pending; }); + t.after(loop.stop); + loop.wake(); t.mock.timers.tick(0); await settle(); + assert.equal(calls,1); + t.mock.timers.tick(4999); await settle(); assert.equal(calls,1); + t.mock.timers.tick(1); await settle(); assert.equal(calls,2); + t.mock.timers.tick(9999); await settle(); assert.equal(calls,2); + pending=false; t.mock.timers.tick(1); await settle(); assert.equal(calls,3); + t.mock.timers.tick(60000); await settle(); assert.equal(calls,3); +}); + +test('background pauses timers and foreground triggers immediate synchronization', async t => { + t.mock.timers.enable({apis:['setTimeout']}); + let calls=0; + const loop=createSyncLoop(async()=>{ calls++; return true; }); + t.after(loop.stop); + loop.wake(); t.mock.timers.tick(0); await settle(); + loop.setActive(false); t.mock.timers.tick(60000); await settle(); assert.equal(calls,1); + loop.setActive(true); t.mock.timers.tick(0); await settle(); assert.equal(calls,2); +}); + +test('overlapping wakeups never run concurrent flushes and stop prevents late rescheduling', async t => { + t.mock.timers.enable({apis:['setTimeout']}); + let calls=0, resolve; + const loop=createSyncLoop(()=>{ calls++; return new Promise(r=>{resolve=r;}); }); + loop.wake(); t.mock.timers.tick(0); await settle(); + loop.wake(); loop.wake(); loop.retry(); t.mock.timers.tick(10000); await settle(); + assert.equal(calls,1); + loop.stop(); resolve(true); await settle(); + t.mock.timers.tick(60000); await settle(); assert.equal(calls,1); +}); From cfa24ea2d579dd587cafc8b31424354829aa2663 Mon Sep 17 00:00:00 2001 From: Muawiya Amir Date: Sat, 12 Sep 2026 16:19:52 +0500 Subject: [PATCH 06/59] fix: preserve queued choices during server reconciliation --- mobile/src/services/pendingProgress.ts | 32 +++++++++++++++++++ .../src/services/remoteProgressRepository.ts | 7 ++-- mobile/tests/pendingProgress.test.mjs | 32 +++++++++++++++++++ 3 files changed, 69 insertions(+), 2 deletions(-) create mode 100644 mobile/src/services/pendingProgress.ts create mode 100644 mobile/tests/pendingProgress.test.mjs diff --git a/mobile/src/services/pendingProgress.ts b/mobile/src/services/pendingProgress.ts new file mode 100644 index 0000000..63617d4 --- /dev/null +++ b/mobile/src/services/pendingProgress.ts @@ -0,0 +1,32 @@ +import type { ProgressState } from '../types'; +import type { QueuedMutation } from './mutationOutbox'; + +/** A server refresh must not erase actions still waiting for acknowledgement. */ +export function withPendingProgress( + server: ProgressState, + previous: ProgressState, + mutations: QueuedMutation[], +): ProgressState { + let state = server; + for (const mutation of mutations) { + if (mutation.kind === 'like' || mutation.kind === 'save') { + const key = mutation.kind === 'like' ? 'likes' : 'bookmarks'; + const ids = state[key].filter(id => id !== mutation.slug); + if (mutation.desired) ids.push(mutation.slug); + state = { ...state, [key]: ids }; + if (mutation.kind === 'save') { + const saved = (state.savedConcepts ?? []).filter(row => row.conceptId !== mutation.slug); + const row = previous.savedConcepts?.find(row => row.conceptId === mutation.slug) + ?? server.savedConcepts?.find(row => row.conceptId === mutation.slug); + if (mutation.desired && row) saved.push(row); + state = { ...state, savedConcepts: saved }; + } + } else if (mutation.kind === 'learn' && mutation.date === state.assignment?.date) { + const record = previous.learned.find(row => row.date === mutation.date); + if (record && !state.learned.some(row => row.date === mutation.date)) { + state = { ...state, learned: [...state.learned, record], stats: previous.stats ?? state.stats }; + } + } + } + return state; +} diff --git a/mobile/src/services/remoteProgressRepository.ts b/mobile/src/services/remoteProgressRepository.ts index f1bea56..4b83b5d 100644 --- a/mobile/src/services/remoteProgressRepository.ts +++ b/mobile/src/services/remoteProgressRepository.ts @@ -4,6 +4,7 @@ import { Category, DailyPayload, ProgressState } from '../types'; import { todayKey } from './dates'; import { cacheSavedConcepts, conceptCache } from './conceptApi'; import { toConcept } from './dailyApi'; +import { withPendingProgress } from './pendingProgress'; import { clearQueue, dequeue, enqueue, keyOf, pending, QueuedMutation } from './mutationQueue'; import { ProgressRepository } from './progressRepository'; import { EMPTY_PROGRESS } from './storage'; @@ -96,6 +97,8 @@ export class RemoteProgressRepository implements ProgressRepository { } private async fromState(payload: StatePayload, epoch: number): Promise { + if (epoch !== this.epoch) return EMPTY_PROGRESS; + const state = withPendingProgress(toProgressState(payload), this.cache, await pending()); if (epoch !== this.epoch) return EMPTY_PROGRESS; // Keep each day's full text for later History/Saved reading, and download // saved bodies without holding up the initial screen. @@ -105,8 +108,8 @@ export class RemoteProgressRepository implements ProgressRepository { await conceptCache.set(concept.id, concept, contentEpoch).catch(() => {}); } if (epoch !== this.epoch) return EMPTY_PROGRESS; - void cacheSavedConcepts(payload.bookmarks, contentEpoch); - return this.remember(toProgressState(payload), epoch); + void cacheSavedConcepts(state.bookmarks, contentEpoch); + return this.remember(state, epoch); } /** Drop the in-memory state; the module singleton outlives a sign-out. The diff --git a/mobile/tests/pendingProgress.test.mjs b/mobile/tests/pendingProgress.test.mjs new file mode 100644 index 0000000..842697c --- /dev/null +++ b/mobile/tests/pendingProgress.test.mjs @@ -0,0 +1,32 @@ +import assert from 'node:assert/strict'; +import { test } from 'node:test'; +import { withPendingProgress } from '../src/services/pendingProgress.ts'; + +const server={learned:[],assignment:{conceptId:'today',date:'2026-09-12'},followedTopics:[],likes:['liked'],bookmarks:[],savedConcepts:[],stats:{current:0,longest:2,totalLearned:2}}; +test('a server refresh after a retryable error preserves pending unlike and saved metadata', () => { + const saved={conceptId:'today',title:'Today',topicName:'Computer Science'}; + const previous={...server,likes:[],bookmarks:['today'],savedConcepts:[saved]}; + const next=withPendingProgress(server,previous,[{kind:'like',slug:'liked',desired:false},{kind:'save',slug:'today',desired:true}]); + assert.deepEqual(next.likes,[]); + assert.deepEqual(next.bookmarks,['today']); + assert.deepEqual(next.savedConcepts,[saved]); + assert.deepEqual(server.likes,['liked']); +}); +test('unacknowledged unsave hides its server row and acknowledged changes use fresh state', () => { + const saved={conceptId:'saved',title:'Saved',topicName:'Computer Science'}; + const state={...server,bookmarks:['saved'],savedConcepts:[saved]}; + const next=withPendingProgress(state,state,[{kind:'save',slug:'saved',desired:false}]); + assert.deepEqual(next.savedConcepts,[]); + assert.deepEqual(next.bookmarks,[]); + assert.equal(withPendingProgress(state,next,[]),state); +}); +test('preserves same-day offline completion without duplicating or backdating it', () => { + const record={conceptId:'today',date:'2026-09-12'}; + const previous={...server,learned:[record],stats:{current:1,longest:2,totalLearned:3}}; + const queued=[{kind:'learn',date:'2026-09-12'}]; + const next=withPendingProgress(server,previous,queued); + assert.deepEqual(next.learned,[record]); + assert.equal(next.stats.totalLearned,3); + assert.deepEqual(withPendingProgress(next,previous,queued).learned,[record]); + assert.deepEqual(withPendingProgress({...server,assignment:{conceptId:'next',date:'2026-09-13'}},previous,queued).learned,[]); +}); From d19e1fe1b44a846567d33cd131ce4c429a97cf1a Mon Sep 17 00:00:00 2001 From: Muawiya Amir Date: Sat, 12 Sep 2026 16:24:35 +0500 Subject: [PATCH 07/59] fix: prevent late requests from restoring signed-out actions --- mobile/src/api/client.ts | 8 + mobile/src/services/accountCaches.ts | 2 + .../src/services/remoteProgressRepository.ts | 20 ++- mobile/tests/README.md | 42 ++++- mobile/tests/offline.browser.cjs | 158 ++++++++++++++++++ 5 files changed, 224 insertions(+), 6 deletions(-) create mode 100644 mobile/tests/offline.browser.cjs diff --git a/mobile/src/api/client.ts b/mobile/src/api/client.ts index c90439e..8da63cd 100644 --- a/mobile/src/api/client.ts +++ b/mobile/src/api/client.ts @@ -26,6 +26,12 @@ export class ApiError extends Error { type TokenProvider = () => Promise; let getAccessToken: TokenProvider = async () => null; +let accountEpoch = 0; + +/** Cancel requests still waiting for a token when the account is cleared. */ +export function invalidateAccountRequests(): void { + accountEpoch += 1; +} /** Registered once by the auth layer in Phase 3. */ export function setTokenProvider(provider: TokenProvider): void { @@ -77,7 +83,9 @@ export async function apiRequest(path: string, options: RequestOptions = {}): throw new ApiError(0, 'EXPO_PUBLIC_API_BASE_URL is not set'); } + const epoch = accountEpoch; const token = await getAccessToken(); + if (epoch !== accountEpoch) throw new ApiError(401, 'Account changed'); const headers: Record = { Accept: 'application/json' }; if (options.body !== undefined) headers['Content-Type'] = 'application/json'; if (token) headers.Authorization = `Bearer ${token}`; diff --git a/mobile/src/services/accountCaches.ts b/mobile/src/services/accountCaches.ts index 96f7279..b74b5b8 100644 --- a/mobile/src/services/accountCaches.ts +++ b/mobile/src/services/accountCaches.ts @@ -7,6 +7,7 @@ * progress) deliberately stay out. */ +import { invalidateAccountRequests } from '../api/client'; import { clearDailyCache } from './dailyApi'; import { conceptCache } from './conceptApi'; import { clearNotificationPrefsCache } from './notifications'; @@ -14,6 +15,7 @@ import { clearServerStateCache } from './remoteProgressRepository'; import { clearTopicsCache } from './topicsApi'; export async function clearAccountCaches(): Promise { + invalidateAccountRequests(); await Promise.all([ clearServerStateCache(), clearDailyCache(), diff --git a/mobile/src/services/remoteProgressRepository.ts b/mobile/src/services/remoteProgressRepository.ts index 4b83b5d..f9e0d6b 100644 --- a/mobile/src/services/remoteProgressRepository.ts +++ b/mobile/src/services/remoteProgressRepository.ts @@ -114,10 +114,10 @@ export class RemoteProgressRepository implements ProgressRepository { /** Drop the in-memory state; the module singleton outlives a sign-out. The * offline queue is account data too, so it goes with it. */ - forget(): void { + forget(): Promise { this.epoch += 1; this.cache = EMPTY_PROGRESS; - clearQueue().catch(() => {}); + return clearQueue(); } async loadCached(): Promise { @@ -182,6 +182,7 @@ export class RemoteProgressRepository implements ProgressRepository { try { done = await apiRequest('/v1/daily/complete', { method: 'POST' }); } catch (err) { + if (epoch !== this.epoch) return EMPTY_PROGRESS; if (isOffline(err)) { // Queue the completion (with the date — it can only be replayed today) // and persist the optimistic record + streak bump so History AND the @@ -203,6 +204,7 @@ export class RemoteProgressRepository implements ProgressRepository { } throw err; } + if (epoch !== this.epoch) return EMPTY_PROGRESS; await dequeue('learn'); // Reload the full state so History shows the true server record — the actual @@ -245,9 +247,11 @@ export class RemoteProgressRepository implements ProgressRepository { method: 'PUT', body: { topics: slugs }, }); + if (epoch !== this.epoch) return EMPTY_PROGRESS; await dequeue('topics'); return this.fromState(payload, epoch); } catch (err) { + if (epoch !== this.epoch) return EMPTY_PROGRESS; if (isOffline(err)) { await enqueue({ kind: 'topics', slugs }); return this.remember({ ...this.cache, followedTopics: next }, epoch); @@ -280,9 +284,11 @@ export class RemoteProgressRepository implements ProgressRepository { }; try { await this.toggle(conceptId, 'like', currently); + if (epoch !== this.epoch) return EMPTY_PROGRESS; await dequeue(`like:${conceptId}`); return this.remember(next, epoch); } catch (err) { + if (epoch !== this.epoch) return EMPTY_PROGRESS; if (isOffline(err)) { await enqueue({ kind: 'like', slug: conceptId, desired }); return this.remember(next, epoch); @@ -325,12 +331,14 @@ export class RemoteProgressRepository implements ProgressRepository { try { await this.toggle(conceptId, 'save', currently); } catch (err) { + if (epoch !== this.epoch) return EMPTY_PROGRESS; if (isOffline(err)) { await enqueue({ kind: 'save', slug: conceptId, desired }); return this.remember(patched(), epoch); } throw err; } + if (epoch !== this.epoch) return EMPTY_PROGRESS; await dequeue(`save:${conceptId}`); // The save/unsave has already persisted. Refresh the full state so the saved // list (which needs each concept's title/topic) reflects it — but if that @@ -370,8 +378,10 @@ export class RemoteProgressRepository implements ProgressRepository { try { await this.replay(m); + if (epoch !== this.epoch) return null; await dequeue(keyOf(m), m); // guarded: don't clobber a newer same-key intent } catch (err) { + if (epoch !== this.epoch) return null; if (isOffline(err)) return null; // still offline — keep the rest queued if (err instanceof ApiError && err.status >= 500) continue; // transient — retry next time await dequeue(keyOf(m), m); // 4xx: unfixable, drop so it can't block forever @@ -413,6 +423,8 @@ export const remoteProgressRepository = new RemoteProgressRepository(); /** Forget everything: the disk cache AND the singleton's in-memory copy. * Called on sign-out so the next account can never see this one's data. */ export async function clearServerStateCache(): Promise { - remoteProgressRepository.forget(); - await AsyncStorage.removeItem(CACHE_KEY).catch(() => {}); + await Promise.all([ + remoteProgressRepository.forget(), + AsyncStorage.removeItem(CACHE_KEY), + ]); } diff --git a/mobile/tests/README.md b/mobile/tests/README.md index 2b18d4e..044834b 100644 --- a/mobile/tests/README.md +++ b/mobile/tests/README.md @@ -12,10 +12,48 @@ For UI checks, use a test account and exercise these flows on preview: - Explicitly sign out, then reopen: the previous account's content must be gone. - Open without cached app data while offline: Today, History, Saved, Stats, and Personalization should explain unavailable content. Try again after reconnecting. -- Open a history/saved concept unavailable offline, then retry online in place. +- While online, open a concept and let saved lessons download. Restart offline: + previously viewed lessons and saved lessons should retain full text/examples. + A lesson never downloaded should show the unavailable state and retry online. +- Open Personalization offline after an online session. Follow/unfollow topics, + including server-added topics, restart offline, and confirm the choices persist. +- Like/save and change topics offline, then restore connectivity while staying + on the same screen. The queue should drain automatically (native retry delay + grows from 5 to at most 30 seconds). Repeat by reopening/foregrounding the app. + Background timers stop; closed-app OS synchronization is outside this scope. +- Interrupt connectivity during replay and try rapid repeated toggles. The last + choice should survive, including across a restart. Sign out with pending work + and confirm the next account sees no old lessons, catalog, or queued actions. - Try signing in, signing up, and resetting a password offline: show connection advice without raw Java/JavaScript diagnostics. Invalid credentials remain clear. - Check light/dark themes and large text. The offline cloud should float gently, remain still with reduced motion, and stop while its screen/app is inactive. -These targeted tests do not cover the separate mutation-queue work in issue #157. +The Node tests cover storage ordering, account cleanup races, cached topics, +latest-intent coalescing, and retry scheduling. End-to-end API replay and native +lifecycle behavior still require the UI checks above; they are not simulated by +the storage unit tests. + +## Mocked browser regression + +`offline.browser.cjs` exercises the actual exported app with dummy authentication +and intercepted API calls. It uses an existing Playwright installation; set +`PLAYWRIGHT_TEST_MODULE` to its `playwright/test` module if it is outside this +project, and optionally set `PLAYWRIGHT_CHROMIUM_PATH` to a Chromium executable. +No live account or backend is used. From `mobile/`, export and run: + +```sh +CI=1 EXPO_NO_DOTENV=1 EXPO_NO_TELEMETRY=1 EXPO_OFFLINE=1 \ + EXPO_PUBLIC_API_BASE_URL=http://127.0.0.1:4781/api \ + EXPO_PUBLIC_SUPABASE_URL=http://127.0.0.1:4781 \ + EXPO_PUBLIC_SUPABASE_ANON_KEY=test-public-key \ + npx expo export --platform web --output-dir /tmp/one-concept-offline +node tests/offline.browser.cjs /tmp/one-concept-offline \ + --no-event --retry-error --signout-inflight +``` + +The flags exercise timer-only reconnection, a 503 during replay, and a request +that fails after sign-out. Omit `--no-event` to test the browser online event. +For the unchanged pre-fix export, `--baseline` asserts the original #133 failures. +The test uses port 4781 and closes its server/browser afterward. Optional +`OFFLINE_SCREENSHOT_PATH` saves the offline detail view for visual inspection. diff --git a/mobile/tests/offline.browser.cjs b/mobile/tests/offline.browser.cjs new file mode 100644 index 0000000..5dd1885 --- /dev/null +++ b/mobile/tests/offline.browser.cjs @@ -0,0 +1,158 @@ +const { chromium, expect } = require(process.env.PLAYWRIGHT_TEST_MODULE || 'playwright/test'); +const http = require('node:http'); +const fs = require('node:fs'); +const path = require('node:path'); +const assert = require('node:assert/strict'); +const appVersion = require('../app.config.js').expo.version; +const root = process.argv[2]; +if (!root || !fs.existsSync(path.join(root, 'index.html'))) throw new Error('Pass an Expo web export directory as the first argument.'); +const baseline = process.argv.includes('--baseline'); +const date = new Date().toISOString().slice(0, 10); +const concept = (slug, title) => ({ id: slug, slug, title, summary: `Full explanation of ${title}.`, example: `A concrete example of ${title}.`, topic_slug: 'computer-science', topic_name: 'Computer Science', like_count: 2 }); +const daily = concept('fixture-daily', 'Daily fixture'); +const saved = concept('fixture-saved', 'Saved fixture'); +const unread = concept('fixture-unread', 'Unread saved fixture'); +const state = { display_name: 'Fixture', timezone: 'UTC', today: date, followed_topics: ['computer-science'], learned: [], likes: [], bookmarks: [saved.slug, unread.slug], saved: [saved, unread].map(c => ({concept_slug:c.slug, title:c.title, topic_name:c.topic_name})), stats: {current:0,longest:0,total_learned:0}, assignment_slug: daily.slug, daily: { assigned_for: date, assigned_at: new Date().toISOString(), completed_at: null, learned:false, outside_followed_topics:false, concept:daily } }; +let online = true; +let failLikes = false; +let holdLike = false, releaseLike; +const writes = []; +const errors = []; +const session = { access_token:'fixture', refresh_token:'fixture-refresh', token_type:'bearer', expires_in:3600, expires_at:Math.floor(Date.now()/1000)+3600, user:{ id:'11111111-1111-1111-1111-111111111111', email:'fixture@example.invalid', aud:'authenticated', role:'authenticated', app_metadata:{}, user_metadata:{}, created_at:'2026-01-01T00:00:00Z' } }; +const server = http.createServer((req,res) => { + const relative = decodeURIComponent(new URL(req.url, 'http://localhost').pathname); + const file = path.join(root, relative === '/' ? 'index.html' : relative); + try { const body = fs.readFileSync(file); res.setHeader('Content-Type', ({'.html':'text/html','.js':'application/javascript','.ttf':'font/ttf','.png':'image/png'})[path.extname(file)] || 'application/octet-stream'); res.end(body); } + catch { res.statusCode=404; res.end(); } +}); +(async () => { + await new Promise(r => server.listen(4781, '127.0.0.1', r)); + const browser = await chromium.launch({ executablePath:process.env.PLAYWRIGHT_CHROMIUM_PATH, headless:true, args:['--no-sandbox'] }); + try { + const context = await browser.newContext({viewport:{width:390,height:844}}); + await context.addInitScript(({session, appVersion}) => { + Object.defineProperty(navigator, 'share', {configurable:true,value:async data=>{window.fixtureShared=data;}}); + if (!localStorage.getItem('fixture-seeded')) { + localStorage.setItem('sb-127-auth-token', JSON.stringify(session)); + localStorage.setItem('one-concept/last-seen-version/v1',appVersion); + localStorage.setItem('fixture-seeded','yes'); + } + }, {session, appVersion}); + await context.route('**/api/**', async route => { + if (!online) return route.abort('internetdisconnected'); + const req = route.request(); const endpoint = new URL(req.url()).pathname.replace('/api',''); + const method = req.method(); + if (holdLike && endpoint.endsWith('/like')) { await new Promise(r=>{releaseLike=r;}); return route.abort('internetdisconnected'); } + if (method !== 'GET') writes.push({endpoint,method,body:req.postDataJSON()}); + if (failLikes && endpoint.endsWith('/like')) return route.fulfill({status:503,contentType:'application/json',body:'{}'}); + let body = {}; + if (endpoint === '/v1/me/topics' && method === 'PUT') state.followed_topics = req.postDataJSON().topics; + const interaction = endpoint.match(/^\/v1\/concepts\/([^/]+)\/(like|save)$/); + if (interaction) { + const key=interaction[2]==='like'?'likes':'bookmarks'; const slug=interaction[1]; + state[key] = state[key].filter(id=>id!==slug); + if(method==='PUT') state[key].push(slug); + } + if (endpoint === '/v1/me/state' || endpoint === '/v1/me/topics') body=state; + else if (endpoint === '/v1/topics') body=[{slug:'computer-science',name:'Computer Science',concept_count:25},{slug:'new-topic',name:'New topic',concept_count:12}].map(t=>({...t,following:state.followed_topics.includes(t.slug)})); + else if (endpoint.startsWith('/v1/concepts/') && method === 'GET') body=[daily,saved,unread].find(c=>endpoint.endsWith(c.slug)); + else if (endpoint === '/v1/me/notifications') body={enabled:false,reminder_times:[]}; + await route.fulfill({status:200,contentType:'application/json',body:JSON.stringify(body)}); + }); + await context.route('**/auth/v1/**', route => route.abort('internetdisconnected')); + const page=await context.newPage(); page.setDefaultTimeout(12000); + page.on('pageerror',e=>errors.push(e.message)); + const profile=async()=>page.getByText('Profile',{exact:true}).last().click(); + const close=async()=>page.getByRole('button',{name:'Close',exact:true}).last().click(); + const queue=async()=>page.evaluate(()=>JSON.parse(localStorage.getItem('one-concept/mutation-queue/v1')||'{}')); + await page.goto('http://127.0.0.1:4781'); + await expect(page.getByText(daily.summary,{exact:true})).toBeVisible(); + await profile(); await page.getByText('Saved concepts',{exact:true}).click(); + await page.getByRole('button',{name:'Open Saved fixture',exact:true}).click(); + await expect(page.getByText(saved.summary,{exact:true})).toBeVisible(); + console.log('CONTROL: saved detail loads online'); + online=false; await page.reload(); + await expect(page.getByText(daily.summary,{exact:true})).toBeVisible(); + await profile(); await page.getByText('Saved concepts',{exact:true}).click(); + await page.getByRole('button',{name:'Open Saved fixture',exact:true}).click(); + if (baseline) await expect(page.getByText('This concept couldn’t be loaded. Check your connection and try again.',{exact:true})).toBeVisible(); + else await expect(page.getByText(saved.summary,{exact:true})).toBeVisible(); + console.log(baseline?'CONFIRMED: previously viewed saved body unavailable offline':'PASS: viewed saved body survives offline restart'); + if (process.env.OFFLINE_SCREENSHOT_PATH) await page.screenshot({path:process.env.OFFLINE_SCREENSHOT_PATH}); + if (!baseline) { + await page.getByRole('button',{name:'Share',exact:true}).last().click(); + await expect.poll(()=>page.evaluate(()=>window.fixtureShared?.text)).toContain(saved.summary); + console.log('PASS: sharing cached text works offline'); + } + await close(); + if (!baseline) { + await page.getByRole('button',{name:'Open Unread saved fixture',exact:true}).click(); + await expect(page.getByText(unread.summary,{exact:true})).toBeVisible(); + console.log('PASS: saved body downloaded without opening its detail'); await close(); + } + online=true; await page.reload(); + await expect(page.getByText(daily.summary,{exact:true})).toBeVisible(); + await profile(); await page.getByText('Personalize your feed',{exact:true}).click(); + await expect(page.getByText('New topic',{exact:true})).toBeVisible(); + online=false; await page.getByText('Follow',{exact:true}).click(); + await expect(page.getByText('Following',{exact:true})).toHaveCount(2); + await page.waitForTimeout(500); + if (baseline) assert.equal((await queue()).topics, undefined); + else assert.deepEqual((await queue()).topics.slugs,['computer-science','new-topic']); + console.log(baseline?'CONFIRMED: offline topic change is not in persistent queue':'PASS: offline topic change is queued'); + await page.reload(); await expect(page.getByText(daily.summary,{exact:true})).toBeVisible(); + await profile(); await page.getByText('Personalize your feed',{exact:true}).click(); + if(baseline) await expect(page.getByText('Connect to load your topics and choose what to learn next.',{exact:true})).toBeVisible(); + else { await expect(page.getByText('New topic',{exact:true})).toBeVisible(); await expect(page.getByText('Following',{exact:true})).toHaveCount(2); } + console.log(baseline?'CONFIRMED: warm topic catalog lost on offline restart':'PASS: catalog and followed choices survive offline restart'); + await close(); await page.getByText('Today',{exact:true}).last().click(); + await page.getByRole('button',{name:'Like',exact:true}).click(); + await expect.poll(async()=>(await queue())['like:fixture-daily']?.desired).toBe(true); + console.log('CONTROL: offline like is persistently queued'); + const before=writes.length; online=true; + if (!process.argv.includes('--no-event')) await page.evaluate(()=>window.dispatchEvent(new Event('online'))); + if(baseline) { await page.waitForTimeout(2500); assert.equal(writes.length,before); assert.equal((await queue())['like:fixture-daily'].desired,true); } + else { await expect.poll(async()=>Object.keys(await queue()).length,{timeout:35000}).toBe(0); assert(state.likes.includes('fixture-daily')); assert(state.followed_topics.includes('new-topic')); } + console.log(baseline?'CONFIRMED: connectivity restore alone does not flush queue':'PASS: reconnection automatically flushes likes and topics'); + if (!baseline) { + await profile(); await page.getByText('Personalize your feed',{exact:true}).click(); + await expect(page.getByText('Following',{exact:true})).toHaveCount(2); + await close(); await page.getByText('Today',{exact:true}).last().click(); + if (process.argv.includes('--retry-error')) { + online=false; await page.getByRole('button',{name:'Unlike',exact:true}).click(); + await expect.poll(async()=>(await queue())['like:fixture-daily']?.desired).toBe(false); + failLikes=true; online=true; + await page.evaluate(()=>window.dispatchEvent(new Event('online'))); + await page.waitForTimeout(1500); + await expect(page.getByRole('button',{name:'Like',exact:true})).toBeVisible(); + assert.equal((await queue())['like:fixture-daily'].desired,false); + console.log('PASS: transient replay error preserves pending unlike in the UI'); + failLikes=false; await page.evaluate(()=>window.dispatchEvent(new Event('online'))); + await expect.poll(async()=>Object.keys(await queue()).length).toBe(0); + } + online=false; + await page.getByRole('button',{name:'Save for later',exact:true}).click(); + await expect.poll(async()=>(await queue())['save:fixture-daily']?.desired).toBe(true); + await page.reload(); await expect(page.getByText(daily.summary,{exact:true})).toBeVisible(); + await expect(page.getByRole('button',{name:'Remove from saved',exact:true})).toBeVisible(); + await profile(); await page.getByText('Saved concepts',{exact:true}).click(); + await page.getByRole('button',{name:'Open Daily fixture',exact:true}).click(); + await expect(page.getByText(daily.summary,{exact:true}).last()).toBeVisible(); + await close(); await page.getByRole('button',{name:'Back',exact:true}).click(); + console.log('PASS: offline save survives restart and its full text opens from Saved'); + if (process.argv.includes('--signout-inflight')) { + online=true; holdLike=true; + await page.getByText('Today',{exact:true}).last().click(); + await page.getByRole('button',{name:/^(Like|Unlike)$/}).click(); + await expect.poll(()=>Boolean(releaseLike)).toBe(true); + await profile(); + } + await page.getByText('Sign out',{exact:true}).click(); + await expect(page.getByText('Welcome back — sign in to pick up your streak.',{exact:true})).toBeVisible(); + if (releaseLike) { releaseLike(); await page.waitForTimeout(500); } + await expect.poll(async()=>page.evaluate(()=>Object.keys(localStorage).filter(key=>key.startsWith('one-concept/concepts/') || key.startsWith('one-concept/topics/') || key==='one-concept/mutation-queue/v1').length)).toBe(0); + console.log('PASS: sign-out removes full lessons, topics, and queued actions'); + } + assert.deepEqual(errors,[]); console.log('No browser runtime errors'); + } finally { await browser.close(); await new Promise(r=>server.close(r)); } +})().catch(e=>{console.error(e);process.exitCode=1;server.close();}); From d10bd935614ce1fcf063c621ffd88a5a8c0bddfd Mon Sep 17 00:00:00 2001 From: Muawiya Amir Date: Sat, 12 Sep 2026 16:25:56 +0500 Subject: [PATCH 08/59] docs: record issue 133 validation and handoff --- docs/CODEBASE_MAP.md | 44 ++++++++++++++++++++++++++++---------------- docs/WORK_LOG.md | 38 ++++++++++++++++++++++++++++++++++++-- 2 files changed, 64 insertions(+), 18 deletions(-) diff --git a/docs/CODEBASE_MAP.md b/docs/CODEBASE_MAP.md index d4505f8..6cbd441 100644 --- a/docs/CODEBASE_MAP.md +++ b/docs/CODEBASE_MAP.md @@ -34,7 +34,7 @@ inside a root stack, with a concept-detail modal above them. | `ProfileScreen.tsx` | Account, reminder preferences, theme, sign-out, and links to profile subpages. | | `PersonalizationScreen.tsx` | Server topic catalog and follow controls through `useTopics`. | | `SavedScreen.tsx` | Saved concepts and detail navigation. | -| `ConceptDetailScreen.tsx` | Full lesson fetched by slug, with bundled catalog fallback. | +| `ConceptDetailScreen.tsx` | Cached full lesson first, then online refresh by slug; bundled catalog fallback. | | `AuthScreen.tsx` | Sign-in, sign-up, and password recovery. | | `AboutScreen.tsx` | Branding and app information. | @@ -61,26 +61,37 @@ typography, shadows, and scaling; `ThemeContext` persists light/dark preference. `api/fetchWithTimeout.ts` bounds API and auth fetches to 15 seconds. - `ProgressContext.tsx` is the shared UI state owner. It loads cached state before revalidation, applies optimistic actions, serializes mutation requests, - and flushes queued work on foreground/connectivity events. + and flushes queued work on the same mutation chain. `services/syncLoop.ts` + retries while offline or actions remain, using 5–30 second backoff. Foreground + and browser reconnect events wake it immediately; backgrounding pauses timers. + This remains compatible with the current APK and has no closed-app worker. Screen retries use its serialized `refresh`; topic and detail screens have their own retry paths. Failed loads do not substitute demo lessons or totals for an authenticated account. - `services/progressRepository.ts` defines the persistence interface. `remoteProgressRepository.ts` implements API state, account caching, optimistic - offline fallbacks, and replay. `localProgressRepository.ts` and `storage.ts` + offline fallbacks, and replay. `pendingProgress.ts` retains unacknowledged + likes, saves, and same-day completions during server reconciliation. + `localProgressRepository.ts` and `storage.ts` retain local/demo support; this is not a separate visible guest navigation flow. -- `mutationQueue.ts` stores the latest intent per like/save/topic/completion key - in AsyncStorage. Replay discards stale-day completions, retains retryable - failures, and reconciles state. It does not backdate server completion. +- `mutationQueue.ts` wires AsyncStorage to `mutationOutbox.ts`, which serializes + disk writes and stores the latest intent per like/save/topic/completion key. + Replay discards stale-day completions, retains retryable failures, and + reconciles state. It does not backdate server completion. - `accountCaches.ts` centralizes account cache cleanup. The remote repository's - epoch guards prevent some late results from being persisted after a wipe. + epoch guards reject late mutation callbacks after a wipe; the API invalidates + requests still waiting for an old account's token during cleanup. Device theme/demo state is separate from account data. - `dailyApi.ts` maps server concepts to UI types and clears an old daily cache; - current daily data arrives in `/v1/me/state`. `conceptApi.ts` fetches full - concepts by slug. UI concept IDs are slugs, while the database also has UUIDs. -- `hooks/useTopics.ts` and `services/topicsApi.ts` load the dynamic server catalog - and replace follow sets. This path is separate from the progress repository's - queued topic mutations; inspect the caller before assuming offline support. + current daily data arrives in `/v1/me/state`. `conceptApi.ts` persists full + lessons by slug, including each cached daily lesson and missing saved lessons + downloaded with three workers. Offline reading requires a completed download. + `offlineCache.ts` provides per-entry storage and fences late writes on sign-out. + UI concept IDs are slugs, while the database also has UUIDs. +- `hooks/useTopics.ts`, `services/topicsApi.ts`, and `topicStore.ts` share the + cached dynamic topic catalog. Follow changes enter the same durable outbox as + other actions; queued choices override stale server responses until replay. + Both the catalog and full-concept cache participate in account cleanup. - `services/notifications.ts` handles permissions, Android channel setup, Expo tokens, timezone sync, preference caching, and deregistration before sign-out. - `data/concepts.ts`, `services/dailyConcept.ts`, `dates.ts`, `streak.ts`, and @@ -168,7 +179,8 @@ session pooler. Applied migrations must not be rewritten. - Mobile dependencies/scripts are in `mobile/package.json` and `package-lock.json`; use npm. `npm run typecheck` runs `tsc --noEmit`; `npm test` uses Node 24's - built-in runner for session recovery, auth messages, and request timeouts. + built-in runner for session recovery, auth messages, request timeouts, offline + cache cleanup, outbox ordering, topic persistence, and sync scheduling. Expo provides Android/iOS/web development commands. Native project folders are not tracked. EAS profiles separate development, preview, production, and production APKs. - Backend dependencies are pinned in `requirements.txt`/`requirements-dev.txt`. @@ -207,8 +219,8 @@ the implementation or older documentation: only. Both are used by the authenticated application today. - `mobile/DEPLOYMENT.md` describes an older OTA trigger and fewer workflows. Use current workflow YAML plus `RELEASING.md` to trace release behavior. -- The roadmap still lists offline reading as future work although cached state - and queued progress writes exist. This does not establish complete offline - coverage for every screen (topic personalization has its own request path). +- The roadmap's older offline milestones predate the current full-lesson cache, + cached personalization, and foreground queue synchronization. Closed-app OS + background scheduling remains outside the current APK's capabilities. - The backend README's test-count/phase notes are historical. See [WORK_LOG.md](WORK_LOG.md) for the actual local validation baseline. diff --git a/docs/WORK_LOG.md b/docs/WORK_LOG.md index cd2ad42..81e306c 100644 --- a/docs/WORK_LOG.md +++ b/docs/WORK_LOG.md @@ -28,10 +28,44 @@ claims as completed work. - Planned atomic commits: persistent full-concept reading with account cleanup and tests; cached topic catalog and queued follows with tests; automatic sync triggers with tests; validation, codebase map, and PR handoff. -- Clarification pending: keep the current APK with sync while open/reopened, or - add native OS background scheduling (new APK, OS-controlled execution timing). +- Owner chose to keep the current APK: automatically retry while the app is open + or reopened. No OS background worker, native dependency, or version/runtime bump. - Read the exact Expo SDK 57 documentation before mobile edits. Other issues, including #182's password-visibility request, remain outside this PR. +- Additional replay checks reproduced two related failures before their fixes: + a 503 reverted a queued unlike in the UI; a request failing after sign-out + recreated the old action queue. Preserve pending choices during reconciliation + and fence late request callbacks/token resolution after account cleanup. +- Implemented full per-lesson storage and missing saved-lesson downloads; + shared cached topics with durable follows; serialized outbox writes; automatic + foreground retry with 5–30 second backoff, immediate browser/foreground wakeup, + and no idle polling once reachable with an empty queue. Saved reading requires + the lesson to have finished downloading during an online session. +- **Validation:** all 28 Node 24 regression tests, TypeScript, Android/web Expo + production exports, and whitespace checks passed. The mocked Chromium flow + passed offline restart, unopened saved-lesson downloads, cached sharing, follows, + likes/saves, timer-only and browser-event reconnect, 503 retention/recovery, + and sign-out with a request in flight. Inspected the offline detail screenshot. + No live backend, physical phone, native share sheet, or production deployment + was tested; no backend code changed and backend tests were not run. +- **Repeatability:** `mobile/tests/offline.browser.cjs` and its README retain the + before/after reproduction flow, dummy public configuration, and optional race + checks. New focused Node tests cover storage, outbox, topics, reconciliation, + and scheduler behavior without adding dependencies. + +| Change | Commit | +| --- | --- | +| Baseline reproduction and scope | `0e82f03` | +| Full offline lesson storage and saved downloads | `0085285` | +| Cached topics and queued follow choices | `41da1ab` | +| Serialized durable outbox | `f8d1ca6` | +| Automatic foreground synchronization | `1a82c2e` | +| Preserve pending choices during reconciliation | `cfa24ea` | +| Sign-out request fence and browser regression | `d19e1fe` | +| Validation and navigation guide | `docs: record issue 133 validation and handoff` | + +- **Handoff:** publish one PR into `develop` for owner review. Preserve the + individual commits; do not merge the PR or prepare a release in this task. ## Working agreement — 2026-09-11 From 5d769d40ba62373d822fdc5867e86140de115489 Mon Sep 17 00:00:00 2001 From: Muawiya Amir Date: Sat, 12 Sep 2026 16:27:51 +0500 Subject: [PATCH 09/59] docs: link issue 133 pull request --- docs/WORK_LOG.md | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/docs/WORK_LOG.md b/docs/WORK_LOG.md index 81e306c..7a6ecfc 100644 --- a/docs/WORK_LOG.md +++ b/docs/WORK_LOG.md @@ -7,9 +7,10 @@ claims as completed work. ## Current status -- Assigned scope: only [#133](https://github.com/Coding-Moves/one-concept/issues/133). - Confirm the problems first, then open one PR into `develop` for offline reading, - personalization persistence, and automatic action synchronization. +- Completed scope: only [#133](https://github.com/Coding-Moves/one-concept/issues/133). + [PR #183](https://github.com/Coding-Moves/one-concept/pull/183) is open into + `develop` for offline reading, personalization persistence, and automatic + action synchronization. Problems were reproduced before implementation. - Branch: `codex/133-offline-reading-sync`, based on refreshed `origin/develop` at `89a8fb8`. The starting tree matches the tested baseline. - Release #179 and card follow-up #180 are merged; #181 synchronized `main` @@ -64,8 +65,11 @@ claims as completed work. | Sign-out request fence and browser regression | `d19e1fe` | | Validation and navigation guide | `docs: record issue 133 validation and handoff` | -- **Handoff:** publish one PR into `develop` for owner review. Preserve the - individual commits; do not merge the PR or prepare a release in this task. +- **PR:** [#183](https://github.com/Coding-Moves/one-concept/pull/183), opened + into `develop` with all individual commits and the owner's configured identity. + Publication bookkeeping: `docs: link issue 133 pull request`. +- **Handoff:** ready for owner review. The PR remains open; merging and release + preparation are the owner's next steps, outside this task. ## Working agreement — 2026-09-11 From d9a07f533b7ad97e7c4bc1c759166a5d79474075 Mon Sep 17 00:00:00 2001 From: Muawiya Amir Date: Sat, 12 Sep 2026 16:42:27 +0500 Subject: [PATCH 10/59] fix: preserve pending actions across midnight --- mobile/src/context/ProgressContext.tsx | 19 +++++++++--------- mobile/tests/offline.browser.cjs | 27 ++++++++++++++++++++++++-- 2 files changed, 34 insertions(+), 12 deletions(-) diff --git a/mobile/src/context/ProgressContext.tsx b/mobile/src/context/ProgressContext.tsx index 51de35a..783f2ce 100644 --- a/mobile/src/context/ProgressContext.tsx +++ b/mobile/src/context/ProgressContext.tsx @@ -82,25 +82,27 @@ export function ProgressProvider({ children, repository: override }: Props) { const accountEpoch = useRef(0); const confirmed = useRef(null); + // Only an account/source change invalidates queued actions. A new calendar + // day refreshes the assignment without cancelling taps still in flight. useEffect(() => { - let cancelled = false; accountEpoch.current += 1; pending.current = 0; confirmed.current = null; setLoading(true); setProgress(EMPTY_PROGRESS); + return () => { accountEpoch.current += 1; }; + }, [repository, userId]); + + useEffect(() => { + let cancelled = false; chain.current = chain.current.then(async () => { if (cancelled) return; - // The repository just swapped (sign-in or sign-out). The old account's - // state must not stay on screen while the new source loads — wiping the - // caches below is not enough when the leak lives in React state. - // Paint from the last known state immediately — on a slow connection // the difference between this and waiting on the network is the whole // perceived speed of the app. The fresh load replaces it silently. const cached = await repository.loadCached?.(); if (cached && !cancelled) { - setProgress(cached); + if (pending.current === 0) setProgress(cached); setLoading(false); } @@ -118,10 +120,7 @@ export function ProgressProvider({ children, repository: override }: Props) { setLoading(false); if (userId && getConnectivity()) void fetchTopics().catch(() => {}); }).catch(() => { if (!cancelled) setLoading(false); }); - return () => { - cancelled = true; - accountEpoch.current += 1; - }; + return () => { cancelled = true; }; }, [repository, today, userId]); // The day's assignment is pinned once made, even if the concept's topic is diff --git a/mobile/tests/offline.browser.cjs b/mobile/tests/offline.browser.cjs index 5dd1885..b121e80 100644 --- a/mobile/tests/offline.browser.cjs +++ b/mobile/tests/offline.browser.cjs @@ -7,6 +7,7 @@ const appVersion = require('../app.config.js').expo.version; const root = process.argv[2]; if (!root || !fs.existsSync(path.join(root, 'index.html'))) throw new Error('Pass an Expo web export directory as the first argument.'); const baseline = process.argv.includes('--baseline'); +const midnight = process.argv.includes('--midnight'); const date = new Date().toISOString().slice(0, 10); const concept = (slug, title) => ({ id: slug, slug, title, summary: `Full explanation of ${title}.`, example: `A concrete example of ${title}.`, topic_slug: 'computer-science', topic_name: 'Computer Science', like_count: 2 }); const daily = concept('fixture-daily', 'Daily fixture'); @@ -18,7 +19,7 @@ let failLikes = false; let holdLike = false, releaseLike; const writes = []; const errors = []; -const session = { access_token:'fixture', refresh_token:'fixture-refresh', token_type:'bearer', expires_in:3600, expires_at:Math.floor(Date.now()/1000)+3600, user:{ id:'11111111-1111-1111-1111-111111111111', email:'fixture@example.invalid', aud:'authenticated', role:'authenticated', app_metadata:{}, user_metadata:{}, created_at:'2026-01-01T00:00:00Z' } }; +const session = { access_token:'fixture', refresh_token:'fixture-refresh', token_type:'bearer', expires_in:864000, expires_at:Math.floor(Date.now()/1000)+864000, user:{ id:'11111111-1111-1111-1111-111111111111', email:'fixture@example.invalid', aud:'authenticated', role:'authenticated', app_metadata:{}, user_metadata:{}, created_at:'2026-01-01T00:00:00Z' } }; const server = http.createServer((req,res) => { const relative = decodeURIComponent(new URL(req.url, 'http://localhost').pathname); const file = path.join(root, relative === '/' ? 'index.html' : relative); @@ -29,7 +30,7 @@ const server = http.createServer((req,res) => { await new Promise(r => server.listen(4781, '127.0.0.1', r)); const browser = await chromium.launch({ executablePath:process.env.PLAYWRIGHT_CHROMIUM_PATH, headless:true, args:['--no-sandbox'] }); try { - const context = await browser.newContext({viewport:{width:390,height:844}}); + const context = await browser.newContext({viewport:{width:390,height:844},timezoneId:'UTC'}); await context.addInitScript(({session, appVersion}) => { Object.defineProperty(navigator, 'share', {configurable:true,value:async data=>{window.fixtureShared=data;}}); if (!localStorage.getItem('fixture-seeded')) { @@ -65,8 +66,30 @@ const server = http.createServer((req,res) => { const profile=async()=>page.getByText('Profile',{exact:true}).last().click(); const close=async()=>page.getByRole('button',{name:'Close',exact:true}).last().click(); const queue=async()=>page.evaluate(()=>JSON.parse(localStorage.getItem('one-concept/mutation-queue/v1')||'{}')); + if (midnight) await page.clock.setFixedTime(new Date(date+'T23:59:00Z')); await page.goto('http://127.0.0.1:4781'); await expect(page.getByText(daily.summary,{exact:true})).toBeVisible(); + if (midnight) { + holdLike=true; + await page.getByRole('button',{name:'Like',exact:true}).click(); + await expect.poll(()=>Boolean(releaseLike)).toBe(true); + await page.clock.setFixedTime(new Date(new Date(date+'T23:59:00Z').getTime()+120000)); + await page.getByRole('button',{name:'Save for later',exact:true}).click(); + // Let the date-change effect run while Save is waiting behind Like. + await page.waitForTimeout(300); + online=false; holdLike=false; releaseLike(); releaseLike=undefined; + await expect.poll(async()=>(await queue())['save:fixture-daily']?.desired).toBe(true); + await expect(page.getByRole('button',{name:'Remove from saved',exact:true})).toBeVisible(); + await page.reload(); + await expect(page.getByRole('button',{name:'Remove from saved',exact:true})).toBeVisible(); + online=true; await page.evaluate(()=>window.dispatchEvent(new Event('online'))); + await expect.poll(async()=>Object.keys(await queue()).length).toBe(0); + assert(state.bookmarks.includes(daily.slug)); + assert(writes.some(w=>w.endpoint==='/v1/concepts/fixture-daily/save' && w.method==='PUT')); + assert.deepEqual(errors,[]); + console.log('PASS: save queued across midnight survives restart and syncs'); + return; + } await profile(); await page.getByText('Saved concepts',{exact:true}).click(); await page.getByRole('button',{name:'Open Saved fixture',exact:true}).click(); await expect(page.getByText(saved.summary,{exact:true})).toBeVisible(); From 409b1afdecff172e714347120ab1e56ab97267c0 Mon Sep 17 00:00:00 2001 From: Muawiya Amir Date: Sat, 12 Sep 2026 16:44:44 +0500 Subject: [PATCH 11/59] fix: retain backoff after partial sync failures --- mobile/src/services/syncLoop.ts | 13 ++++---- mobile/tests/offline.browser.cjs | 32 ++++++++++++++++++++ mobile/tests/syncLoop.test.mjs | 52 ++++++++++++++++++++++++++++++++ 3 files changed, 91 insertions(+), 6 deletions(-) diff --git a/mobile/src/services/syncLoop.ts b/mobile/src/services/syncLoop.ts index a6a2a6c..e8079f7 100644 --- a/mobile/src/services/syncLoop.ts +++ b/mobile/src/services/syncLoop.ts @@ -22,23 +22,24 @@ export function createSyncLoop(run: () => Promise, initiallyActive = tr try { retry = await run(); } catch { /* retry after transient failure */ } finally { running = false; - if (wakeAgain) { - wakeAgain = false; - schedule(0); - } else if (retry) { + const repeat = wakeAgain; + wakeAgain = false; + // A successful request can report online before a later request in + // this run fails. Such wakeups must not bypass or reset retry backoff. + if (retry) { schedule(delay); delay = Math.min(delay * 2, 30000); } else { delay = 5000; + if (repeat) schedule(0); } } }, ms); }; const wake = () => { if (!active || stopped) return; - delay = 5000; if (running) wakeAgain = true; - else schedule(0); + else { delay = 5000; schedule(0); } }; return { diff --git a/mobile/tests/offline.browser.cjs b/mobile/tests/offline.browser.cjs index b121e80..a41fe53 100644 --- a/mobile/tests/offline.browser.cjs +++ b/mobile/tests/offline.browser.cjs @@ -16,6 +16,8 @@ const unread = concept('fixture-unread', 'Unread saved fixture'); const state = { display_name: 'Fixture', timezone: 'UTC', today: date, followed_topics: ['computer-science'], learned: [], likes: [], bookmarks: [saved.slug, unread.slug], saved: [saved, unread].map(c => ({concept_slug:c.slug, title:c.title, topic_name:c.topic_name})), stats: {current:0,longest:0,total_learned:0}, assignment_slug: daily.slug, daily: { assigned_for: date, assigned_at: new Date().toISOString(), completed_at: null, learned:false, outside_followed_topics:false, concept:daily } }; let online = true; let failLikes = false; +let failTopics = false; +let stateRequests = 0, topicRequests = 0; let holdLike = false, releaseLike; const writes = []; const errors = []; @@ -43,6 +45,11 @@ const server = http.createServer((req,res) => { if (!online) return route.abort('internetdisconnected'); const req = route.request(); const endpoint = new URL(req.url()).pathname.replace('/api',''); const method = req.method(); + if (endpoint === '/v1/me/state') stateRequests++; + if (endpoint === '/v1/topics') { + topicRequests++; + if (failTopics) return route.abort('connectionreset'); + } if (holdLike && endpoint.endsWith('/like')) { await new Promise(r=>{releaseLike=r;}); return route.abort('internetdisconnected'); } if (method !== 'GET') writes.push({endpoint,method,body:req.postDataJSON()}); if (failLikes && endpoint.endsWith('/like')) return route.fulfill({status:503,contentType:'application/json',body:'{}'}); @@ -69,6 +76,31 @@ const server = http.createServer((req,res) => { if (midnight) await page.clock.setFixedTime(new Date(date+'T23:59:00Z')); await page.goto('http://127.0.0.1:4781'); await expect(page.getByText(daily.summary,{exact:true})).toBeVisible(); + if (process.argv.includes('--partial-connectivity')) { + await expect.poll(()=>topicRequests).toBeGreaterThan(0); + await page.waitForTimeout(300); + const beforeState=stateRequests, beforeTopics=topicRequests; + failTopics=true; + await page.evaluate(()=>{window.dispatchEvent(new Event('offline'));window.dispatchEvent(new Event('online'));}); + await expect.poll(()=>topicRequests-beforeTopics).toBeGreaterThan(0); + await page.waitForTimeout(1000); + assert.equal(stateRequests-beforeState,1); + assert.equal(topicRequests-beforeTopics,1); + await expect.poll(()=>topicRequests-beforeTopics,{timeout:7000}).toBe(2); + await page.waitForTimeout(1000); + assert.equal(stateRequests-beforeState,2); + assert.equal(topicRequests-beforeTopics,2); + // Recover without an online event: the backed-off timer must still retry. + failTopics=false; + await expect.poll(()=>topicRequests-beforeTopics,{timeout:12000}).toBe(3); + await expect(page.getByText('Offline — changes will sync when you reconnect',{exact:true})).toHaveCount(0); + await page.waitForTimeout(1000); + assert.equal(stateRequests-beforeState,3); + assert.equal(topicRequests-beforeTopics,3); + assert.deepEqual(errors,[]); + console.log('PASS: partial connectivity backs off, recovers automatically, and stops polling'); + return; + } if (midnight) { holdLike=true; await page.getByRole('button',{name:'Like',exact:true}).click(); diff --git a/mobile/tests/syncLoop.test.mjs b/mobile/tests/syncLoop.test.mjs index 055448e..f784d1f 100644 --- a/mobile/tests/syncLoop.test.mjs +++ b/mobile/tests/syncLoop.test.mjs @@ -27,6 +27,58 @@ test('background pauses timers and foreground triggers immediate synchronization loop.setActive(true); t.mock.timers.tick(0); await settle(); assert.equal(calls,2); }); +test('partial connectivity wakeups retain exponential backoff until synchronization succeeds', async t => { + t.mock.timers.enable({apis:['setTimeout']}); + let calls=0, failing=true; + const loop=createSyncLoop(async()=>{ + calls++; + if (failing) { + // A state response reports online; the subsequent topics fetch fails. + loop.wake(); + loop.retry(); + } + return failing; + }); + t.after(loop.stop); + loop.wake(); t.mock.timers.tick(0); await settle(); + for (const delay of [5000,10000,20000,30000,30000]) { + const before=calls; + t.mock.timers.tick(delay-1); await settle(); assert.equal(calls,before); + t.mock.timers.tick(1); await settle(); assert.equal(calls,before+1); + } + failing=false; + t.mock.timers.tick(30000); await settle(); assert.equal(calls,7); + t.mock.timers.tick(60000); await settle(); assert.equal(calls,7); +}); + +test('a reconnect while waiting for retry triggers an immediate attempt', async t => { + t.mock.timers.enable({apis:['setTimeout']}); + let calls=0, failing=true; + const loop=createSyncLoop(async()=>{ calls++; return failing; }); + t.after(loop.stop); + loop.wake(); t.mock.timers.tick(0); await settle(); + t.mock.timers.tick(1000); await settle(); assert.equal(calls,1); + failing=false; loop.wake(); t.mock.timers.tick(0); await settle(); + assert.equal(calls,2); + t.mock.timers.tick(60000); await settle(); assert.equal(calls,2); +}); + +test('wakeups during successful synchronization coalesce into one additional attempt', async t => { + t.mock.timers.enable({apis:['setTimeout']}); + let calls=0, resolve; + const loop=createSyncLoop(()=>{ + calls++; + return calls===1 ? new Promise(r=>{resolve=r;}) : Promise.resolve(false); + }); + t.after(loop.stop); + loop.wake(); t.mock.timers.tick(0); await settle(); + loop.wake(); loop.wake(); + t.mock.timers.tick(1000); await settle(); assert.equal(calls,1); + resolve(false); await settle(); + t.mock.timers.tick(0); await settle(); assert.equal(calls,2); + t.mock.timers.tick(60000); await settle(); assert.equal(calls,2); +}); + test('overlapping wakeups never run concurrent flushes and stop prevents late rescheduling', async t => { t.mock.timers.enable({apis:['setTimeout']}); let calls=0, resolve; From 15e047eadaf84e1c8cee1b347b6ada57a7a2bb3e Mon Sep 17 00:00:00 2001 From: Muawiya Amir Date: Sat, 12 Sep 2026 16:45:55 +0500 Subject: [PATCH 12/59] docs: record PR 183 review fixes --- docs/CODEBASE_MAP.md | 7 +++++-- docs/WORK_LOG.md | 29 +++++++++++++++++++++++++++++ mobile/tests/README.md | 7 +++++++ 3 files changed, 41 insertions(+), 2 deletions(-) diff --git a/docs/CODEBASE_MAP.md b/docs/CODEBASE_MAP.md index 6cbd441..fb8727b 100644 --- a/docs/CODEBASE_MAP.md +++ b/docs/CODEBASE_MAP.md @@ -62,8 +62,11 @@ typography, shadows, and scaling; `ThemeContext` persists light/dark preference. - `ProgressContext.tsx` is the shared UI state owner. It loads cached state before revalidation, applies optimistic actions, serializes mutation requests, and flushes queued work on the same mutation chain. `services/syncLoop.ts` - retries while offline or actions remain, using 5–30 second backoff. Foreground - and browser reconnect events wake it immediately; backgrounding pauses timers. + retries while offline or actions remain, using 5–30 second backoff, including + when only some requests succeed. Daily refreshes preserve pending actions; + account/source changes invalidate them and clear the displayed state. Foreground + and browser reconnect events wake an idle loop immediately; backgrounding + pauses timers. This remains compatible with the current APK and has no closed-app worker. Screen retries use its serialized `refresh`; topic and detail screens have their own retry paths. Failed loads do not substitute demo lessons or totals diff --git a/docs/WORK_LOG.md b/docs/WORK_LOG.md index 7a6ecfc..e1fd244 100644 --- a/docs/WORK_LOG.md +++ b/docs/WORK_LOG.md @@ -7,6 +7,9 @@ claims as completed work. ## Current status +- Review follow-up complete in the same PR #183: pending actions survive midnight + and partial connection failures retain retry backoff. Each fix has its own + commit with regression coverage; prior commits and APK compatibility remain. - Completed scope: only [#133](https://github.com/Coding-Moves/one-concept/issues/133). [PR #183](https://github.com/Coding-Moves/one-concept/pull/183) is open into `develop` for offline reading, personalization persistence, and automatic @@ -16,6 +19,32 @@ claims as completed work. - Release #179 and card follow-up #180 are merged; #181 synchronized `main` back into `develop`. This task does not authorize another production release. +## PR #183 review fixes — 2026-09-12 + +- Owner requested both findings fixed in the existing PR against `develop`. + Planned and delivered one fix/test commit per finding, followed by this + documentation handoff. Read the exact Expo SDK 57 documentation before edits. +- Reproduced both problems before editing: a save waiting behind a like across + midnight never persisted (the unchanged `develop` control succeeded); successful + state requests followed by failed topics requests caused rapid repeated fetches. + The new committed browser scenarios fail on the previous PR export. +- `d9a07f5` — `fix: preserve pending actions across midnight`: split account + invalidation from daily refresh, preserve pending counts, and prevent cached + previews from overwriting pending optimistic actions. Browser coverage checks + queued save persistence through date change, offline restart, and replay. +- `409b1af` — `fix: retain backoff after partial sync failures`: failed attempts + retain backoff even when their own requests report connectivity changes. + Tests cover the full 5/10/20/30-second progression, immediate reconnect while + waiting, coalesced successful wakeups, and automatic recovery in the browser. +- **Validation:** 31 Node 24 tests, TypeScript, Android/web production exports, + and whitespace checks passed. Both new browser scenarios and the full offline + flow passed, including timer-only reconnect, 503 recovery, offline save restart, + and sign-out with an in-flight request. No browser runtime errors. No backend + changes; backend tests, live services, and physical-device checks were not run. +- **Handoff:** both fixes stay in [PR #183](https://github.com/Coding-Moves/one-concept/pull/183) + on `codex/133-offline-reading-sync`. Native runtime, version, and dependencies + are unchanged. Documentation commit: `docs: record PR 183 review fixes`. + ## Offline reading and synchronization (#133) — 2026-09-12 - Before editing, exported the unchanged web app and exercised it in Chromium diff --git a/mobile/tests/README.md b/mobile/tests/README.md index 044834b..6c6b03c 100644 --- a/mobile/tests/README.md +++ b/mobile/tests/README.md @@ -50,10 +50,17 @@ CI=1 EXPO_NO_DOTENV=1 EXPO_NO_TELEMETRY=1 EXPO_OFFLINE=1 \ npx expo export --platform web --output-dir /tmp/one-concept-offline node tests/offline.browser.cjs /tmp/one-concept-offline \ --no-event --retry-error --signout-inflight +node tests/offline.browser.cjs /tmp/one-concept-offline --midnight +node tests/offline.browser.cjs /tmp/one-concept-offline --partial-connectivity ``` The flags exercise timer-only reconnection, a 503 during replay, and a request that fails after sign-out. Omit `--no-event` to test the browser online event. +The separate `--midnight` scenario holds a like request across the date change, +queues Save behind it, and verifies persistence through offline restart and replay. +`--partial-connectivity` serves progress but fails topics, checks that requests +back off, and restores topics to verify automatic recovery without a browser event. +The scheduler unit tests check the full 5–30 second delay progression. For the unchanged pre-fix export, `--baseline` asserts the original #133 failures. The test uses port 4781 and closes its server/browser afterward. Optional `OFFLINE_SCREENSHOT_PATH` saves the offline detail view for visual inspection. From 50d73abf3b1fd655ef4f800d682495e9e817502e Mon Sep 17 00:00:00 2001 From: Muawiya Amir Date: Sat, 12 Sep 2026 18:28:48 +0500 Subject: [PATCH 13/59] docs: record database idle reconnect reproduction --- docs/WORK_LOG.md | 36 +++++++++++++++++++++++++++--------- 1 file changed, 27 insertions(+), 9 deletions(-) diff --git a/docs/WORK_LOG.md b/docs/WORK_LOG.md index e1fd244..4becd79 100644 --- a/docs/WORK_LOG.md +++ b/docs/WORK_LOG.md @@ -7,18 +7,36 @@ claims as completed work. ## Current status -- Review follow-up complete in the same PR #183: pending actions survive midnight - and partial connection failures retain retry backoff. Each fix has its own - commit with regression coverage; prior commits and APK compatibility remain. -- Completed scope: only [#133](https://github.com/Coding-Moves/one-concept/issues/133). - [PR #183](https://github.com/Coding-Moves/one-concept/pull/183) is open into - `develop` for offline reading, personalization persistence, and automatic - action synchronization. Problems were reproduced before implementation. -- Branch: `codex/133-offline-reading-sync`, based on refreshed `origin/develop` - at `89a8fb8`. The starting tree matches the tested baseline. +- Active scope: [#149](https://github.com/Coding-Moves/one-concept/issues/149), + reducing database reconnect work on requests after idle time. Owner authorized + a focused backend fix in a separate PR into `develop`. +- Branch: `codex/149-db-connection-warmup`, from refreshed `origin/develop` + at `f34ba77`. PR #183, including both review fixes for #133, is merged. +- Plan: reproduce idle expiry with PostgreSQL 16 before implementation; add a + bounded, configurable warm-up with lifecycle and reconnect tests; document + measured outcomes and open the PR. Keep connection safety checks and the + current pooler mode. No mobile, migration, or production configuration changes. +- The issue's production 600 ms figure has not been independently reproduced. + All regression tests and before/after experiments use disposable test services. - Release #179 and card follow-up #180 are merged; #181 synchronized `main` back into `develop`. This task does not authorize another production release. +## Database connections after idle (#149) — 2026-09-12 + +- Reproduced before implementation using the unchanged app engine/session and + a disposable PostgreSQL 16 container on loopback port 55434. Set the test + server's idle-session timeout to 1.5 seconds, then waited two seconds between + requests. SQLAlchemy connection events confirm all three post-idle requests + created a new physical connection: 260.23, 221.45, and 220.35 ms. Immediate + reuse took 10.91, 10.24, and 9.40 ms with zero new connections. +- This reproduces the request-time reconnect mechanism under controlled idle + expiry, not Supavisor's deployed timeout or the issue's production 600 ms figure. +- Intended fix: configurable, bounded probes in the FastAPI lifespan, reusing + the most recently returned connection. Keep `pool_pre_ping`, transaction-mode + configuration, and current connection limits. Cancel probes before pool disposal. +- Planned commits: reproduction/scope; warm connection lifecycle with regression + tests; measured validation, operational instructions, and PR handoff. + ## PR #183 review fixes — 2026-09-12 - Owner requested both findings fixed in the existing PR against `develop`. From 6bf0399f1e83e375a5be3c077159ccfd96e75e73 Mon Sep 17 00:00:00 2001 From: Muawiya Amir Date: Sat, 12 Sep 2026 18:36:27 +0500 Subject: [PATCH 14/59] fix: keep API database connections warm between requests --- backend/.env.example | 5 + backend/app/config.py | 5 + backend/app/db/keepalive.py | 55 ++++++++ backend/app/db/session.py | 36 ++--- backend/app/main.py | 22 +++- backend/tests/test_db_keepalive.py | 137 ++++++++++++++++++++ backend/tests/test_db_keepalive_postgres.py | 105 +++++++++++++++ 7 files changed, 346 insertions(+), 19 deletions(-) create mode 100644 backend/app/db/keepalive.py create mode 100644 backend/tests/test_db_keepalive.py create mode 100644 backend/tests/test_db_keepalive_postgres.py diff --git a/backend/.env.example b/backend/.env.example index 75f8176..b7b9e39 100644 --- a/backend/.env.example +++ b/backend/.env.example @@ -13,6 +13,11 @@ DATABASE_URL=postgresql://postgres.PROJECT:PASSWORD@aws-0-REGION.pooler.supabase # Session-mode pooler (port 5432). Used for migrations and any DDL. DIRECT_URL=postgresql://postgres.PROJECT:PASSWORD@aws-0-REGION.pooler.supabase.com:5432/postgres +# Best-effort API connection warm-up. Zero interval disables it. +# The timeout covers pool checkout, reconnect, and the SELECT 1 probe. +DB_KEEPALIVE_INTERVAL_SECONDS=30 +DB_KEEPALIVE_TIMEOUT_SECONDS=5 + # Settings → API SUPABASE_URL=https://PROJECT.supabase.co # Public anon key (Settings → API). Safe to expose — it's already in the mobile diff --git a/backend/app/config.py b/backend/app/config.py index 46654c3..f034b63 100644 --- a/backend/app/config.py +++ b/backend/app/config.py @@ -1,5 +1,6 @@ from functools import lru_cache +from pydantic import Field from pydantic_settings import BaseSettings, SettingsConfigDict @@ -17,6 +18,10 @@ class Settings(BaseSettings): database_url: str direct_url: str | None = None + # Best-effort API pool warm-up; zero disables it. Workers do not start it. + db_keepalive_interval_seconds: float = Field(default=30, ge=0, allow_inf_nan=False) + db_keepalive_timeout_seconds: float = Field(default=5, gt=0, allow_inf_nan=False) + supabase_url: str supabase_jwks_url: str # Present for legacy HS256 projects; this project signs with ES256 via JWKS. diff --git a/backend/app/db/keepalive.py b/backend/app/db/keepalive.py new file mode 100644 index 0000000..fc5927d --- /dev/null +++ b/backend/app/db/keepalive.py @@ -0,0 +1,55 @@ +"""Keep a reusable API connection warm without adding work to user requests.""" + +import asyncio +import logging +from time import perf_counter + +from sqlalchemy.ext.asyncio import AsyncConnection, AsyncEngine + +logger = logging.getLogger("uvicorn.error") + + +async def _release(connection: AsyncConnection, timeout: float) -> None: + try: + async with asyncio.timeout(timeout): + await connection.close() + except BaseException: + # A failed rollback must not leave a borrowed or broken pool slot. + await connection.invalidate() + raise + + +async def _probe(engine: AsyncEngine, timeout: float) -> None: + connection = None + try: + async with asyncio.timeout(timeout): + connection = await engine.connect() + await connection.exec_driver_sql("SELECT 1") + finally: + if connection is not None: + # SQLAlchemy's context exit shields close(), but cancellation can + # return before that close finishes. Retain and await cleanup so + # lifespan disposal cannot race a connection still being returned. + cleanup = asyncio.create_task(_release(connection, timeout)) + try: + await asyncio.shield(cleanup) + except asyncio.CancelledError: + await cleanup + raise + + +async def keep_database_warm(engine: AsyncEngine, interval: float, timeout: float) -> None: + """Probe immediately, then periodically; cancellation belongs to the lifespan.""" + while True: + started = perf_counter() + try: + await _probe(engine, timeout) + logger.debug("Database warm-up completed in %.1f ms", (perf_counter() - started) * 1000) + except Exception as error: + # Cleanup errors must not turn a shutdown cancellation into a retry. + if asyncio.current_task().cancelling(): + raise asyncio.CancelledError from error + # Outages must not stop the API or cause tight retry loops. Log only + # the exception type: driver messages can contain connection details. + logger.warning("Database warm-up failed (%s); retrying later", type(error).__name__) + await asyncio.sleep(interval) diff --git a/backend/app/db/session.py b/backend/app/db/session.py index 684f2c0..78d102f 100644 --- a/backend/app/db/session.py +++ b/backend/app/db/session.py @@ -1,8 +1,8 @@ from collections.abc import AsyncIterator -from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine +from sqlalchemy.ext.asyncio import AsyncEngine, AsyncSession, async_sessionmaker, create_async_engine -from app.config import get_settings +from app.config import Settings, get_settings settings = get_settings() @@ -11,20 +11,24 @@ # is not only safe but necessary: without it every request pays a fresh # TCP + TLS + auth handshake to the database region, which dominates response # time when the database is far away. -_connect_args: dict = {} -if settings.uses_transaction_pooler: - _connect_args["statement_cache_size"] = 0 - -engine = create_async_engine( - settings.sqlalchemy_url, - echo=False, - pool_size=5, - max_overflow=5, - pool_recycle=1800, - pool_pre_ping=True, - connect_args=_connect_args, - execution_options={"compiled_cache": None} if settings.uses_transaction_pooler else {}, -) +def create_db_engine(config: Settings) -> AsyncEngine: + connect_args = {"statement_cache_size": 0} if config.uses_transaction_pooler else {} + return create_async_engine( + config.sqlalchemy_url, + echo=False, + pool_size=5, + max_overflow=5, + pool_recycle=1800, + pool_pre_ping=True, + # Keep the most recently used connection hot instead of cycling through + # every idle slot. Other slots still reconnect safely on demand. + pool_use_lifo=True, + connect_args=connect_args, + execution_options={"compiled_cache": None} if config.uses_transaction_pooler else {}, + ) + + +engine = create_db_engine(settings) SessionLocal = async_sessionmaker(engine, expire_on_commit=False, class_=AsyncSession) diff --git a/backend/app/main.py b/backend/app/main.py index 9cd6038..ecf18be 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -1,5 +1,6 @@ +import asyncio import logging -from contextlib import asynccontextmanager +from contextlib import asynccontextmanager, suppress from fastapi import FastAPI from fastapi.middleware.cors import CORSMiddleware @@ -9,6 +10,7 @@ from app.api.v1.router import api_router from app.config import get_settings from app.core.security import JwksCache +from app.db.keepalive import keep_database_warm from app.db.session import engine @@ -22,8 +24,22 @@ async def lifespan(app: FastAPI): settings.environment, settings.uses_transaction_pooler, ) - yield - await engine.dispose() + warmup = None + if settings.db_keepalive_interval_seconds > 0: + warmup = asyncio.create_task( + keep_database_warm( + engine, settings.db_keepalive_interval_seconds, settings.db_keepalive_timeout_seconds + ), + name="database-keepalive", + ) + try: + yield + finally: + if warmup is not None: + warmup.cancel() + with suppress(asyncio.CancelledError): + await warmup + await engine.dispose() def create_app() -> FastAPI: diff --git a/backend/tests/test_db_keepalive.py b/backend/tests/test_db_keepalive.py new file mode 100644 index 0000000..6e7259c --- /dev/null +++ b/backend/tests/test_db_keepalive.py @@ -0,0 +1,137 @@ +"""Pool warming must be bounded, release connections, and follow API lifetime.""" + +import asyncio +from contextlib import suppress +from types import SimpleNamespace +from unittest.mock import AsyncMock + +import pytest + +from app.config import Settings +from app.db.keepalive import keep_database_warm + + +async def stop(task): + task.cancel() + with suppress(asyncio.CancelledError): + await task + + +@pytest.mark.parametrize("failure", ["query_error", "query_timeout", "checkout_timeout", "release_timeout"]) +async def test_probe_recovers_without_overlap_or_leaking_a_connection(failure, caplog): + entered = 0 + active = 0 + maximum_active = 0 + recovered = asyncio.Event() + + async def query(sql): + assert sql == "SELECT 1" + if entered == 1: + if failure == "query_error": + raise ConnectionError("private connection details must not be logged") + if failure != "release_timeout": + await asyncio.Event().wait() + else: + recovered.set() + + async def connect(): + nonlocal entered, active, maximum_active + entered += 1 + if entered == 1 and failure == "checkout_timeout": + await asyncio.Event().wait() + active += 1 + maximum_active = max(maximum_active, active) + released = False + + async def invalidate(): + nonlocal active, released + if released: + return + released = True + active -= 1 + + async def close(): + if entered == 1 and failure == "release_timeout": + await asyncio.Event().wait() + await invalidate() + + return SimpleNamespace(exec_driver_sql=query, close=close, invalidate=invalidate) + + task = asyncio.create_task(keep_database_warm(SimpleNamespace(connect=connect), 0.02, 0.02)) + try: + await asyncio.wait_for(recovered.wait(), 2) + finally: + await stop(task) + assert entered == 2 + assert maximum_active == 1 + assert active == 0 + assert "Database warm-up failed" in caplog.text + assert "private connection details" not in caplog.text + + +@pytest.mark.parametrize("fail_request", [False, True]) +@pytest.mark.parametrize("fail_cleanup", [False, True]) +async def test_lifespan_starts_without_waiting_and_cancels_probe_before_disposal(monkeypatch, fail_request, fail_cleanup): + from app import main + + config = Settings( + _env_file=None, database_url="postgresql://test:test@localhost/test", + supabase_url="http://test.invalid", supabase_jwks_url="http://test.invalid/jwks", + db_keepalive_interval_seconds=30, + ) + events = [] + probing = asyncio.Event() + + async def query(sql): + probing.set() + await asyncio.Event().wait() + + async def close(): + events.append("released") + if fail_cleanup: + raise ConnectionError("rollback failed during shutdown") + + connection = SimpleNamespace( + exec_driver_sql=query, + close=close, + invalidate=AsyncMock(), + ) + engine = SimpleNamespace( + connect=AsyncMock(return_value=connection), + dispose=AsyncMock(side_effect=lambda: events.append("disposed")), + ) + monkeypatch.setattr(main, "engine", engine) + monkeypatch.setattr(main, "get_settings", lambda: config) + + async def run(): + async with main.lifespan(main.app): + await asyncio.wait_for(probing.wait(), 2) + if fail_request: + raise RuntimeError("lifespan body failed") + + if fail_request: + with pytest.raises(RuntimeError, match="lifespan body failed"): + await run() + else: + await run() + assert events == ["released", "disposed"] + assert connection.invalidate.await_count == int(fail_cleanup) + + +async def test_zero_interval_disables_warming(monkeypatch): + from app import main + + config = Settings( + _env_file=None, database_url="postgresql://test:test@localhost/test", + supabase_url="http://test.invalid", supabase_jwks_url="http://test.invalid/jwks", + db_keepalive_interval_seconds=0, + ) + probe = AsyncMock() + engine = SimpleNamespace(dispose=AsyncMock()) + monkeypatch.setattr(main, "get_settings", lambda: config) + monkeypatch.setattr(main, "engine", engine) + monkeypatch.setattr(main, "keep_database_warm", probe) + async with main.lifespan(main.app): + await asyncio.sleep(0) + probe.assert_not_called() + engine.dispose.assert_awaited_once() diff --git a/backend/tests/test_db_keepalive_postgres.py b/backend/tests/test_db_keepalive_postgres.py new file mode 100644 index 0000000..5a262ea --- /dev/null +++ b/backend/tests/test_db_keepalive_postgres.py @@ -0,0 +1,105 @@ +"""Reproduce idle expiry with real PostgreSQL, never a configured live database.""" + +import asyncio +from contextlib import suppress +from time import perf_counter + +import pytest +from sqlalchemy import event, text + +from app.config import Settings +from app.db.session import create_db_engine + + +@pytest.mark.parametrize("warming", [False, True]) +async def test_idle_request_reconnects_only_when_warming_is_disabled(database, monkeypatch, warming): + from app import main + + config = Settings( + _env_file=None, database_url=database, + supabase_url="http://test.invalid", supabase_jwks_url="http://test.invalid/jwks", + db_keepalive_interval_seconds=0.2 if warming else 0, + ) + engine = create_db_engine(config) + connections = 0 + + @event.listens_for(engine.sync_engine, "connect") + def connected(connection, _): + nonlocal connections + connections += 1 + # Expire only this test's physical sessions, outside a transaction. + connection.run_async(lambda driver: driver.execute("SET idle_session_timeout = '800ms'")) + + monkeypatch.setattr(main, "engine", engine) + monkeypatch.setattr(main, "get_settings", lambda: config) + + async def sample(): + started = perf_counter() + async with engine.connect() as connection: + assert await connection.scalar(text("SELECT 1")) == 1 + return (perf_counter() - started) * 1000 + + # Start with two checked-in slots, as after a small concurrent traffic burst. + # Requests must reuse a warm slot even when the pool has more than one slot. + async with engine.connect(), engine.connect(): + pass + try: + async with main.lifespan(main.app): + before = connections + cold_or_reused = [] + for _ in range(3): + await asyncio.sleep(1.2) + # Measure outside a probe's checkout, so this is idle latency, + # not a request competing for a currently borrowed connection. + async with asyncio.timeout(5): + while engine.pool.checkedout(): + await asyncio.sleep(0.01) + cold_or_reused.append(round(await sample(), 2)) + assert connections - before == (0 if warming else 3) + print(f"warming={warming}, new_connections={connections-before}, request_ms={cold_or_reused}") + finally: + await engine.dispose() + + +async def test_warmup_releases_transactions_and_recovers_from_a_dead_connection(database): + from app.db.keepalive import keep_database_warm + + config = Settings( + _env_file=None, database_url=database, + supabase_url="http://test.invalid", supabase_jwks_url="http://test.invalid/jwks", + ) + engine = create_db_engine(config) + completed = asyncio.Event() + + @event.listens_for(engine.sync_engine, "after_cursor_execute") + def executed(*_): + completed.set() + + task = asyncio.create_task(keep_database_warm(engine, 0.05, 5)) + try: + await asyncio.wait_for(completed.wait(), 5) + # Cancel immediately after SELECT, potentially during rollback/return. + task.cancel() + with suppress(asyncio.CancelledError): + await task + assert engine.pool.checkedout() == 0 + async with engine.connect() as connection: + driver = (await connection.get_raw_connection()).driver_connection + assert not driver.is_in_transaction() + await connection.scalar(text("SELECT 1")) + pid = await connection.scalar(text("SELECT pg_backend_pid()")) + + # A pool slot can still die between probes; pre-ping must remain enabled. + killer = create_db_engine(config) + try: + async with killer.connect() as connection: + assert await connection.scalar(text("SELECT pg_terminate_backend(:pid)"), {"pid": pid}) + finally: + await killer.dispose() + async with engine.connect() as connection: + assert await connection.scalar(text("SELECT 1")) == 1 + finally: + task.cancel() + with suppress(asyncio.CancelledError): + await task + await engine.dispose() From 679163abf97a488ef5fe6e03d5353dcd12ee067c Mon Sep 17 00:00:00 2001 From: Muawiya Amir Date: Sat, 12 Sep 2026 18:37:27 +0500 Subject: [PATCH 15/59] docs: document database warm-up validation --- backend/README.md | 37 ++++++++++++++++++++++++++++++++++--- docs/CODEBASE_MAP.md | 14 ++++++++++---- docs/WORK_LOG.md | 39 ++++++++++++++++++++++++++++++++------- 3 files changed, 76 insertions(+), 14 deletions(-) diff --git a/backend/README.md b/backend/README.md index ea88e28..6426de3 100644 --- a/backend/README.md +++ b/backend/README.md @@ -143,9 +143,40 @@ statements rather than their complexity: - Connection pooling is on. Without it every request paid a fresh TCP + TLS + auth handshake to the database region, which cost seconds. -The remaining latency is geography. **Deploy the API in the same region as the -database** — on Railway, pick the region closest to your Supabase project — and -these round trips drop to single-digit milliseconds. +Geography and connection reuse both affect latency. **Deploy the API close to +the database**, then compare fresh, immediately reused, and post-idle requests. +Timing a slow query alone does not establish that a new connection was opened. + +The API runs a best-effort `SELECT 1` probe immediately on startup and then +every `DB_KEEPALIVE_INTERVAL_SECONDS` (default 30 seconds, measured after each +probe finishes). It borrows from the same pool as requests and promptly returns +the connection, rolling back the implicit transaction. The pool reuses its most +recently returned connection, so low traffic can use a warm slot while surplus +idle slots expire. `pool_pre_ping` remains enabled for dead connections. + +`DB_KEEPALIVE_TIMEOUT_SECONDS` (default 5 seconds) bounds checkout, reconnect, +and query together. Rollback/return has a separate budget of the same duration; +failed cleanup invalidates the connection. A failed attempt logs only the +exception type and retries after the interval; it does not block API startup. +Shutdown cancels the task before disposing the engine. Set the interval to `0` +to disable probes. Each API +process runs its own task; importing the engine in cron workers starts no task. +Pool size and overflow limits remain 5 each. A cold startup or a burst requiring +additional connections can still pay connection setup time. + +Reproduce idle expiry without production services using: + +```bash +DATABASE_URL=postgresql+asyncpg://postgres:postgres@127.0.0.1:55433/postgres \ +SUPABASE_URL=http://test.invalid SUPABASE_JWKS_URL=http://test.invalid/jwks \ +GENERATION_ENABLED=false GEMINI_API_KEY= \ + .venv/bin/python -m pytest tests/test_db_keepalive_postgres.py -q -s +``` + +The PostgreSQL 16 tests set short idle expiry only on their own sessions and +compare connection counts and request timings with warming disabled/enabled. +They also check transaction cleanup and recovery from a terminated connection. +These timings describe the local test environment, not deployed Supavisor latency. ## Connection strings diff --git a/docs/CODEBASE_MAP.md b/docs/CODEBASE_MAP.md index fb8727b..c607dc1 100644 --- a/docs/CODEBASE_MAP.md +++ b/docs/CODEBASE_MAP.md @@ -108,8 +108,12 @@ typography, shadows, and scaling; `ThemeContext` persists light/dark preference. ## Backend request and service flow `main.py` configures CORS, routes, production documentation visibility, a shared -JWKS cache, and engine cleanup. `config.py` loads settings and normalizes pooler -URLs; `db/session.py` creates the async engine/session dependency. +JWKS cache, and engine cleanup. Its lifespan owns `db/keepalive.py`'s configurable +database probes; checkout/query and connection return are bounded, failures retry, +and cancellation awaits cleanup before engine disposal. `config.py` loads settings +and normalizes pooler URLs; `db/session.py` creates the async engine/session +dependency and reuses the most recently returned connection to keep a hot slot. +The existing pre-ping, transaction pooler mode, and pool limits remain in place. `deps.py` obtains identity from bearer tokens verified by `core/security.py` (ES256, issuer, audience, expiry, and subject). `core/errors.py` formats auth errors. @@ -189,8 +193,10 @@ session pooler. Applied migrations must not be rewritten. - Backend dependencies are pinned in `requirements.txt`/`requirements-dev.txt`. From `backend/`, run `.venv/bin/python -m uvicorn app.main:app --reload --port 8000` for development and `.venv/bin/python -m pytest` for tests after configuration. -- Seven test modules cover HTTP contracts, token validation, daily selection, - writes/streaks, generation, reminders, and notification preferences. +- Nine test modules cover HTTP contracts, token validation, daily selection, + writes/streaks, generation, reminders, notification preferences, and connection + warm-up/cleanup. The pool integration checks compare real PostgreSQL idle expiry + with warming disabled/enabled and print timing plus physical-connection counts. `tests/conftest.py` supplies a disposable PostgreSQL 16 database through Podman on port 55433, applies every migration, and disables live generation. HTTP calls to Gemini/Expo are mocked. Database-dependent tests skip if Podman cannot start. diff --git a/docs/WORK_LOG.md b/docs/WORK_LOG.md index 4becd79..9ab1c0b 100644 --- a/docs/WORK_LOG.md +++ b/docs/WORK_LOG.md @@ -7,15 +7,14 @@ claims as completed work. ## Current status -- Active scope: [#149](https://github.com/Coding-Moves/one-concept/issues/149), - reducing database reconnect work on requests after idle time. Owner authorized - a focused backend fix in a separate PR into `develop`. +- Completed implementation: [#149](https://github.com/Coding-Moves/one-concept/issues/149), + reducing database reconnect work on requests after idle time. Validated and + preparing the focused backend PR into `develop`. - Branch: `codex/149-db-connection-warmup`, from refreshed `origin/develop` at `f34ba77`. PR #183, including both review fixes for #133, is merged. -- Plan: reproduce idle expiry with PostgreSQL 16 before implementation; add a - bounded, configurable warm-up with lifecycle and reconnect tests; document - measured outcomes and open the PR. Keep connection safety checks and the - current pooler mode. No mobile, migration, or production configuration changes. +- Delivered: idle-expiry reproduction, bounded/configurable warm-up with lifecycle + and reconnect coverage, and operational instructions. Connection safety checks + and current pooler mode remain. No mobile, migration, or production changes. - The issue's production 600 ms figure has not been independently reproduced. All regression tests and before/after experiments use disposable test services. - Release #179 and card follow-up #180 are merged; #181 synchronized `main` @@ -36,6 +35,32 @@ claims as completed work. configuration, and current connection limits. Cancel probes before pool disposal. - Planned commits: reproduction/scope; warm connection lifecycle with regression tests; measured validation, operational instructions, and PR handoff. +- Implemented an immediate then periodic API lifespan probe (30-second default, + zero to disable), reusing the most recently returned pool connection. Checkout/ + query and rollback/return each have a five-second default budget. Failures retry + after the interval; logs omit raw connection details. Shutdown waits for cleanup, + including when cleanup itself fails. The task does not start in cron workers. +- A real-PostgreSQL test caught cancellation returning before SQLAlchemy finished + connection cleanup. Fixed it before handoff and added failure/cancellation coverage. +- **Validation:** all 105 backend tests passed, with real PostgreSQL 16 integration + coverage and no skips. Ruff (`F,E9`) and whitespace checks passed. The final + controlled comparison (800 ms test-session idle timeout, 1.2-second gaps) was: + + | Warm-up | Three request times (ms) | New physical connections | + | --- | --- | --- | + | Disabled | 223.98, 209.61, 234.05 | 3 | + | Enabled, 200 ms test interval | 7.93, 8.18, 8.29 | 0 | + + Both cases use the same engine factory, query, and disposable database. Timings + are reported rather than asserted; connection counts are the regression check. + Covered query/checkout/cleanup timeouts, recovery, cancellation, transaction + release, terminated connections, disabled probes, and lifespan failures. +- **Limits:** no live Supabase/Railway measurement, production deployment, mobile + test, or native build. The issue's deployed timeout/600 ms figure remains + unverified. Cold startup and additional connections during bursts can still pay + setup cost. Each API process adds a periodic probe with the configured interval. +- Commits: `50d73ab` — reproduction and scope; `6bf0399` — implementation/tests; + documentation handoff — `docs: document database warm-up validation`. ## PR #183 review fixes — 2026-09-12 From 0a2330fde994963497843a539f1f21618fb27abb Mon Sep 17 00:00:00 2001 From: Muawiya Amir Date: Sat, 12 Sep 2026 18:38:29 +0500 Subject: [PATCH 16/59] docs: link database warm-up pull request --- docs/WORK_LOG.md | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/docs/WORK_LOG.md b/docs/WORK_LOG.md index 9ab1c0b..d42cbdf 100644 --- a/docs/WORK_LOG.md +++ b/docs/WORK_LOG.md @@ -9,7 +9,8 @@ claims as completed work. - Completed implementation: [#149](https://github.com/Coding-Moves/one-concept/issues/149), reducing database reconnect work on requests after idle time. Validated and - preparing the focused backend PR into `develop`. + published in [PR #184](https://github.com/Coding-Moves/one-concept/pull/184) + into `develop`. - Branch: `codex/149-db-connection-warmup`, from refreshed `origin/develop` at `f34ba77`. PR #183, including both review fixes for #133, is merged. - Delivered: idle-expiry reproduction, bounded/configurable warm-up with lifecycle @@ -60,7 +61,12 @@ claims as completed work. unverified. Cold startup and additional connections during bursts can still pay setup cost. Each API process adds a periodic probe with the configured interval. - Commits: `50d73ab` — reproduction and scope; `6bf0399` — implementation/tests; - documentation handoff — `docs: document database warm-up validation`. + `679163a` — validation and operational documentation. +- **PR:** [#184](https://github.com/Coding-Moves/one-concept/pull/184), open into + `develop` with all individual commits and the owner's configured identity. + Publication bookkeeping: `docs: link database warm-up pull request`. + Disposable reproduction/test containers were removed. Ready for owner review; + merging and deployment were not performed. ## PR #183 review fixes — 2026-09-12 From 64afb6d23bea3cfed08185fe902c25745fb09eed Mon Sep 17 00:00:00 2001 From: Muawiya Amir Date: Sat, 12 Sep 2026 19:24:30 +0500 Subject: [PATCH 17/59] docs: record bounded startup scope and reproduction --- docs/WORK_LOG.md | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/docs/WORK_LOG.md b/docs/WORK_LOG.md index d42cbdf..32d6867 100644 --- a/docs/WORK_LOG.md +++ b/docs/WORK_LOG.md @@ -7,6 +7,16 @@ claims as completed work. ## Current status +- Active scope: [#150](https://github.com/Coding-Moves/one-concept/issues/150), + the next open issue in ascending order. Branch `codex/150-bounded-startup-state` + starts at refreshed `origin/develop` (`a9aab63`); PR #184 is merged. +- Plan: reproduce growing state payloads; add capped startup metadata and cursor + endpoints; adapt mobile Stats/Saved without losing totals, search, or offline + data; validate and open one PR into `develop`. Keep the existing last-ten History + UI (#159 is separate). Older clients retain the legacy contract; updated JS + opts into compact state. No new APK, dependency, migration, or release planned. +- Read the exact Expo SDK 57 documentation before mobile changes. Each coherent + backend, mobile, and documentation change will retain its own commit/tests. - Completed implementation: [#149](https://github.com/Coding-Moves/one-concept/issues/149), reducing database reconnect work on requests after idle time. Validated and published in [PR #184](https://github.com/Coding-Moves/one-concept/pull/184) @@ -21,6 +31,16 @@ claims as completed work. - Release #179 and card follow-up #180 are merged; #181 synchronized `main` back into `develop`. This task does not authorize another production release. +## Bounded startup state (#150) — 2026-09-12 + +- Confirmed before implementation with disposable PostgreSQL 16 and 365 completed + and saved concepts: both the existing request and `?compact=true` returned all + 365 detail rows in each list, about 151 KB. The new regression failed at the + expected 50-row limit. No production service was used. +- Keep legacy state responses for older clients during backend/OTA rollout. The + updated client will request compact metadata, retain exact aggregate totals, + and load older Saved metadata in pages when needed. Full History UI is separate. + ## Database connections after idle (#149) — 2026-09-12 - Reproduced before implementation using the unchanged app engine/session and From 09f4b2292541f1063f0c3a6bf3eb7388a3763f32 Mon Sep 17 00:00:00 2001 From: Muawiya Amir Date: Sat, 12 Sep 2026 19:26:27 +0500 Subject: [PATCH 18/59] docs: prefer more focused commits per pull request --- AGENTS.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/AGENTS.md b/AGENTS.md index 2cdabb4..ad8690e 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -13,6 +13,9 @@ requires the exact Expo SDK 57 documentation before writing mobile code. - Deliver assigned work in coherent chunks, with one PR per chunk. - Give every small, meaningful change its own commit. Commit incrementally; do not wait until the end and put the entire task into one large commit. +- Prefer a higher number of focused commits per PR. Separate independently + reviewable behavior, UI, and documentation changes instead of bundling them; + keep regression tests with the behavior they verify. - Keep as many meaningful, atomic commits as the chunk naturally produces in its PR. There is no numeric maximum or minimum. Do not split a coherent change into broken fragments or make empty commits to inflate the count. From ce38299e4227e087b30f2105ef846a0997ab8fb9 Mon Sep 17 00:00:00 2001 From: Muawiya Amir Date: Sat, 12 Sep 2026 19:26:47 +0500 Subject: [PATCH 19/59] feat: bound opt-in startup metadata and paginate collections --- backend/app/api/v1/me.py | 47 ++++++-- backend/app/schemas/me.py | 15 +++ backend/app/services/collections.py | 100 ++++++++++++++++ backend/app/services/state.py | 41 +++++-- backend/tests/test_state_pagination.py | 157 +++++++++++++++++++++++++ 5 files changed, 345 insertions(+), 15 deletions(-) create mode 100644 backend/app/services/collections.py create mode 100644 backend/tests/test_state_pagination.py diff --git a/backend/app/api/v1/me.py b/backend/app/api/v1/me.py index 683fe85..b6c7e42 100644 --- a/backend/app/api/v1/me.py +++ b/backend/app/api/v1/me.py @@ -1,14 +1,18 @@ -from datetime import time +from datetime import date, time -from fastapi import APIRouter, Depends, HTTPException, status +from fastapi import APIRouter, Depends, HTTPException, Query, status from sqlalchemy import ARRAY, Time, bindparam, text from sqlalchemy.ext.asyncio import AsyncSession from app.db.session import get_db from app.deps import CurrentUser, get_current_user from app.schemas.daily import ConceptOut, DailyOut -from app.schemas.me import LearnedOut, ProfileIn, SavedConceptOut, StateOut, StreakOut, TopicsIn +from app.schemas.me import ( + HistoryPageOut, LearnedOut, ProfileIn, SavedConceptOut, SavedPageOut, StateOut, + StreakOut, TopicsIn, +) from app.schemas.notifications import NotificationPrefs, PushTokenIn +from app.services.collections import history_page, saved_page from app.services.interactions import set_followed_topics from app.services.selection import DailyResult, get_or_create_daily from app.services.state import load_state @@ -59,12 +63,16 @@ def _to_state_out(state) -> StateOut: for s in state.saved ], stats=StreakOut(**vars(state.stats)), + learned_before_window=state.learned_before_window, + history_next_cursor=state.history_next_cursor, + saved_next_cursor=state.saved_next_cursor, assignment_slug=state.assignment_slug, ) @router.get("/state", response_model=StateOut) async def get_state( + compact: bool = False, user: CurrentUser = Depends(get_current_user), db: AsyncSession = Depends(get_db), ) -> StateOut: @@ -73,11 +81,11 @@ async def get_state( Bootstrapping only runs when the state query finds no profile, so the common path costs a single round trip. """ - state = await load_state(db, user.id) + state = await load_state(db, user.id, compact=compact) if state is None: await ensure_bootstrapped(db, user.id, user.email) await db.commit() - state = await load_state(db, user.id) + state = await load_state(db, user.id, compact=compact) out = _to_state_out(state) # Fold today's concept in so the app needs one startup round trip (#102). # Same create-on-first-call behaviour as GET /v1/daily. @@ -85,6 +93,29 @@ async def get_state( return out +@router.get("/history", response_model=HistoryPageOut) +async def get_history( + cursor: date | None = None, + limit: int = Query(default=50, ge=1, le=100), + user: CurrentUser = Depends(get_current_user), + db: AsyncSession = Depends(get_db), +) -> HistoryPageOut: + return await history_page(db, user.id, cursor, limit) + + +@router.get("/saved", response_model=SavedPageOut) +async def get_saved( + cursor: str | None = Query(default=None, max_length=200), + limit: int = Query(default=50, ge=1, le=100), + user: CurrentUser = Depends(get_current_user), + db: AsyncSession = Depends(get_db), +) -> SavedPageOut: + try: + return await saved_page(db, user.id, cursor, limit) + except ValueError as exc: + raise HTTPException(status.HTTP_400_BAD_REQUEST, detail="Invalid saved cursor") from exc + + @router.get("/stats", response_model=StreakOut) async def get_stats( user: CurrentUser = Depends(get_current_user), @@ -96,11 +127,12 @@ async def get_stats( @router.put("/topics", response_model=StateOut) async def put_topics( body: TopicsIn, + compact: bool = False, user: CurrentUser = Depends(get_current_user), db: AsyncSession = Depends(get_db), ) -> StateOut: await set_followed_topics(db, user.id, body.topics) - return _to_state_out(await load_state(db, user.id)) + return _to_state_out(await load_state(db, user.id, compact=compact)) @router.post("/push-token", status_code=status.HTTP_204_NO_CONTENT) @@ -217,6 +249,7 @@ async def put_notifications( @router.patch("", response_model=StateOut) async def patch_profile( body: ProfileIn, + compact: bool = False, user: CurrentUser = Depends(get_current_user), db: AsyncSession = Depends(get_db), ) -> StateOut: @@ -244,4 +277,4 @@ async def patch_profile( {"n": body.display_name, "uid": user.id}, ) await db.commit() - return _to_state_out(await load_state(db, user.id)) + return _to_state_out(await load_state(db, user.id, compact=compact)) diff --git a/backend/app/schemas/me.py b/backend/app/schemas/me.py index ecbbbbf..ab1553e 100644 --- a/backend/app/schemas/me.py +++ b/backend/app/schemas/me.py @@ -28,6 +28,16 @@ class LearnedOut(BaseModel): like_count: int = 0 +class HistoryPageOut(BaseModel): + items: list[LearnedOut] + next_cursor: str | None = None + + +class SavedPageOut(BaseModel): + items: list[SavedConceptOut] + next_cursor: str | None = None + + class StateOut(BaseModel): display_name: str | None = None timezone: str @@ -38,6 +48,11 @@ class StateOut(BaseModel): bookmarks: list[str] saved: list[SavedConceptOut] = Field(default_factory=list) stats: StreakOut + # Present for compact clients. Counts exclude the embedded recent window, + # so optimistic/offline completions can still be added by the client. + learned_before_window: dict[str, int] | None = None + history_next_cursor: str | None = None + saved_next_cursor: str | None = None assignment_slug: str | None = None # Today's concept, folded in so the app needs a single round trip at startup # (issue #102). Null when the catalog is exhausted for this user. This GET diff --git a/backend/app/services/collections.py b/backend/app/services/collections.py new file mode 100644 index 0000000..73fcb94 --- /dev/null +++ b/backend/app/services/collections.py @@ -0,0 +1,100 @@ +"""Bounded collection reads; cursor ordering survives deletions between pages.""" + +import base64 +import binascii +import json +import uuid +from datetime import date, datetime + +from sqlalchemy import text +from sqlalchemy.ext.asyncio import AsyncSession + +from app.schemas.me import HistoryPageOut, LearnedOut, SavedConceptOut, SavedPageOut + +STATE_WINDOW = 50 + + +def saved_cursor(saved_at: str, concept_id: str) -> str: + return base64.urlsafe_b64encode(json.dumps([saved_at, concept_id]).encode()).decode().rstrip("=") + + +def parse_saved_cursor(cursor: str | None) -> tuple[datetime | None, uuid.UUID | None]: + if cursor is None: + return None, None + try: + timestamp, concept_id = json.loads(base64.b64decode( + cursor + "=" * (-len(cursor) % 4), altchars=b"-_", validate=True, + )) + if not isinstance(timestamp, str) or not isinstance(concept_id, str): + raise ValueError("Cursor fields must be strings") + at = datetime.fromisoformat(timestamp) + if at.tzinfo is None: + raise ValueError("Timestamp must include a timezone") + return at, uuid.UUID(concept_id) + except (ValueError, TypeError, binascii.Error) as exc: + raise ValueError("Invalid saved cursor") from exc + + +_HISTORY_PAGE = text(""" + with page as materialized ( + select concept_id, assigned_for + from public.daily_assignments + where user_id = :uid and completed_at is not null + and (cast(:before as date) is null or assigned_for < :before) + order by assigned_for desc limit :take + ) + select c.slug as concept_slug, p.assigned_for as learned_on, + c.title, t.name as topic_name, + (select count(*) from public.concept_interactions ci + where ci.concept_id = p.concept_id and ci.liked_at is not null + and ci.user_id <> :uid)::int as like_count + from page p join public.concepts c on c.id = p.concept_id + join public.topics t on t.id = c.topic_id + order by p.assigned_for desc +""") + +_SAVED_PAGE = text(""" + with page as materialized ( + select concept_id, saved_at + from public.concept_interactions + where user_id = :uid and saved_at is not null + and (cast(:before as timestamptz) is null + or (saved_at, concept_id) < (:before, cast(:id as uuid))) + order by saved_at desc, concept_id desc limit :take + ) + select c.slug as concept_slug, c.title, t.name as topic_name, p.saved_at, p.concept_id, + (select count(*) from public.concept_interactions ci + where ci.concept_id = p.concept_id and ci.liked_at is not null + and ci.user_id <> :uid)::int as like_count + from page p join public.concepts c on c.id = p.concept_id + join public.topics t on t.id = c.topic_id + order by p.saved_at desc, p.concept_id desc +""") + + +async def history_page( + db: AsyncSession, user_id: uuid.UUID, before: date | None, limit: int, +) -> HistoryPageOut: + rows = (await db.execute(_HISTORY_PAGE, { + "uid": user_id, "before": before, "take": limit + 1, + })).mappings().all() + items = [LearnedOut(**r) for r in rows[:limit]] + return HistoryPageOut( + items=items, + next_cursor=items[-1].learned_on.isoformat() if len(rows) > limit else None, + ) + + +async def saved_page( + db: AsyncSession, user_id: uuid.UUID, cursor: str | None, limit: int, +) -> SavedPageOut: + before, concept_id = parse_saved_cursor(cursor) + rows = (await db.execute(_SAVED_PAGE, { + "uid": user_id, "before": before, "id": concept_id, "take": limit + 1, + })).mappings().all() + return SavedPageOut( + items=[SavedConceptOut(**r) for r in rows[:limit]], + next_cursor=(saved_cursor(rows[limit - 1]["saved_at"].isoformat(), + str(rows[limit - 1]["concept_id"])) + if len(rows) > limit else None), + ) diff --git a/backend/app/services/state.py b/backend/app/services/state.py index 675ee0b..3a6945b 100644 --- a/backend/app/services/state.py +++ b/backend/app/services/state.py @@ -13,6 +13,7 @@ from sqlalchemy import text from sqlalchemy.ext.asyncio import AsyncSession +from app.services.collections import STATE_WINDOW, saved_cursor from app.services.streaks import StreakStats @@ -46,6 +47,9 @@ class UserState: # user's 20 concepts). `bookmarks` stays as bare slugs for membership counts. saved: list[SavedConcept] stats: StreakStats + learned_before_window: dict[str, int] | None = None + history_next_cursor: str | None = None + saved_next_cursor: str | None = None assignment_slug: str | None = None display_name: str | None = None @@ -68,14 +72,22 @@ class UserState: join public.topics t on t.id = c.topic_id where a.user_id = :uid and a.completed_at is not null ), + recent_learned_rows as materialized ( + select * from learned_rows order by assigned_for desc limit :window_limit + ), + older_counts as ( + select topic_name, count(*)::int as n from learned_rows + where assigned_for < (select min(assigned_for) from recent_learned_rows) + group by topic_name + ), learned as ( select coalesce(json_agg(json_build_object( 'slug', slug, 'title', title, 'topic', topic_name, 'on', assigned_for, 'likes', (select count(*) from public.concept_interactions ci - where ci.concept_id = learned_rows.concept_id + where ci.concept_id = recent_learned_rows.concept_id and ci.liked_at is not null and ci.user_id <> :uid)::int) order by assigned_for desc), '[]'::json) as v - from learned_rows + from recent_learned_rows ), interactions as ( select @@ -85,18 +97,22 @@ class UserState: join public.concepts c on c.id = i.concept_id where i.user_id = :uid ), + saved_rows as materialized ( + select concept_id, saved_at from public.concept_interactions + where user_id = :uid and saved_at is not null + order by saved_at desc, concept_id desc limit :window_limit + ), saved as ( select coalesce(json_agg(json_build_object( 'slug', c.slug, 'title', c.title, 'topic', t.name, + 'at', i.saved_at, 'id', i.concept_id, 'likes', (select count(*) from public.concept_interactions ci where ci.concept_id = c.id and ci.liked_at is not null and ci.user_id <> :uid)::int) - order by i.saved_at desc) filter (where i.saved_at is not null), - '[]'::json) as v - from public.concept_interactions i + order by i.saved_at desc, i.concept_id desc), '[]'::json) as v + from saved_rows i join public.concepts c on c.id = i.concept_id join public.topics t on t.id = c.topic_id - where i.user_id = :uid ), assignment as ( select c.slug @@ -114,6 +130,8 @@ class UserState: prof.today, followed.v as followed_topics, learned.v as learned, + coalesce((select json_object_agg(topic_name, n) from older_counts), + '{}'::json) as learned_before_window, interactions.likes, interactions.saves, saved.v as saved, @@ -127,10 +145,12 @@ class UserState: """) -async def load_state(session: AsyncSession, user_id: uuid.UUID) -> UserState | None: +async def load_state(session: AsyncSession, user_id: uuid.UUID, *, compact: bool = False) -> UserState | None: """Returns None when the user has no profile row yet, so the caller can bootstrap and retry — keeping the common path to a single query.""" - row = (await session.execute(_STATE, {"uid": user_id})).first() + row = (await session.execute(_STATE, { + "uid": user_id, "window_limit": STATE_WINDOW if compact else None, + })).first() if row is None: return None @@ -165,5 +185,10 @@ async def load_state(session: AsyncSession, user_id: uuid.UUID) -> UserState | N longest=row.longest_streak, total_learned=row.total_learned, ), + learned_before_window=dict(row.learned_before_window) if compact else None, + history_next_cursor=(row.learned[-1]["on"] + if compact and row.total_learned > len(row.learned) else None), + saved_next_cursor=(saved_cursor(row.saved[-1]["at"], row.saved[-1]["id"]) + if compact and len(row.saves) > len(row.saved) else None), assignment_slug=row.assignment_slug, ) diff --git a/backend/tests/test_state_pagination.py b/backend/tests/test_state_pagination.py new file mode 100644 index 0000000..eb63eed --- /dev/null +++ b/backend/tests/test_state_pagination.py @@ -0,0 +1,157 @@ +"""Large accounts retain totals and full collection access with compact startup.""" + +import uuid + +import pytest_asyncio +from httpx import ASGITransport, AsyncClient +from sqlalchemy import text + +from app.db.session import get_db +from app.deps import CurrentUser, get_current_user +from app.main import app + + +@pytest_asyncio.fixture +async def collection_client(session, sessionmaker_for_test, user): + prefix = f"collection-{user}-" + await session.execute(text(""" + insert into public.concepts (topic_id, slug, title, summary) + select (select id from public.topics where slug = 'computer-science'), + :prefix || n, 'Lesson ' || n, 'Fixture summary' + from generate_series(1, 365) n + """), {"prefix": prefix}) + await session.execute(text(""" + insert into public.daily_assignments (user_id, concept_id, assigned_for, completed_at) + select :uid, id, current_date - cast(substring(slug from length(:prefix) + 1) as int), now() + from public.concepts where starts_with(slug, :prefix) + """), {"prefix": prefix, "uid": user}) + await session.execute(text(""" + insert into public.concept_interactions (user_id, concept_id, liked_at, saved_at) + select :uid, id, now(), now() + from public.concepts where starts_with(slug, :prefix) + """), {"prefix": prefix, "uid": user}) + await session.commit() + + async def db(): + async with sessionmaker_for_test() as value: + yield value + + app.dependency_overrides[get_db] = db + app.dependency_overrides[get_current_user] = lambda: CurrentUser(id=user, email="fixture@example.invalid") + try: + async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as client: + yield client + finally: + app.dependency_overrides.clear() + + +async def test_compact_state_bounds_details_without_losing_totals(collection_client): + legacy = await collection_client.get("/v1/me/state") + compact = await collection_client.get("/v1/me/state?compact=true") + assert legacy.status_code == compact.status_code == 200 + before, after = legacy.json(), compact.json() + print(f"state bytes: legacy={len(legacy.content)}, compact={len(compact.content)}; " + f"detail rows: learned={len(after['learned'])}, saved={len(after['saved'])}") + assert len(before["learned"]) == len(before["saved"]) == 365 + assert len(after["learned"]) == len(after["saved"]) == 50 + assert len(after["likes"]) == len(after["bookmarks"]) == 365 + assert after["stats"] == before["stats"] == {"current": 365, "longest": 365, "total_learned": 365} + assert after["learned_before_window"] == {"Computer Science": 315} + assert after["history_next_cursor"] and after["saved_next_cursor"] + assert len(compact.content) < len(legacy.content) / 2 + + +async def test_pages_cover_history_and_tied_saves(collection_client): + state = (await collection_client.get('/v1/me/state?compact=true')).json() + for path, field, cursor_field in ( + ('history', 'learned', 'history_next_cursor'), + ('saved', 'saved', 'saved_next_cursor'), + ): + items = state[field][:] + cursor = state[cursor_field] + while cursor: + response = await collection_client.get(f'/v1/me/{path}', params={'cursor': cursor, 'limit': 37}) + assert response.status_code == 200, response.text + page = response.json() + assert 0 < len(page['items']) <= 37 + items.extend(page['items']) + cursor = page['next_cursor'] + assert len(items) == len({item['concept_slug'] for item in items}) == 365 + assert all(item['like_count'] == 0 for item in items) # own likes excluded + full = (await collection_client.get('/v1/me/state')).json()[field] + assert items == full + + +async def test_state_mutations_support_compact_clients(collection_client): + for response in ( + await collection_client.put('/v1/me/topics?compact=true', json={'topics': ['computer-science']}), + await collection_client.patch('/v1/me?compact=true', json={'display_name': 'Updated'}), + ): + assert response.status_code == 200, response.text + assert len(response.json()['learned']) == len(response.json()['saved']) == 50 + assert response.json()['learned_before_window'] == {'Computer Science': 315} + legacy = await collection_client.patch('/v1/me', json={'display_name': 'Legacy'}) + assert len(legacy.json()['learned']) == len(legacy.json()['saved']) == 365 + + +async def test_empty_and_other_user_collections(collection_client, session, user): + from app.services.collections import saved_cursor + from datetime import datetime, timezone + + # A cursor contains ordering only; it cannot choose which user's rows to read. + other = uuid.uuid4() + app.dependency_overrides[get_current_user] = lambda: CurrentUser(id=other, email='other@example.invalid') + for path in ('saved', 'history'): + result = await collection_client.get(f'/v1/me/{path}') + assert result.json() == {'items': [], 'next_cursor': None} + cursor = saved_cursor(datetime.now(timezone.utc).isoformat(), str(user)) + assert (await collection_client.get('/v1/me/saved', params={'cursor': cursor})).json()['items'] == [] + + +async def test_cursor_validation_and_limits(collection_client): + import base64 + import json + + for cursor in ('not-a-cursor', base64.urlsafe_b64encode(json.dumps([ + '2026-01-01T00:00:00', str(uuid.uuid4()), + ]).encode()).decode(), base64.urlsafe_b64encode(b'[42, 42]').decode()): + result = await collection_client.get('/v1/me/saved', params={'cursor': cursor}) + assert result.status_code == 400 + assert (await collection_client.get('/v1/me/history?cursor=bad')).status_code == 422 + for path in ('saved', 'history'): + for limit in (0, 101): + assert (await collection_client.get(f'/v1/me/{path}?limit={limit}')).status_code == 422 + response = await collection_client.get(f'/v1/me/{path}?limit=100') + assert len(response.json()['items']) == 100 + + +async def test_saved_deletion_between_pages_does_not_skip_rows(collection_client, session, user): + first = (await collection_client.get('/v1/me/saved?limit=10')).json() + await session.execute(text(''' + update public.concept_interactions set saved_at = null + where user_id = :uid and concept_id = ( + select id from public.concepts where slug = :slug + ) + '''), {'uid': user, 'slug': first['items'][-1]['concept_slug']}) + await session.commit() + second = (await collection_client.get('/v1/me/saved', params={ + 'cursor': first['next_cursor'], 'limit': 100, + })).json() + all_remaining = (await collection_client.get('/v1/me/state')).json()['saved'] + assert second['items'] == all_remaining[9:109] + + +async def test_exact_window_has_no_cursor(collection_client, session, user): + await session.execute(text(''' + delete from public.daily_assignments where user_id = :uid + and assigned_for < current_date - 50 + '''), {'uid': user}) + await session.execute(text(''' + update public.concept_interactions set saved_at = null where user_id = :uid + and concept_id not in (select concept_id from public.daily_assignments where user_id = :uid) + '''), {'uid': user}) + await session.commit() + state = (await collection_client.get('/v1/me/state?compact=true')).json() + assert len(state['learned']) == len(state['saved']) == 50 + assert state['learned_before_window'] == {} + assert state['history_next_cursor'] is state['saved_next_cursor'] is None From 32cc70a73f21c2f9fb1d265dcc81d60b7466305c Mon Sep 17 00:00:00 2001 From: Muawiya Amir Date: Sat, 12 Sep 2026 19:28:14 +0500 Subject: [PATCH 20/59] fix: preserve full learning totals with recent state windows --- mobile/src/screens/StatsScreen.tsx | 17 +++++------ mobile/src/services/progressTotals.ts | 15 ++++++++++ .../src/services/remoteProgressRepository.ts | 4 +++ mobile/src/types/index.ts | 4 +++ mobile/tests/progressTotals.test.mjs | 28 +++++++++++++++++++ 5 files changed, 58 insertions(+), 10 deletions(-) create mode 100644 mobile/src/services/progressTotals.ts create mode 100644 mobile/tests/progressTotals.test.mjs diff --git a/mobile/src/screens/StatsScreen.tsx b/mobile/src/screens/StatsScreen.tsx index 96fc70a..6154ebf 100644 --- a/mobile/src/screens/StatsScreen.tsx +++ b/mobile/src/screens/StatsScreen.tsx @@ -12,6 +12,7 @@ import { CONCEPTS } from '../data/concepts'; import { ServerTopic } from '../services/topicsApi'; import { scaleFont, radius, spacing, ThemeColors, typography } from '../theme'; import { LearnedRecord } from '../types'; +import { learnedTopicCounts } from '../services/progressTotals'; interface CategoryProgress { label: string; @@ -40,14 +41,10 @@ function demoCategoryProgress(learnedIds: Set): CategoryProgress[] { * bars reflect the 125+ real catalog, not the demo's 20 (issue #35). */ function serverCategoryProgress( topics: ServerTopic[], - learned: LearnedRecord[] + learned: LearnedRecord[], + beforeWindow?: Record, ): CategoryProgress[] { - const learnedByTopic = new Map(); - for (const record of learned) { - if (record.topicName) { - learnedByTopic.set(record.topicName, (learnedByTopic.get(record.topicName) ?? 0) + 1); - } - } + const learnedByTopic = learnedTopicCounts(learned, beforeWindow); return topics.map((t) => ({ label: t.name, total: t.conceptCount, @@ -80,13 +77,13 @@ export function StatsScreen() { const categories = useMemo( () => serverMode - ? serverCategoryProgress(topics, progress.learned) + ? serverCategoryProgress(topics, progress.learned, progress.learnedBeforeWindow) : demoCategoryProgress(new Set(progress.learned.map((r) => r.conceptId))), - [serverMode, topics, progress.learned] + [serverMode, topics, progress.learned, progress.learnedBeforeWindow] ); const overall = serverMode ? { - learned: progress.learned.length, + learned: progress.stats?.totalLearned ?? progress.learned.length, total: topics.reduce((sum, t) => sum + t.conceptCount, 0), } : { diff --git a/mobile/src/services/progressTotals.ts b/mobile/src/services/progressTotals.ts new file mode 100644 index 0000000..18ecd4a --- /dev/null +++ b/mobile/src/services/progressTotals.ts @@ -0,0 +1,15 @@ +import type { LearnedRecord } from '../types'; + +/** Older aggregate counts plus the recent records (including offline completions). */ +export function learnedTopicCounts( + learned: LearnedRecord[], + beforeWindow: Record = {}, +): Map { + const counts = new Map(Object.entries(beforeWindow)); + for (const record of learned) { + if (record.topicName) { + counts.set(record.topicName, (counts.get(record.topicName) ?? 0) + 1); + } + } + return counts; +} diff --git a/mobile/src/services/remoteProgressRepository.ts b/mobile/src/services/remoteProgressRepository.ts index f9e0d6b..1b1c044 100644 --- a/mobile/src/services/remoteProgressRepository.ts +++ b/mobile/src/services/remoteProgressRepository.ts @@ -32,6 +32,8 @@ interface StatePayload { likes: string[]; bookmarks: string[]; saved?: { concept_slug: string; title?: string; topic_name?: string; like_count?: number }[]; + learned_before_window?: Record | null; + saved_next_cursor?: string | null; stats: { current: number; longest: number; total_learned: number }; assignment_slug: string | null; daily?: DailyPayload | null; @@ -60,6 +62,8 @@ function toProgressState(payload: StatePayload): ProgressState { topicName: s.topic_name || '', likeCount: s.like_count ?? 0, })), + learnedBeforeWindow: payload.learned_before_window ?? undefined, + savedNextCursor: payload.saved_next_cursor, // Server-computed, so the day boundary comes from the user's stored // timezone rather than whatever the device clock happens to say. stats: { diff --git a/mobile/src/types/index.ts b/mobile/src/types/index.ts index b51c2d1..f1ddeef 100644 --- a/mobile/src/types/index.ts +++ b/mobile/src/types/index.ts @@ -103,6 +103,10 @@ export interface ProgressState { * which resolves saved items against the bundled catalog instead. */ savedConcepts?: SavedConcept[]; + /** Counts older than the recent learned window; absent on legacy responses. */ + learnedBeforeWindow?: Record; + /** Continuation after the embedded saved window; absent on legacy responses. */ + savedNextCursor?: string | null; /** * Streaks as computed by the server, when the state came from the server. * Absent for purely local state, where the client derives them instead. diff --git a/mobile/tests/progressTotals.test.mjs b/mobile/tests/progressTotals.test.mjs new file mode 100644 index 0000000..e9f9933 --- /dev/null +++ b/mobile/tests/progressTotals.test.mjs @@ -0,0 +1,28 @@ +import assert from 'node:assert/strict'; +import { test } from 'node:test'; +import { learnedTopicCounts } from '../src/services/progressTotals.ts'; +import { withPendingProgress } from '../src/services/pendingProgress.ts'; + +test('compact totals include older topics and an offline completion exactly once', () => { + const record = {conceptId:'today', date:'2026-09-12', topicName:'Computer Science'}; + const server = { + learned:[{conceptId:'yesterday', date:'2026-09-11', topicName:'Computer Science'}], + learnedBeforeWindow:{'Computer Science':314, Mathematics:50}, + assignment:{conceptId:'today',date:'2026-09-12'}, likes:[],bookmarks:[],followedTopics:[], + stats:{current:1,longest:1,totalLearned:365}, + }; + const previous = {...server, learned:[...server.learned,record], + stats:{...server.stats,totalLearned:366}}; + const pending = [{kind:'learn',date:'2026-09-12'}]; + const once = withPendingProgress(server, previous, pending); + const twice = withPendingProgress(once, previous, pending); + assert.deepEqual(learnedTopicCounts(twice.learned,twice.learnedBeforeWindow), + new Map([['Computer Science',316],['Mathematics',50]])); + assert.equal(twice.stats.totalLearned,366); + assert.deepEqual(server.learnedBeforeWindow,{'Computer Science':314,Mathematics:50}); +}); + +test('legacy state still derives categories from its complete learned list', () => { + assert.deepEqual(learnedTopicCounts([{topicName:'Mathematics'},{topicName:'Mathematics'},{}]), + new Map([['Mathematics',2]])); +}); From b62ab19464af64002481d871a2a95e470f8fa38c Mon Sep 17 00:00:00 2001 From: Muawiya Amir Date: Sat, 12 Sep 2026 19:31:58 +0500 Subject: [PATCH 21/59] feat: load older saved concepts with offline collection caching --- mobile/src/hooks/useSavedConcepts.ts | 93 ++++++++++++++++++++++++ mobile/src/screens/SavedScreen.tsx | 31 ++++++-- mobile/src/services/accountCaches.ts | 2 + mobile/src/services/savedApi.ts | 21 ++++++ mobile/tests/offline.browser.cjs | 103 ++++++++++++++++++++++++++- 5 files changed, 241 insertions(+), 9 deletions(-) create mode 100644 mobile/src/hooks/useSavedConcepts.ts create mode 100644 mobile/src/services/savedApi.ts diff --git a/mobile/src/hooks/useSavedConcepts.ts b/mobile/src/hooks/useSavedConcepts.ts new file mode 100644 index 0000000..3b94abb --- /dev/null +++ b/mobile/src/hooks/useSavedConcepts.ts @@ -0,0 +1,93 @@ +import { useCallback, useEffect, useMemo, useState } from 'react'; +import { getConnectivity } from '../api/client'; +import { useAuth } from '../context/AuthContext'; +import { conceptCache } from '../services/conceptApi'; +import { fetchSavedPage, savedCollectionCache } from '../services/savedApi'; +import { ProgressState, SavedConcept } from '../types'; + +/** Fetch older metadata only while Saved is open. Recent rows paint immediately; + * cached pages and already-downloaded lesson bodies preserve offline search. */ +export function useSavedConcepts(progress: ProgressState) { + const { session } = useAuth(); + const userId = session?.user.id ?? null; + const [attempt, setAttempt] = useState(0); + const retry = useCallback(() => setAttempt(n => n + 1), []); + const [loaded, setLoaded] = useState<{ + owner: string | null; items: SavedConcept[]; loading: boolean; failed: boolean; + }>({ owner: null, items: [], loading: false, failed: false }); + // Equivalent state refreshes must not restart a failed page fetch in a loop. + const key = JSON.stringify([progress.bookmarks, progress.savedConcepts, progress.savedNextCursor]); + const snapshot = useMemo(() => ({ + bookmarks: progress.bookmarks, recent: progress.savedConcepts ?? [], cursor: progress.savedNextCursor, + }), [key]); + + useEffect(() => { + if (!userId) return; + let cancelled = false; + const epoch = savedCollectionCache.epoch; + const contentEpoch = conceptCache.epoch; + const active = () => !cancelled && epoch === savedCollectionCache.epoch; + const membership = new Set(snapshot.bookmarks); + const rows = new Map(); + const recent = new Map(snapshot.recent.map(s => [s.conceptId, s])); + const items = () => [...new Map([...recent, ...[...rows].map(([id, row]) => + [id, recent.get(id) ?? row] as const)]).values()].filter(s => membership.has(s.conceptId)); + const publish = (loading: boolean, failed = false) => { + if (active()) setLoaded({ owner: userId, items: items(), loading, failed }); + }; + const persist = () => active() + ? savedCollectionCache.set(userId, items(), epoch).catch(() => {}) : Promise.resolve(); + + publish(true); + void (async () => { + const cached = await savedCollectionCache.get(userId, epoch); + if (!active()) return; + cached?.forEach(s => rows.set(s.conceptId, s)); + publish(!!snapshot.cursor); + let failed = false; + let cursor = snapshot.cursor; + const seen = new Set(); + try { + while (cursor && active() && getConnectivity()) { + if (seen.has(cursor)) throw new Error('Repeated saved cursor'); + seen.add(cursor); + const page = await fetchSavedPage(cursor); + if (!active()) return; + page.items.forEach(s => rows.set(s.conceptId, s)); + cursor = page.nextCursor; + publish(!!cursor); + await persist(); + } + } catch { + failed = true; + } + if (!active()) return; + // Offline users may have downloaded all saved bodies without ever opening + // this screen. Recover their metadata even if the list cache is absent. + const missing = snapshot.bookmarks.filter(id => !rows.has(id) && !recent.has(id)); + for (let offset = 0; offset < missing.length && active(); offset += 20) { + const concepts = await Promise.all(missing.slice(offset, offset + 20) + .map(id => conceptCache.get(id, contentEpoch))); + if (!active()) return; + concepts.forEach(c => { + if (c) rows.set(c.id, { conceptId: c.id, title: c.title, topicName: c.category, likeCount: c.likeCount }); + }); + } + publish(false, failed || items().length < membership.size); + await persist(); + })(); + return () => { cancelled = true; }; + }, [userId, snapshot, attempt]); + + const membership = new Set(progress.bookmarks); + const rows = new Map((loaded.owner === userId ? loaded.items : []).map(s => [s.conceptId, s])); + // Apply optimistic saves/unsaves immediately, before any in-flight page ends. + progress.savedConcepts?.forEach(s => rows.set(s.conceptId, s)); + return { + savedConcepts: progress.savedConcepts === undefined ? undefined + : [...rows.values()].filter(s => membership.has(s.conceptId)), + loading: !!userId && loaded.owner === userId && loaded.loading, + failed: !!userId && loaded.owner === userId && loaded.failed, + retry, + }; +} diff --git a/mobile/src/screens/SavedScreen.tsx b/mobile/src/screens/SavedScreen.tsx index fcbf9ce..086feb0 100644 --- a/mobile/src/screens/SavedScreen.tsx +++ b/mobile/src/screens/SavedScreen.tsx @@ -2,12 +2,13 @@ import { Ionicons } from '@expo/vector-icons'; import { CompositeNavigationProp, useNavigation } from '@react-navigation/native'; import { NativeStackNavigationProp } from '@react-navigation/native-stack'; import { useMemo, useState } from 'react'; -import { FlatList, Pressable, StyleSheet, Text, TextInput, View } from 'react-native'; +import { ActivityIndicator, FlatList, Pressable, StyleSheet, Text, TextInput, View } from 'react-native'; import { CategoryChip } from '../components/CategoryChip'; import { LikeCount } from '../components/LikeCount'; import { UnavailableState } from '../components/UnavailableState'; import { useOnline } from '../context/ConnectivityContext'; import { useProgress } from '../context/ProgressContext'; +import { useSavedConcepts } from '../hooks/useSavedConcepts'; import { useTheme } from '../context/ThemeContext'; import { CONCEPTS_BY_ID } from '../data/concepts'; import { RootStackParamList } from '../navigation'; @@ -34,6 +35,7 @@ export function SavedScreen() { const navigation = useNavigation