From 48f5051775c3a1dd6823b9bde94a522b4685c688 Mon Sep 17 00:00:00 2001 From: Arjun Krishna Date: Sat, 22 Aug 2026 10:12:08 +0530 Subject: [PATCH 01/34] fix(migrate): apply each D1 migration as a single transaction MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Statements ran one at a time, so a failure part-way left the schema half-applied and the retry restarted from the first statement. That is only safe while every statement is idempotent — the first ALTER TABLE would turn a partial failure into a permanent failure loop on an instance nobody can see. db.batch() is a real transaction. The applied_at write rides inside it, so d1_migrations can never record a migration that did not commit. --- worker/src/platform/db/auto-migrate.ts | 33 +++++++++++--------------- 1 file changed, 14 insertions(+), 19 deletions(-) diff --git a/worker/src/platform/db/auto-migrate.ts b/worker/src/platform/db/auto-migrate.ts index e857f48..307f49d 100644 --- a/worker/src/platform/db/auto-migrate.ts +++ b/worker/src/platform/db/auto-migrate.ts @@ -1,17 +1,13 @@ -// Auto-migrator. On the first request after a deploy, applies any bundled -// migrations (worker/src/db/migrations/*.sql, baked into migrations.gen.ts) that -// haven't run yet, tracked in the d1_migrations table. +// Auto-migrator. On the first request after a deploy, applies bundled migrations +// that haven't run yet, tracked in d1_migrations. // -// nodrix is pre-alpha: there's a single baseline migration and instances are -// recreated fresh, so there's no legacy-baseline handling — just "apply what's -// missing, in order." Concurrent isolates are guarded by an INSERT OR IGNORE -// claim; a crash mid-migration leaves applied_at=0, which the next run retries. +// One db.batch() per migration: D1 runs it as a single transaction, so a failure +// part-way leaves the database untouched. exec() — which D1's migration docs +// recommend — stops on error without rolling back. import type { D1Database } from '@cloudflare/workers-types'; import { MIGRATIONS } from './migrations.gen'; -// Module-level guard: at most one migration check per Worker isolate. If it -// failed we null it out so the next request retries. let inFlight: Promise | null = null; export function ensureMigrated(db: D1Database): Promise { @@ -40,8 +36,8 @@ async function applyMigrations(db: D1Database): Promise { .all<{ name: string; applied_at: number }>(); const applied = new Set(res.results.filter((r) => r.applied_at > 0).map((r) => r.name)); - // Recover claims that never completed (worker crashed mid-migration) so the - // loop below retries them from scratch. + // applied_at = 0 means the batch rolled back; releasing the claim is the + // whole of the recovery. for (const r of res.results) { if (r.applied_at === 0) { await db.prepare(`DELETE FROM d1_migrations WHERE name = ? AND applied_at = 0`).bind(r.name).run(); @@ -51,8 +47,7 @@ async function applyMigrations(db: D1Database): Promise { for (const m of MIGRATIONS) { if (applied.has(m.name)) continue; - // Claim first — INSERT OR IGNORE keeps two concurrent isolates from running - // the same DDL twice (second one's changes count is 0, so it skips). + // INSERT OR IGNORE keeps two concurrent isolates from running the same DDL. const claim = await db .prepare(`INSERT OR IGNORE INTO d1_migrations (name, applied_at) VALUES (?, 0)`) .bind(m.name) @@ -61,14 +56,14 @@ async function applyMigrations(db: D1Database): Promise { console.log(`[migrate] applying ${m.name} (${m.statements.length} statements)`); try { - for (const stmt of m.statements) await db.prepare(stmt).run(); - await db - .prepare(`UPDATE d1_migrations SET applied_at = ? WHERE name = ?`) - .bind(Math.floor(Date.now() / 1000), m.name) - .run(); + await db.batch([ + ...m.statements.map((s) => db.prepare(s)), + db + .prepare(`UPDATE d1_migrations SET applied_at = ? WHERE name = ?`) + .bind(Math.floor(Date.now() / 1000), m.name), + ]); console.log(`[migrate] applied ${m.name}`); } catch (e) { - // Roll back the claim so we retry on a later request. await db.prepare(`DELETE FROM d1_migrations WHERE name = ? AND applied_at = 0`).bind(m.name).run(); throw e; } From f55debc593d7ebcdb1aa042c53504504da800be3 Mon Sep 17 00:00:00 2001 From: Arjun Krishna Date: Sat, 22 Aug 2026 10:12:14 +0530 Subject: [PATCH 02/34] feat(do): version Durable Object schemas and migrate them atomically Durable Objects get no migration runner, so each class now carries an ordered list of schema changes and walks itself forward inside transactionSync() the first time it wakes after a deploy. One that throws rolls back rather than stranding storage between versions, and the constructor runs before any method, so no caller sees a half-migrated object. The baseline is the existing schema unchanged, so an object that predates versioning replays it as a no-op and converges on the version a fresh object reaches. SchedulerDO is key-value only and needs none of this. --- .../platform/durable-objects/dashboard-do.ts | 21 ++-- .../platform/durable-objects/project-do.ts | 103 +++++++++--------- worker/src/platform/durable-objects/schema.ts | 15 +++ 3 files changed, 79 insertions(+), 60 deletions(-) create mode 100644 worker/src/platform/durable-objects/schema.ts diff --git a/worker/src/platform/durable-objects/dashboard-do.ts b/worker/src/platform/durable-objects/dashboard-do.ts index d7f4818..84c363a 100644 --- a/worker/src/platform/durable-objects/dashboard-do.ts +++ b/worker/src/platform/durable-objects/dashboard-do.ts @@ -5,6 +5,7 @@ import { validateLayout, variablesFromLayout, chartVariablesFromLayout, type Lay import { newId } from '../lib/ids'; import { userCanAccessProject } from '../lib/roles'; import type { CompactSeries } from '../lib/series'; +import { migrateSchema, type SchemaStep } from './schema'; // Cap on points per chart series in the bootstrap snapshot (mirrors the public // /state full snapshot). Dense ingest is stride-sampled to this. @@ -41,13 +42,23 @@ type AckMsg = { type: 'ack'; req: string; ok: boolean; reason?: string }; type ClientMsg = | { type: 'control'; req?: string; variable: string; value?: unknown }; +const SCHEMA: SchemaStep[] = [ + (sql) => { + sql.exec(` + CREATE TABLE IF NOT EXISTS subscribed_project ( + project_id TEXT PRIMARY KEY + ); + `); + }, +]; + export class DashboardDO extends DurableObject { private sql: SqlStorage; constructor(ctx: DurableObjectState, env: Env) { super(ctx, env); this.sql = ctx.storage.sql; - this.initSchema(); + migrateSchema(ctx, SCHEMA); } override async fetch(request: Request): Promise { @@ -275,12 +286,4 @@ export class DashboardDO extends DurableObject { ); this.sql.exec(`DELETE FROM subscribed_project`); } - - private initSchema(): void { - this.sql.exec(` - CREATE TABLE IF NOT EXISTS subscribed_project ( - project_id TEXT PRIMARY KEY - ); - `); - } } diff --git a/worker/src/platform/durable-objects/project-do.ts b/worker/src/platform/durable-objects/project-do.ts index e44eeee..bdf95c9 100644 --- a/worker/src/platform/durable-objects/project-do.ts +++ b/worker/src/platform/durable-objects/project-do.ts @@ -10,6 +10,7 @@ import { toCompactSeries, type CompactSeries } from '../lib/series'; import { chunk, MAX_BOUND_PARAMS } from '../lib/sql'; import { parseDeviceMessage } from '../../domains/telemetry/ws-protocol'; import { upsertVariables } from '../../domains/telemetry/variables'; +import { migrateSchema, type SchemaStep } from './schema'; // Project Durable Object (one per project id, SQLite-backed): latest variable // state, recent ring buffer, pending control writes, and the R2 flush cursor. @@ -55,6 +56,56 @@ export type FlushResult = { newCursor: number; }; +const SCHEMA: SchemaStep[] = [ + (sql) => { + sql.exec(` + CREATE TABLE IF NOT EXISTS latest_state ( + variable TEXT PRIMARY KEY, + value TEXT NOT NULL, + received_at INTEGER NOT NULL + ); + `); + sql.exec(` + CREATE TABLE IF NOT EXISTS ring_buffer ( + rowid INTEGER PRIMARY KEY AUTOINCREMENT, + ts INTEGER NOT NULL, + variable TEXT NOT NULL, + value TEXT NOT NULL + ); + `); + sql.exec(`CREATE INDEX IF NOT EXISTS idx_ring_buffer_ts ON ring_buffer(ts);`); + // Serves the per-variable series reads (chart snapshots + delta polls); the + // ts-only index above stays for age-based eviction. + sql.exec(`CREATE INDEX IF NOT EXISTS idx_ring_buffer_var_ts ON ring_buffer(variable, ts);`); + sql.exec(` + CREATE TABLE IF NOT EXISTS pending_control ( + id TEXT PRIMARY KEY, + variable TEXT NOT NULL, + value TEXT NOT NULL, + created_at INTEGER NOT NULL, + delivered_at INTEGER + ); + `); + sql.exec(` + CREATE TABLE IF NOT EXISTS flush_meta ( + k TEXT PRIMARY KEY, + v TEXT + ); + `); + sql.exec(` + CREATE TABLE IF NOT EXISTS subscriptions ( + dashboard_id TEXT PRIMARY KEY + ); + `); + sql.exec(` + CREATE TABLE IF NOT EXISTS auto_cache ( + k TEXT PRIMARY KEY, + v TEXT + ); + `); + }, +]; + export class ProjectDO extends DurableObject { private sql: SqlStorage; private projectId(): string { @@ -68,7 +119,7 @@ export class ProjectDO extends DurableObject { constructor(ctx: DurableObjectState, env: Env) { super(ctx, env); this.sql = ctx.storage.sql; - this.initSchema(); + migrateSchema(ctx, SCHEMA); } // WS connect calls this so the DO has its project_id for socket-driven ingest — @@ -601,56 +652,6 @@ export class ProjectDO extends DurableObject { return { flushed: rows.length, keys, newCursor }; } - private initSchema(): void { - this.sql.exec(` - CREATE TABLE IF NOT EXISTS latest_state ( - variable TEXT PRIMARY KEY, - value TEXT NOT NULL, - received_at INTEGER NOT NULL - ); - `); - this.sql.exec(` - CREATE TABLE IF NOT EXISTS ring_buffer ( - rowid INTEGER PRIMARY KEY AUTOINCREMENT, - ts INTEGER NOT NULL, - variable TEXT NOT NULL, - value TEXT NOT NULL - ); - `); - this.sql.exec(`CREATE INDEX IF NOT EXISTS idx_ring_buffer_ts ON ring_buffer(ts);`); - // Serves the per-variable series reads (chart snapshots + delta polls); the - // ts-only index above stays for age-based eviction. - this.sql.exec( - `CREATE INDEX IF NOT EXISTS idx_ring_buffer_var_ts ON ring_buffer(variable, ts);` - ); - this.sql.exec(` - CREATE TABLE IF NOT EXISTS pending_control ( - id TEXT PRIMARY KEY, - variable TEXT NOT NULL, - value TEXT NOT NULL, - created_at INTEGER NOT NULL, - delivered_at INTEGER - ); - `); - this.sql.exec(` - CREATE TABLE IF NOT EXISTS flush_meta ( - k TEXT PRIMARY KEY, - v TEXT - ); - `); - this.sql.exec(` - CREATE TABLE IF NOT EXISTS subscriptions ( - dashboard_id TEXT PRIMARY KEY - ); - `); - this.sql.exec(` - CREATE TABLE IF NOT EXISTS auto_cache ( - k TEXT PRIMARY KEY, - v TEXT - ); - `); - } - private evictRingBuffer(now: number): void { // Gate the actual eviction on a cheap last_evict_at lookup so the age DELETE, // COUNT, and overflow DELETE don't run on every single ingest. diff --git a/worker/src/platform/durable-objects/schema.ts b/worker/src/platform/durable-objects/schema.ts new file mode 100644 index 0000000..9eb7835 --- /dev/null +++ b/worker/src/platform/durable-objects/schema.ts @@ -0,0 +1,15 @@ +export type SchemaStep = (sql: SqlStorage) => void; + +// Objects that predate versioning have the tables but no version row, so the +// baseline must stay CREATE ... IF NOT EXISTS — they replay it as a no-op. +export function migrateSchema(ctx: DurableObjectState, steps: SchemaStep[]): void { + ctx.storage.transactionSync(() => { + const sql = ctx.storage.sql; + sql.exec(`CREATE TABLE IF NOT EXISTS schema_version (v INTEGER PRIMARY KEY)`); + const from = sql.exec<{ v: number }>(`SELECT v FROM schema_version`).toArray()[0]?.v ?? 0; + if (from >= steps.length) return; + for (let i = from; i < steps.length; i++) steps[i]!(sql); + sql.exec(`DELETE FROM schema_version`); + sql.exec(`INSERT INTO schema_version (v) VALUES (?)`, steps.length); + }); +} From d2cc81bfbc65ca0cb73b7b8904e2e213c891d5ab Mon Sep 17 00:00:00 2001 From: Arjun Krishna Date: Sat, 22 Aug 2026 10:12:22 +0530 Subject: [PATCH 03/34] feat(deploy): rebuild wrangler.toml from upstream on every deploy The build preserved each deployment's wrangler.toml verbatim, which froze its topology at whatever the Deploy button wrote on day one: a binding, cron or compatibility flag added upstream never reached anyone who had already deployed. Bindings, flags and build config now come from upstream's carrier template, while the Worker name, account, routes, resource ids and vars come from the deployment. Taking upstream's name instead would create a second Worker and orphan the live one, so identity is merged in by binding rather than by position. The build script is fetched from master while the source clone is the latest release tag, so a release predating the merge script falls back to the previous behaviour instead of failing. --- deploy/wrangler.toml | 9 +- scripts/build-from-upstream.sh | 24 +++-- scripts/merge-wrangler.test.ts | 116 ++++++++++++++++++++++ scripts/merge-wrangler.ts | 171 +++++++++++++++++++++++++++++++++ wrangler.toml | 6 +- 5 files changed, 313 insertions(+), 13 deletions(-) create mode 100644 scripts/merge-wrangler.test.ts create mode 100644 scripts/merge-wrangler.ts diff --git a/deploy/wrangler.toml b/deploy/wrangler.toml index 474906d..37c3864 100644 --- a/deploy/wrangler.toml +++ b/deploy/wrangler.toml @@ -1,7 +1,8 @@ -# nodrix deploy carrier. The Deploy to Cloudflare button clones only this -# subdir; the build command pulls the real source from upstream over it. Don't -# edit by hand — manage the deployment from the Cloudflare dashboard. Keep the -# bindings in sync with the repo-root wrangler.toml. +# nodrix deploy carrier. The Deploy to Cloudflare button clones only this subdir; +# the build command pulls the real source from upstream over it and rebuilds the +# deployment's wrangler.toml from this file, keeping only its Worker name, +# account, routes, resource IDs and vars. Everything else here reaches every +# existing deployment on its next build — keep it in sync with the root config. name = "nodrix" main = "worker/src/index.ts" compatibility_date = "2025-05-01" diff --git a/scripts/build-from-upstream.sh b/scripts/build-from-upstream.sh index 038a57a..f189294 100755 --- a/scripts/build-from-upstream.sh +++ b/scripts/build-from-upstream.sh @@ -9,9 +9,11 @@ # source of truth — meaning code changes in upstream never reach them. With # this script, every deploy: # -# 1. Preserves the user's wrangler.toml (which has their resource IDs, -# filled by the Deploy button on day 1 and never changed since). -# 2. Replaces every other file with the upstream source's contents. +# 1. Replaces every file with the upstream source's contents. +# 2. Rebuilds wrangler.toml from upstream's carrier template, keeping only the +# deployment's identity (Worker name, account, routes, resource IDs, vars). +# Keeping the whole file instead froze the topology at day 1, so a binding +# or flag added upstream never reached anyone who had already deployed. # 3. Runs upstream's build pipeline. # # Result: the user's clone is functionally a config carrier. Code = upstream. @@ -31,6 +33,7 @@ UPSTREAM_REPO="${NODRIX_UPSTREAM_REPO:-decoded-cipher/nodrix}" DEPLOY_CHANNEL="${NODRIX_DEPLOY_CHANNEL:-release}" UPSTREAM_DIR="/tmp/nodrix-upstream" WRANGLER_BACKUP="/tmp/nodrix-wrangler.toml" +WRANGLER_MERGED="/tmp/nodrix-wrangler.merged.toml" if [ -z "${WORKERS_CI_COMMIT_SHA:-}" ]; then echo "[build-from-upstream] not in Workers Builds CI — running local build chain" @@ -43,7 +46,7 @@ fi echo "[build-from-upstream] CI build — pulling upstream ${UPSTREAM_REPO} (${DEPLOY_CHANNEL} channel)" -# 1. Preserve user's wrangler.toml. +# 1. Save the deployment's wrangler.toml; step 4 merges it back. if [ ! -f wrangler.toml ]; then echo "[build-from-upstream] no wrangler.toml in cwd — refusing to proceed" >&2 exit 1 @@ -114,8 +117,17 @@ for dir in web worker scripts; do done done -# 4. Restore user's wrangler.toml in case upstream had its own (which it does). -cp "${WRANGLER_BACKUP}" wrangler.toml +# 4. Rebuild wrangler.toml. This script comes from master but the clone is the +# release tag, so a release predating the merge script falls back instead of +# failing. +if [ -f scripts/merge-wrangler.ts ] && [ -f ./deploy/wrangler.toml ]; then + echo "[build-from-upstream] merging deployment identity into upstream wrangler.toml" + bun scripts/merge-wrangler.ts "${WRANGLER_BACKUP}" ./deploy/wrangler.toml > "${WRANGLER_MERGED}" + mv "${WRANGLER_MERGED}" wrangler.toml +else + echo "[build-from-upstream] upstream has no merge script — keeping wrangler.toml as-is" + cp "${WRANGLER_BACKUP}" wrangler.toml +fi # 4b. Drop the nested deploy/ that the overlay just brought in. The clone root # IS the deploy carrier; upstream's own deploy/ dir is dead weight here and diff --git a/scripts/merge-wrangler.test.ts b/scripts/merge-wrangler.test.ts new file mode 100644 index 0000000..d3eb176 --- /dev/null +++ b/scripts/merge-wrangler.test.ts @@ -0,0 +1,116 @@ +// Getting this wrong on a live deployment points it at resources it doesn't own, +// or renames the Worker. Run with `bun test scripts/merge-wrangler.test.ts`. + +import { test, expect } from 'bun:test'; +import { mergeWrangler } from './merge-wrangler'; + +// Renamed, on a custom domain, tracking a fork, and predating three template changes. +const DEPLOYMENT = `name = "home-iot" +main = "worker/src/index.ts" +compatibility_date = "2025-05-01" +compatibility_flags = ["nodejs_compat"] +account_id = "acc_123" +routes = [ + { pattern = "iot.example.com", custom_domain = true } +] + +[[d1_databases]] +binding = "DB" +database_name = "home-iot-db" +database_id = "aaaa-bbbb-cccc" + +[[kv_namespaces]] +binding = "KV" +id = "kv_deadbeef" + +[[r2_buckets]] +binding = "R2" +bucket_name = "home-iot-telemetry" + +[vars] +NODRIX_UPSTREAM_REPO = "someone/nodrix-fork" +`; + +const TEMPLATE = `name = "nodrix" +main = "worker/src/index.ts" +compatibility_date = "2026-01-15" +compatibility_flags = ["nodejs_compat"] + +[build] +command = "curl -fsSL https://example.invalid/build.sh | bash" + +[[d1_databases]] +binding = "DB" +database_name = "nodrix" +database_id = "PLACEHOLDER_FILLED_BY_DEPLOY_OR_WRANGLER" +migrations_dir = "worker/src/platform/db/migrations" + +[[kv_namespaces]] +binding = "KV" +id = "PLACEHOLDER_FILLED_BY_DEPLOY_OR_WRANGLER" + +[[r2_buckets]] +binding = "R2" +bucket_name = "nodrix-telemetry" + +[[durable_objects.bindings]] +name = "PROJECT_DO" +class_name = "ProjectDO" + +[[migrations]] +tag = "v2" +new_sqlite_classes = ["DeviceDO"] + +[vars] +NODRIX_UPSTREAM_REPO = "decoded-cipher/nodrix" +NODRIX_FEATURE_FLAG = "on" + +[triggers] +crons = ["0 0 * * *"] +`; + +const merged = mergeWrangler(DEPLOYMENT, TEMPLATE); + +test('keeps the deployment worker name', () => { + expect(merged).toContain('name = "home-iot"'); + expect(merged).not.toContain('name = "nodrix"'); +}); + +test('keeps resource ids the deployment owns', () => { + expect(merged).toContain('database_id = "aaaa-bbbb-cccc"'); + expect(merged).toContain('database_name = "home-iot-db"'); + expect(merged).toContain('id = "kv_deadbeef"'); + expect(merged).toContain('bucket_name = "home-iot-telemetry"'); + expect(merged).not.toContain('PLACEHOLDER'); +}); + +test('keeps account and routes the template never declares', () => { + expect(merged).toContain('account_id = "acc_123"'); + expect(merged).toContain('pattern = "iot.example.com"'); +}); + +test('takes compatibility settings and build config from upstream', () => { + expect(merged).toContain('compatibility_date = "2026-01-15"'); + expect(merged).toContain('https://example.invalid/build.sh'); +}); + +test('takes new bindings, migrations and triggers from upstream', () => { + expect(merged).toContain('class_name = "ProjectDO"'); + expect(merged).toContain('new_sqlite_classes = ["DeviceDO"]'); + expect(merged).toContain('crons = ["0 0 * * *"]'); + expect(merged).toContain('migrations_dir = "worker/src/platform/db/migrations"'); +}); + +test('keeps an overridden var and adds one the deployment predates', () => { + expect(merged).toContain('NODRIX_UPSTREAM_REPO = "someone/nodrix-fork"'); + expect(merged).toContain('NODRIX_FEATURE_FLAG = "on"'); +}); + +test('is idempotent against its own output', () => { + expect(mergeWrangler(merged, TEMPLATE)).toBe(merged); +}); + +test('a fresh deployment carrying only placeholders is left as the template', () => { + const fresh = mergeWrangler(TEMPLATE, TEMPLATE); + expect(fresh).toBe(TEMPLATE); +}); diff --git a/scripts/merge-wrangler.ts b/scripts/merge-wrangler.ts new file mode 100644 index 0000000..88ac0ef --- /dev/null +++ b/scripts/merge-wrangler.ts @@ -0,0 +1,171 @@ +// Rebuilds a deployment's wrangler.toml: bindings, flags and build config from +// upstream's carrier template, identity from the deployment's own file. +// +// Keeping the deployment's file verbatim froze its topology at whatever the +// Deploy button wrote on day one; taking upstream's verbatim would rename the +// Worker, creating a second one and orphaning the live one. + +import { readFileSync } from 'node:fs'; + +const PRESERVED_TOP_KEYS = ['name', 'account_id', 'workers_dev', 'preview_urls', 'route', 'routes']; + +const PRESERVED_RESOURCE_KEYS: Record = { + d1_databases: ['database_id', 'database_name'], + kv_namespaces: ['id'], + r2_buckets: ['bucket_name'], +}; + +type Section = { header: string; lines: string[] }; + +function keyOf(line: string): string | null { + const m = /^\s*([A-Za-z_][A-Za-z0-9_.-]*)\s*=/.exec(line); + return m ? m[1]! : null; +} + +// TOML arrays and inline tables can span lines; those join into one entry. +function unclosed(text: string): boolean { + let depth = 0; + let quote = ''; + for (let i = 0; i < text.length; i++) { + const c = text[i]!; + if (quote) { + if (c === '\\') i++; + else if (c === quote) quote = ''; + continue; + } + if (c === '"' || c === "'") quote = c; + else if (c === '#') break; + else if (c === '[' || c === '{') depth++; + else if (c === ']' || c === '}') depth--; + } + return depth > 0; +} + +function parse(text: string): Section[] { + const sections: Section[] = [{ header: '', lines: [] }]; + const raw = text.split('\n'); + for (let i = 0; i < raw.length; i++) { + let line = raw[i]!; + if (line.trim().startsWith('[')) { + sections.push({ header: line.trim(), lines: [] }); + continue; + } + if (keyOf(line)) { + while (unclosed(line) && i + 1 < raw.length) line += '\n' + raw[++i]!; + } + sections[sections.length - 1]!.lines.push(line); + } + return sections; +} + +function valueOf(line: string): string { + const eq = line.indexOf('='); + return line.slice(eq + 1).trim().replace(/\s*#.*$/, '').replace(/^["']|["']$/g, ''); +} + +function lookup(section: Section, key: string): string | undefined { + for (const l of section.lines) if (keyOf(l) === key) return l; + return undefined; +} + +function arrayName(header: string): string | null { + const m = /^\[\[([A-Za-z0-9_.-]+)\]\]$/.exec(header); + return m ? m[1]! : null; +} + +export function mergeWrangler(sourceText: string, templateText: string): string { + const source = parse(sourceText); + const template = parse(templateText); + + const sourceTop = source[0]!; + const sourceResources = new Map(); + let sourceVars: Section | undefined; + for (const s of source) { + const name = arrayName(s.header); + if (name && name in PRESERVED_RESOURCE_KEYS) { + const binding = lookup(s, 'binding'); + if (binding) sourceResources.set(`${name}:${valueOf(binding)}`, s); + } else if (s.header === '[vars]') { + sourceVars = s; + } + } + + const out: string[] = []; + const usedTopKeys = new Set(); + + for (const s of template) { + if (s.header) out.push(s.header); + + const name = arrayName(s.header); + const resourceKeys = name ? PRESERVED_RESOURCE_KEYS[name] : undefined; + let resource: Section | undefined; + if (resourceKeys) { + const binding = lookup(s, 'binding'); + if (binding) { + resource = sourceResources.get(`${name}:${valueOf(binding)}`); + if (!resource) { + console.error( + `[merge-wrangler] ${name} binding ${valueOf(binding)} is new upstream — this deployment has no id for it` + ); + } + } + } + + for (const line of s.lines) { + const key = keyOf(line); + if (!key) { + out.push(line); + continue; + } + if (!s.header && PRESERVED_TOP_KEYS.includes(key)) { + const own = lookup(sourceTop, key); + usedTopKeys.add(key); + out.push(own ?? line); + continue; + } + if (resource && resourceKeys!.includes(key)) { + out.push(lookup(resource, key) ?? line); + continue; + } + if (s.header === '[vars]' && sourceVars) { + out.push(lookup(sourceVars, key) ?? line); + continue; + } + out.push(line); + } + + // Custom domains and account ids the template never declares. + if (!s.header) { + const extra = PRESERVED_TOP_KEYS.filter((k) => !usedTopKeys.has(k) && lookup(sourceTop, k)); + if (extra.length) { + let at = out.length; + while (at > 0 && !keyOf(out[at - 1]!)) at--; + out.splice(at, 0, ...extra.map((k) => lookup(sourceTop, k)!)); + } + } + if (s.header === '[vars]' && sourceVars) { + const declared = new Set(s.lines.map(keyOf).filter(Boolean)); + for (const line of sourceVars.lines) { + const k = keyOf(line); + if (k && !declared.has(k)) out.push(line); + } + } + } + + if (sourceVars && !template.some((s) => s.header === '[vars]')) { + out.push('', '[vars]', ...sourceVars.lines.filter((l) => keyOf(l))); + } + + return out.join('\n').replace(/\n{3,}/g, '\n\n').replace(/\n*$/, '\n'); +} + +if (import.meta.main) { + const [, , sourcePath, templatePath] = process.argv; + if (!sourcePath || !templatePath) { + console.error('usage: merge-wrangler.ts '); + process.exit(1); + } + process.stdout.write( + mergeWrangler(readFileSync(sourcePath, 'utf8'), readFileSync(templatePath, 'utf8')) + ); +} diff --git a/wrangler.toml b/wrangler.toml index dbcd3ab..fc9f170 100644 --- a/wrangler.toml +++ b/wrangler.toml @@ -9,9 +9,9 @@ compatibility_flags = ["nodejs_compat"] # bindings change; they can't be symlinked (the subdir must be self-contained). # # Build pipeline. scripts/build-from-upstream.sh, when running under Workers -# Builds (detected via WORKERS_CI_COMMIT_SHA), replaces the clone's working -# tree with upstream source before invoking the build chain. The user's -# wrangler.toml (resource IDs from the Deploy button) is preserved across it. +# Builds (detected via WORKERS_CI_COMMIT_SHA), replaces the clone's working tree +# with upstream source, then rebuilds its wrangler.toml from deploy/wrangler.toml +# — bindings from upstream, resource IDs and Worker name from the deployment. # # Locally the same script falls through to the standard build chain, so # `bun run build` / `wrangler deploy --dry-run` still work for development. From 00b7ee0c6359906ff4e6cc30e896d02488288b7d Mon Sep 17 00:00:00 2001 From: Arjun Krishna Date: Sat, 22 Aug 2026 10:30:02 +0530 Subject: [PATCH 04/34] feat(web): add a single-owner Web Serial port composable MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The monitor holds an exclusive reader on the port and the flasher needs it closed, so both go through claim(), which tears down the read loop, closes the port, hands it over, and restores the monitor at its previous baud. Two owners racing for the same lock is what produces "port already open" errors that survive a reload. Console lines are source-tagged, because flash progress never arrives over the port — esptool-js speaks binary to the ROM loader and reports separately. Web Serial is absent from TypeScript's DOM lib, hence the types dependency. --- web/package.json | 1 + web/src/composables/useSerialPort.ts | 216 +++++++++++++++++++++++++++ web/tsconfig.json | 2 +- 3 files changed, 218 insertions(+), 1 deletion(-) create mode 100644 web/src/composables/useSerialPort.ts diff --git a/web/package.json b/web/package.json index 88243f0..80abfed 100644 --- a/web/package.json +++ b/web/package.json @@ -23,6 +23,7 @@ }, "devDependencies": { "@tailwindcss/vite": "^4.0.0", + "@types/w3c-web-serial": "^1.0.8", "@vitejs/plugin-vue": "^5.2.1", "tailwindcss": "^4.0.0", "typescript": "^5.6.3", diff --git a/web/src/composables/useSerialPort.ts b/web/src/composables/useSerialPort.ts new file mode 100644 index 0000000..54fea15 --- /dev/null +++ b/web/src/composables/useSerialPort.ts @@ -0,0 +1,216 @@ +// Single owner of the page's one SerialPort. The monitor holds an exclusive +// reader and the flasher needs it closed, so both go through claim(). + +import { ref, shallowRef } from 'vue'; + +export type PortMode = 'monitor' | 'flash' | 'provision'; +export type PortState = 'unsupported' | 'closed' | 'opening' | 'open' | 'busy'; + +// Flash progress comes from esptool-js, not over the port. +export type LogSource = 'device' | 'flash' | 'system'; +export type LogLine = { source: LogSource; text: string; at: number }; + +export const serialSupported = + typeof navigator !== 'undefined' && 'serial' in navigator && window.isSecureContext; + +// 74880 is the ESP8266 boot ROM's rate; its reset banner is mojibake elsewhere. +export const BAUD_RATES = [9600, 19200, 38400, 57600, 74880, 115200, 230400, 460800, 921600]; + +// The SDK dots through a Wi-Fi connect without newlines. +const PARTIAL_LINE_FLUSH_MS = 250; + +const port = shallowRef(null); +const state = ref(serialSupported ? 'closed' : 'unsupported'); +const mode = ref(null); +const baudRate = ref(115200); +const lastError = ref(null); + +const listeners = new Set<(line: LogLine) => void>(); +let reader: ReadableStreamDefaultReader | null = null; +let readLoop: Promise | null = null; +let carry = ''; +let flushTimer: ReturnType | null = null; + +let chain: Promise = Promise.resolve(); +function enqueue(fn: () => Promise): Promise { + const run = chain.then(fn, fn); + chain = run.catch(() => {}); + return run; +} + +function deliver(source: LogSource, text: string) { + const line = { source, text, at: Date.now() }; + for (const fn of listeners) fn(line); +} + +export function emit(source: LogSource, text: string) { + deliver(source, text); +} + +function flushCarry() { + if (!carry) return; + deliver('device', carry); + carry = ''; +} + +function absorb(chunk: string) { + carry += chunk.replace(/\r/g, ''); + const parts = carry.split('\n'); + carry = parts.pop() ?? ''; + for (const line of parts) deliver('device', line); + if (flushTimer) clearTimeout(flushTimer); + if (carry) flushTimer = setTimeout(flushCarry, PARTIAL_LINE_FLUSH_MS); +} + +async function pump(p: SerialPort) { + const decoder = new TextDecoder(); + while (p.readable && mode.value === 'monitor') { + reader = p.readable.getReader(); + try { + for (;;) { + const { value, done } = await reader.read(); + if (done) return; + if (value) absorb(decoder.decode(value, { stream: true })); + } + } catch (e) { + lastError.value = (e as Error).message; + return; + } finally { + try { reader.releaseLock(); } catch { /* already released */ } + reader = null; + } + } +} + +async function teardown() { + mode.value = null; + if (flushTimer) { clearTimeout(flushTimer); flushTimer = null; } + flushCarry(); + if (reader) { try { await reader.cancel(); } catch { /* stream already dead */ } } + if (readLoop) { try { await readLoop; } catch { /* surfaced via lastError */ } readLoop = null; } + const p = port.value; + if (p) { try { await p.close(); } catch { /* already closed */ } } + state.value = 'closed'; +} + +async function request(): Promise { + if (!serialSupported) throw new Error('Web Serial is unavailable in this browser'); + try { + port.value = await navigator.serial.requestPort(); + lastError.value = null; + return true; + } catch { + return false; + } +} + +async function beginMonitor(p: SerialPort, baud: number, note: string): Promise { + state.value = 'opening'; + try { + await p.open({ baudRate: baud }); + } catch (e) { + state.value = 'closed'; + lastError.value = (e as Error).message; + throw e; + } + baudRate.value = baud; + mode.value = 'monitor'; + state.value = 'open'; + deliver('system', `${note} at ${baud} baud`); + readLoop = pump(p).finally(() => { + if (mode.value === 'monitor') { mode.value = null; state.value = 'closed'; } + }); +} + +function startMonitor(baud: number): Promise { + return enqueue(async () => { + const p = port.value; + if (!p) throw new Error('No port selected'); + if (mode.value === 'monitor') return; + await beginMonitor(p, baud, 'Connected'); + }); +} + +function stopMonitor(): Promise { + return enqueue(teardown); +} + +// Web Serial can't reconfigure a live port; buffered output is lost. +function setBaud(baud: number): Promise { + return enqueue(async () => { + if (baud === baudRate.value && mode.value === 'monitor') return; + const p = port.value; + const wasMonitoring = mode.value === 'monitor'; + if (wasMonitoring) await teardown(); + baudRate.value = baud; + if (wasMonitoring && p) await beginMonitor(p, baud, 'Reconnected'); + }); +} + +// esptool-js opens and closes the port itself, so hand it over closed. +function claim(next: PortMode, fn: (raw: SerialPort) => Promise): Promise { + return enqueue(async () => { + const p = port.value; + if (!p) throw new Error('No port selected'); + const wasMonitoring = mode.value === 'monitor'; + const baud = baudRate.value; + await teardown(); + mode.value = next; + state.value = 'busy'; + deliver('system', `Monitor released — port handed to ${next}`); + try { + return await fn(p); + } finally { + mode.value = null; + state.value = 'closed'; + if (wasMonitoring) { + try { await beginMonitor(p, baud, 'Monitor resumed'); } catch { /* surfaced as lastError */ } + } + } + }); +} + +async function write(data: string | Uint8Array): Promise { + const p = port.value; + if (!p?.writable) throw new Error('Port is not open for writing'); + const writer = p.writable.getWriter(); + try { + await writer.write(typeof data === 'string' ? new TextEncoder().encode(data) : data); + } finally { + writer.releaseLock(); + } +} + +function onLine(fn: (line: LogLine) => void): () => void { + listeners.add(fn); + return () => listeners.delete(fn); +} + +if (serialSupported) { + navigator.serial.addEventListener('disconnect', (e) => { + if (e.target !== port.value) return; + port.value = null; + mode.value = null; + state.value = 'closed'; + lastError.value = 'Device disconnected'; + deliver('system', 'Device disconnected'); + }); +} + +export function useSerialPort() { + return { + supported: serialSupported, + port, + state, + mode, + baudRate, + lastError, + request, + startMonitor, + stopMonitor, + setBaud, + claim, + write, + onLine, + }; +} diff --git a/web/tsconfig.json b/web/tsconfig.json index ede1d48..a0cca3a 100644 --- a/web/tsconfig.json +++ b/web/tsconfig.json @@ -3,7 +3,7 @@ "compilerOptions": { "jsx": "preserve", "lib": ["ES2023", "DOM", "DOM.Iterable"], - "types": ["vite/client"], + "types": ["vite/client", "w3c-web-serial"], "noEmit": true, "allowImportingTsExtensions": false, "useDefineForClassFields": true From daed5a6ae407bce4f45194107bbedbb371883261 Mon Sep 17 00:00:00 2001 From: Arjun Krishna Date: Sat, 22 Aug 2026 10:32:40 +0530 Subject: [PATCH 05/34] feat(web): classify and buffer serial console output MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Splits the stream into levels the console can colour: SDK lines by their message, ESP-IDF lines by their severity letter, boot ROM output by its prefixes, and anything else as plain sketch output. Entries commit once per animation frame rather than per line — a board at 115200 baud can outrun per-line reactivity. Pausing holds new lines aside instead of dropping them. Tracks whether recent output is mostly replacement and control characters, which is what a wrong baud rate looks like. --- web/src/composables/useSerialLog.ts | 117 ++++++++++++++++++++++++++++ 1 file changed, 117 insertions(+) create mode 100644 web/src/composables/useSerialLog.ts diff --git a/web/src/composables/useSerialLog.ts b/web/src/composables/useSerialLog.ts new file mode 100644 index 0000000..bab7072 --- /dev/null +++ b/web/src/composables/useSerialLog.ts @@ -0,0 +1,117 @@ +import { computed, ref, shallowRef } from 'vue'; +import { useSerialPort, type LogLine, type LogSource } from './useSerialPort'; + +export type LogLevel = 'info' | 'ok' | 'warn' | 'error' | 'system'; +export type LogEntry = { + id: number; + at: number; + source: LogSource; + text: string; + level: LogLevel; + tag?: string; +}; + +const MAX_ENTRIES = 2000; +// A wrong baud rate reads as replacement and control characters, not silence. +const GARBLED_RATIO = 0.2; +const GARBLED_CHARS = /[\uFFFD\u0000-\u0008\u000B\u000C\u000E-\u001F]/g; + +const BOOT_PREFIXES = ['rst:0x', 'ets ', 'load 0x', 'configsip:', 'clk_drv:', 'mode:DIO', 'entry 0x']; + +const NODRIX_LEVELS: [RegExp, LogLevel][] = [ + [/^no wifi network set/, 'error'], + [/^wifi connected/, 'ok'], + [/^connected$/, 'ok'], + [/^disconnected$/, 'warn'], + [/^server error/, 'error'], + [/^unhandled control/, 'warn'], + [/^key dropped/, 'warn'], + [/^telemetry buffer full/, 'warn'], +]; + +function classify(line: LogLine, id: number): LogEntry { + const base = { id, at: line.at, source: line.source, text: line.text }; + if (line.source === 'flash') return { ...base, level: 'info', tag: 'flash' }; + if (line.source === 'system') return { ...base, level: 'system' }; + + const nodrix = /^\[nodrix\]\s*(.*)$/.exec(line.text); + if (nodrix) { + const body = nodrix[1] ?? ''; + const match = NODRIX_LEVELS.find(([re]) => re.test(body)); + return { ...base, text: body, level: match?.[1] ?? 'info', tag: 'nodrix' }; + } + + const idf = /^([EWIDV])\s\(\d+\)\s/.exec(line.text); + if (idf) { + const level: LogLevel = idf[1] === 'E' ? 'error' : idf[1] === 'W' ? 'warn' : 'info'; + return { ...base, level, tag: 'esp' }; + } + + if (BOOT_PREFIXES.some((p) => line.text.startsWith(p))) return { ...base, level: 'system', tag: 'boot' }; + + return { ...base, level: 'info' }; +} + +const entries = shallowRef([]); +const paused = ref(false); +let held: LogEntry[] = []; +let queue: LogEntry[] = []; +let frame: number | null = null; +let nextId = 1; + +function cap(list: LogEntry[]): LogEntry[] { + return list.length > MAX_ENTRIES ? list.slice(list.length - MAX_ENTRIES) : list; +} + +// A board at 115200 can outrun per-line reactivity, so commit once a frame. +function schedule() { + if (frame !== null) return; + frame = requestAnimationFrame(() => { + frame = null; + if (!queue.length) return; + entries.value = cap(entries.value.concat(queue)); + queue = []; + }); +} + +useSerialPort().onLine((line) => { + const entry = classify(line, nextId++); + if (paused.value) { + held = cap(held.concat(entry)); + return; + } + queue.push(entry); + schedule(); +}); + +const garbled = computed(() => { + const recent = entries.value.filter((e) => e.source === 'device').slice(-20); + if (recent.length < 5) return false; + const text = recent.map((e) => e.text).join(''); + if (!text.length) return false; + return (text.match(GARBLED_CHARS) ?? []).length / text.length > GARBLED_RATIO; +}); + +function setPaused(on: boolean) { + paused.value = on; + if (!on && held.length) { + entries.value = cap(entries.value.concat(held)); + held = []; + } +} + +function clear() { + entries.value = []; + queue = []; + held = []; +} + +function toText(): string { + return entries.value + .map((e) => `${new Date(e.at).toISOString().slice(11, 23)} ${e.tag ? `[${e.tag}] ` : ''}${e.text}`) + .join('\n'); +} + +export function useSerialLog() { + return { entries, paused, garbled, setPaused, clear, toText }; +} From cb2803389a94f5dc4b9da61922b612567356c3e7 Mon Sep 17 00:00:00 2001 From: Arjun Krishna Date: Sat, 22 Aug 2026 11:00:19 +0530 Subject: [PATCH 06/34] feat(web): add the serial console MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A Device section under each project, with a console that connects a board over Web Serial and shows what it prints. Output follows the tail only while the view is at the bottom — a chatty board otherwise makes it impossible to read back through a fault. Browsers without Web Serial get a page naming the ones that do and saying this is the only part of nodrix that needs USB, rather than a dead button. The hub takes the tabbed shape Variables and Automations already use, with one tab for now. --- web/src/layouts/Sidebar.vue | 10 +- web/src/pages/project/device/DeviceHub.vue | 41 +++++ .../pages/project/device/SerialConsole.vue | 146 ++++++++++++++++++ web/src/router.ts | 7 + 4 files changed, 203 insertions(+), 1 deletion(-) create mode 100644 web/src/pages/project/device/DeviceHub.vue create mode 100644 web/src/pages/project/device/SerialConsole.vue diff --git a/web/src/layouts/Sidebar.vue b/web/src/layouts/Sidebar.vue index 4ae667b..f6070d2 100644 --- a/web/src/layouts/Sidebar.vue +++ b/web/src/layouts/Sidebar.vue @@ -22,7 +22,7 @@ const hasProject = computed(() => projId.value !== ''); type IconName = | 'home' | 'folder' | 'dashboards' | 'variable' | 'bolt' - | 'integrations' | 'users' | 'key' | 'settings' | 'audit'; + | 'integrations' | 'users' | 'key' | 'settings' | 'audit' | 'device'; type NavItem = { label: string; @@ -58,6 +58,13 @@ const projectScoped = computed(() => { matchPath: (path) => path === `/p/${id}/dashboards` || path.startsWith(`/p/${id}/d/`), }, + { + label: 'Device', + to: `/p/${id}/device`, + icon: 'device', + disabled: !hasProject.value, + matchPath: (path) => path.startsWith(`/p/${id}/device`), + }, { label: 'Automations', to: `/p/${id}/automations`, @@ -105,6 +112,7 @@ const ICON_PATHS: Record = { key: 'M15.75 5.25a3 3 0 0 1 3 3m3 0a6 6 0 0 1-7.029 5.912c-.563-.097-1.159.026-1.563.43L10.5 17.25H8.25v2.25H6v2.25H2.25v-2.818c0-.597.237-1.17.659-1.591l6.499-6.499c.404-.404.527-1 .43-1.563A6 6 0 1 1 21.75 8.25Z', settings: 'M4.5 12a7.5 7.5 0 0 0 .104 1.243l-1.32 1.02a.75.75 0 0 0-.176.957l1.5 2.598a.75.75 0 0 0 .912.328l1.561-.624a7.45 7.45 0 0 0 2.155 1.244l.236 1.66a.75.75 0 0 0 .742.643h3a.75.75 0 0 0 .742-.643l.237-1.66a7.45 7.45 0 0 0 2.154-1.244l1.561.624a.75.75 0 0 0 .912-.328l1.5-2.598a.75.75 0 0 0-.176-.957l-1.32-1.02A7.51 7.51 0 0 0 19.5 12c0-.42-.035-.832-.103-1.232l1.319-1.02a.75.75 0 0 0 .176-.958l-1.5-2.598a.75.75 0 0 0-.912-.327l-1.561.624A7.46 7.46 0 0 0 14.764 5.245l-.236-1.66A.75.75 0 0 0 13.786 3h-3a.75.75 0 0 0-.742.643l-.237 1.66a7.45 7.45 0 0 0-2.154 1.244l-1.561-.624a.75.75 0 0 0-.912.327l-1.5 2.598a.75.75 0 0 0 .176.958l1.32 1.02C4.535 11.168 4.5 11.58 4.5 12Zm10.5 0a3 3 0 1 1-6 0 3 3 0 0 1 6 0Z', audit: 'M9 5h6m-6 0H7a2 2 0 0 0-2 2v12a2 2 0 0 0 2 2h10a2 2 0 0 0 2-2V7a2 2 0 0 0-2-2h-2M9 5a2 2 0 1 1 6 0M9 12h6m-6 4h4', + device: 'M8.25 3v1.5M4.5 8.25H3m18 0h-1.5M4.5 12H3m18 0h-1.5m-15 3.75H3m18 0h-1.5M8.25 19.5V21M12 3v1.5m0 15V21m3.75-18v1.5m0 15V21m-9-1.5h10.5a2.25 2.25 0 0 0 2.25-2.25V6.75a2.25 2.25 0 0 0-2.25-2.25H6.75A2.25 2.25 0 0 0 4.5 6.75v10.5a2.25 2.25 0 0 0 2.25 2.25Zm.75-12h9v9h-9v-9Z', }; function iconFor(name: IconName): FunctionalComponent { diff --git a/web/src/pages/project/device/DeviceHub.vue b/web/src/pages/project/device/DeviceHub.vue new file mode 100644 index 0000000..9dde30f --- /dev/null +++ b/web/src/pages/project/device/DeviceHub.vue @@ -0,0 +1,41 @@ + + + diff --git a/web/src/pages/project/device/SerialConsole.vue b/web/src/pages/project/device/SerialConsole.vue new file mode 100644 index 0000000..12e8242 --- /dev/null +++ b/web/src/pages/project/device/SerialConsole.vue @@ -0,0 +1,146 @@ + + + diff --git a/web/src/router.ts b/web/src/router.ts index 88ae672..98ecc10 100644 --- a/web/src/router.ts +++ b/web/src/router.ts @@ -43,6 +43,13 @@ const routes: RouteRecordRaw[] = [ { path: 'tokens', name: 'variable-tokens', component: () => import('./pages/project/variables/ConnectionTokens.vue'), meta: { title: 'Connection tokens' } }, ], }, + { + path: 'device', + component: () => import('./pages/project/device/DeviceHub.vue'), + children: [ + { path: '', name: 'serial-console', component: () => import('./pages/project/device/SerialConsole.vue'), meta: { title: 'Console' } }, + ], + }, // Automations + Integrations live under one hub with two tabs. { path: 'automations', From d49d63d8f5da17f826f6a0a6034b4fe4fe1e4c1c Mon Sep 17 00:00:00 2001 From: Arjun Krishna Date: Sat, 22 Aug 2026 11:12:52 +0530 Subject: [PATCH 07/34] feat(web): explain what the board is doing in plain language MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Turns SDK debug output into a banner above the console — "Token rejected", "The server refused the connection" — instead of leaving a red line for the reader to interpret. The newest significant line wins, so recovering from a fault clears the stale error. Wi-Fi state is tracked separately from cloud state, which is what makes a compound reading possible: "Wi-Fi is up" in front of a server failure says the radio is fine and the problem is further along. diagnose() takes an entry array and holds no reactive state, so the rules are tested directly rather than through a mounted component. --- web/src/composables/useSerialDiagnosis.ts | 105 ++++++++++++++++++ .../pages/project/device/SerialConsole.vue | 13 +++ web/test/serial-diagnosis.test.ts | 69 ++++++++++++ 3 files changed, 187 insertions(+) create mode 100644 web/src/composables/useSerialDiagnosis.ts create mode 100644 web/test/serial-diagnosis.test.ts diff --git a/web/src/composables/useSerialDiagnosis.ts b/web/src/composables/useSerialDiagnosis.ts new file mode 100644 index 0000000..3d40137 --- /dev/null +++ b/web/src/composables/useSerialDiagnosis.ts @@ -0,0 +1,105 @@ +import { computed, type ComputedRef } from 'vue'; +import { useSerialLog, type LogEntry } from './useSerialLog'; + +export type Diagnosis = { + tone: 'ok' | 'warn' | 'error'; + headline: string; + detail?: string; +}; + +type Rule = { re: RegExp; build: (m: RegExpMatchArray) => Diagnosis }; + +const RULES: Rule[] = [ + { + re: /^no wifi network set/, + build: () => ({ + tone: 'error', + headline: 'No Wi-Fi network configured', + detail: 'The sketch never called addAP(), so it has nothing to join.', + }), + }, + { + re: /^connect refused/, + build: () => ({ + tone: 'error', + headline: 'The server refused the connection', + detail: 'The socket never opened once. The token is wrong, or the host is.', + }), + }, + { + re: /-> 401/, + build: () => ({ tone: 'error', headline: 'Token rejected', detail: 'This token is not valid for this instance.' }), + }, + { + re: /-> 403/, + build: () => ({ tone: 'error', headline: 'Token has no access to this project' }), + }, + { + re: /-> 404/, + build: () => ({ tone: 'error', headline: 'No such endpoint', detail: 'The host is probably wrong.' }), + }, + { + re: /-> 429/, + build: () => ({ tone: 'warn', headline: 'Rate limited', detail: 'The board is sending faster than the instance accepts.' }), + }, + { + re: /-> -\d+ \(no connection/, + build: () => ({ + tone: 'error', + headline: "Can't reach the server", + detail: 'DNS, TLS or the network dropped it before any reply came back.', + }), + }, + { + re: /^socket error/, + build: () => ({ tone: 'error', headline: 'Socket error' }), + }, + { + re: /^server error:\s*(\S+)/, + build: (m) => ({ tone: 'error', headline: `Server rejected the message`, detail: `It replied with ${m[1]}.` }), + }, + { + re: /^connected$/, + build: () => ({ tone: 'ok', headline: 'Connected' }), + }, + { + re: /^disconnected$/, + build: () => ({ tone: 'warn', headline: 'Disconnected', detail: 'The link was up and dropped. It will retry.' }), + }, +]; + +function match(entry: LogEntry): Diagnosis | null { + if (entry.tag !== 'nodrix') return null; + for (const rule of RULES) { + const m = entry.text.match(rule.re); + if (m) return rule.build(m); + } + return null; +} + +// Wi-Fi state is separate from cloud state: knowing the network is up is what +// turns "disconnected" into a statement about the server rather than the radio. +function wifiUp(entries: LogEntry[]): boolean { + for (let i = entries.length - 1; i >= 0; i--) { + const e = entries[i]!; + if (e.tag === 'nodrix' && /^wifi connected/.test(e.text)) return true; + } + return false; +} + +export function diagnose(list: LogEntry[]): Diagnosis | null { + for (let i = list.length - 1; i >= 0; i--) { + const found = match(list[i]!); + if (!found) continue; + if (found.tone !== 'ok' && wifiUp(list)) { + return { ...found, detail: `Wi-Fi is up. ${found.detail ?? ''}`.trim() }; + } + return found; + } + return null; +} + +export function useSerialDiagnosis(): ComputedRef { + const { entries } = useSerialLog(); + return computed(() => diagnose(entries.value)); +} diff --git a/web/src/pages/project/device/SerialConsole.vue b/web/src/pages/project/device/SerialConsole.vue index 12e8242..97d2bc5 100644 --- a/web/src/pages/project/device/SerialConsole.vue +++ b/web/src/pages/project/device/SerialConsole.vue @@ -2,10 +2,12 @@ import { computed, nextTick, ref, watch } from 'vue'; import { useSerialPort, BAUD_RATES } from '../../../composables/useSerialPort'; import { useSerialLog, type LogEntry } from '../../../composables/useSerialLog'; +import { useSerialDiagnosis } from '../../../composables/useSerialDiagnosis'; import { toast } from '../../../lib/toast'; const { supported, port, state, lastError, request, startMonitor, stopMonitor, setBaud } = useSerialPort(); const { entries, paused, garbled, setPaused, clear, toText } = useSerialLog(); +const diagnosis = useSerialDiagnosis(); const viewport = ref(null); const stuckToBottom = ref(true); @@ -54,6 +56,12 @@ async function copyAll() { } // The viewport is dark in both themes, so these take no light variant. +const DIAGNOSIS_TONE = { + ok: 'border-emerald-300 bg-emerald-50 text-emerald-900 dark:border-emerald-900 dark:bg-emerald-950/40 dark:text-emerald-200', + warn: 'border-amber-300 bg-amber-50 text-amber-900 dark:border-amber-900 dark:bg-amber-950/40 dark:text-amber-200', + error: 'border-red-300 bg-red-50 text-red-900 dark:border-red-900 dark:bg-red-950/40 dark:text-red-200', +}; + const TONE: Record = { ok: 'text-emerald-400', warn: 'text-amber-400', @@ -119,6 +127,11 @@ function stamp(at: number) { +
+

{{ diagnosis.headline }}

+

{{ diagnosis.detail }}

+
+

This looks like the wrong baud rate. Most nodrix sketches use 115200.

diff --git a/web/test/serial-diagnosis.test.ts b/web/test/serial-diagnosis.test.ts new file mode 100644 index 0000000..1b20b97 --- /dev/null +++ b/web/test/serial-diagnosis.test.ts @@ -0,0 +1,69 @@ +// The console's plain-language read of SDK debug output. Run with +// `bun test web/test/serial-diagnosis.test.ts`. + +import { test, expect } from 'bun:test'; +import { diagnose } from '../src/composables/useSerialDiagnosis'; +import type { LogEntry } from '../src/composables/useSerialLog'; + +let id = 0; +function nodrix(text: string): LogEntry { + return { id: ++id, at: 0, source: 'device', text, level: 'info', tag: 'nodrix' }; +} +function sketch(text: string): LogEntry { + return { id: ++id, at: 0, source: 'device', text, level: 'info' }; +} + +test('says nothing until the SDK says something', () => { + expect(diagnose([])).toBeNull(); + expect(diagnose([sketch('Booting...'), sketch('temp=21.5')])).toBeNull(); +}); + +test('names a missing Wi-Fi config', () => { + const d = diagnose([nodrix('no wifi network set (call addAP)')]); + expect(d?.tone).toBe('error'); + expect(d?.headline).toBe('No Wi-Fi network configured'); +}); + +test('reads a 401 as a rejected token', () => { + const d = diagnose([nodrix('POST /v1/telemetry -> 401 (token rejected)')]); + expect(d?.tone).toBe('error'); + expect(d?.headline).toBe('Token rejected'); +}); + +test('separates a refused handshake from a dropped link', () => { + expect(diagnose([nodrix('connect refused - check token and host')])?.headline) + .toBe('The server refused the connection'); + expect(diagnose([nodrix('connected'), nodrix('disconnected')])?.headline) + .toBe('Disconnected'); +}); + +test('qualifies a failure with Wi-Fi being up', () => { + const d = diagnose([ + nodrix('wifi connected: 192.168.1.42'), + nodrix('POST /v1/telemetry -> -1 (no connection - check host, DNS or TLS)'), + ]); + expect(d?.headline).toBe("Can't reach the server"); + expect(d?.detail).toStartWith('Wi-Fi is up.'); +}); + +test('does not qualify a success with Wi-Fi noise', () => { + const d = diagnose([nodrix('wifi connected: 192.168.1.42'), nodrix('connected')]); + expect(d).toEqual({ tone: 'ok', headline: 'Connected' }); +}); + +test('reports the most recent state, not the first', () => { + const d = diagnose([ + nodrix('POST /v1/telemetry -> 401 (token rejected)'), + nodrix('connected'), + ]); + expect(d?.headline).toBe('Connected'); +}); + +test('carries the server code through', () => { + const d = diagnose([nodrix('server error: variable_limit')]); + expect(d?.detail).toBe('It replied with variable_limit.'); +}); + +test('ignores sketch output that looks like SDK output', () => { + expect(diagnose([sketch('connect refused - check token and host')])).toBeNull(); +}); From 65c0caef88b02a845353cb2a1ac57c12e96b7652 Mon Sep 17 00:00:00 2001 From: Arjun Krishna Date: Sat, 22 Aug 2026 11:21:51 +0530 Subject: [PATCH 08/34] feat(db): give every project devices and scope variables to them Adds a devices table and a default device per project, then rebuilds project_variables with device_id in its unique key. Telemetry that names no device lands on the default, so a plain curl with only a token keeps working. project_variables is rebuilt rather than altered because device_id is NOT NULL with a foreign key and the unique key changes shape. Nothing references that table by foreign key, so the rebuild needs no deferral. Default device ids derive from the project id, which keeps the migration deterministic without generating ids in SQL, and one default per project is enforced by a partial unique index rather than left to convention. Tested against SQLite: an instance seeded before the upgrade and a fresh install come out with byte-identical schemas, and replaying the baseline over a migrated database does not reinstate the old two-column unique index. --- worker/src/platform/db/migrations.gen.ts | 14 ++ .../platform/db/migrations/0002_devices.sql | 48 +++++++ worker/test/migrations.test.ts | 128 ++++++++++++++++++ 3 files changed, 190 insertions(+) create mode 100644 worker/src/platform/db/migrations/0002_devices.sql create mode 100644 worker/test/migrations.test.ts diff --git a/worker/src/platform/db/migrations.gen.ts b/worker/src/platform/db/migrations.gen.ts index 3566f99..9fa0820 100644 --- a/worker/src/platform/db/migrations.gen.ts +++ b/worker/src/platform/db/migrations.gen.ts @@ -47,5 +47,19 @@ export const MIGRATIONS: Migration[] = [ "CREATE INDEX IF NOT EXISTS idx_invites_email ON invites(email) WHERE email IS NOT NULL", "CREATE TABLE IF NOT EXISTS invite_projects (\n invite_id TEXT NOT NULL REFERENCES invites(id) ON DELETE CASCADE,\n project_id TEXT NOT NULL REFERENCES projects(id) ON DELETE CASCADE,\n PRIMARY KEY (invite_id, project_id)\n)" ] + }, + { + "name": "0002_devices", + "statements": [ + "CREATE TABLE IF NOT EXISTS devices (\n id TEXT PRIMARY KEY,\n project_id TEXT NOT NULL REFERENCES projects(id) ON DELETE CASCADE,\n name TEXT NOT NULL,\n chip TEXT,\n firmware_version TEXT,\n is_default INTEGER NOT NULL DEFAULT 0,\n first_seen INTEGER,\n last_seen INTEGER,\n created_at INTEGER NOT NULL\n)", + "CREATE INDEX IF NOT EXISTS idx_devices_project ON devices(project_id)", + "CREATE UNIQUE INDEX IF NOT EXISTS idx_devices_default\n ON devices(project_id) WHERE is_default = 1", + "INSERT INTO devices (id, project_id, name, is_default, created_at)\nSELECT 'dev_' || substr(p.id, 5), p.id, 'Default', 1, p.created_at\nFROM projects p\nWHERE NOT EXISTS (SELECT 1 FROM devices d WHERE d.project_id = p.id AND d.is_default = 1)", + "CREATE TABLE project_variables_new (\n id TEXT PRIMARY KEY,\n project_id TEXT NOT NULL REFERENCES projects(id) ON DELETE CASCADE,\n device_id TEXT NOT NULL REFERENCES devices(id) ON DELETE CASCADE,\n key TEXT NOT NULL,\n unit TEXT,\n created_at INTEGER NOT NULL,\n updated_at INTEGER NOT NULL,\n last_seen INTEGER\n)", + "INSERT INTO project_variables_new (id, project_id, device_id, key, unit, created_at, updated_at, last_seen)\nSELECT v.id, v.project_id, d.id, v.key, v.unit, v.created_at, v.updated_at, v.last_seen\nFROM project_variables v\nJOIN devices d ON d.project_id = v.project_id AND d.is_default = 1", + "DROP TABLE project_variables", + "ALTER TABLE project_variables_new RENAME TO project_variables", + "CREATE UNIQUE INDEX idx_project_variables_key\n ON project_variables(project_id, device_id, key)" + ] } ]; diff --git a/worker/src/platform/db/migrations/0002_devices.sql b/worker/src/platform/db/migrations/0002_devices.sql new file mode 100644 index 0000000..1478cc6 --- /dev/null +++ b/worker/src/platform/db/migrations/0002_devices.sql @@ -0,0 +1,48 @@ +-- Devices. Rows appear the first time a board is seen; there is no enrolment. +CREATE TABLE IF NOT EXISTS devices ( + id TEXT PRIMARY KEY, + project_id TEXT NOT NULL REFERENCES projects(id) ON DELETE CASCADE, + name TEXT NOT NULL, + chip TEXT, + firmware_version TEXT, + is_default INTEGER NOT NULL DEFAULT 0, + first_seen INTEGER, + last_seen INTEGER, + created_at INTEGER NOT NULL +); +CREATE INDEX IF NOT EXISTS idx_devices_project ON devices(project_id); + +-- Exactly one default per project, enforced rather than assumed. +CREATE UNIQUE INDEX IF NOT EXISTS idx_devices_default + ON devices(project_id) WHERE is_default = 1; + +-- Every project gets a default device. Telemetry that names no device lands +-- here, so a plain curl with only a token keeps working. +INSERT INTO devices (id, project_id, name, is_default, created_at) +SELECT 'dev_' || substr(p.id, 5), p.id, 'Default', 1, p.created_at +FROM projects p +WHERE NOT EXISTS (SELECT 1 FROM devices d WHERE d.project_id = p.id AND d.is_default = 1); + +-- project_variables gains device_id. Rebuilt rather than altered: the column is +-- NOT NULL with a foreign key, and the unique key it belongs to changes shape. +CREATE TABLE project_variables_new ( + id TEXT PRIMARY KEY, + project_id TEXT NOT NULL REFERENCES projects(id) ON DELETE CASCADE, + device_id TEXT NOT NULL REFERENCES devices(id) ON DELETE CASCADE, + key TEXT NOT NULL, + unit TEXT, + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL, + last_seen INTEGER +); + +INSERT INTO project_variables_new (id, project_id, device_id, key, unit, created_at, updated_at, last_seen) +SELECT v.id, v.project_id, d.id, v.key, v.unit, v.created_at, v.updated_at, v.last_seen +FROM project_variables v +JOIN devices d ON d.project_id = v.project_id AND d.is_default = 1; + +DROP TABLE project_variables; +ALTER TABLE project_variables_new RENAME TO project_variables; + +CREATE UNIQUE INDEX idx_project_variables_key + ON project_variables(project_id, device_id, key); diff --git a/worker/test/migrations.test.ts b/worker/test/migrations.test.ts new file mode 100644 index 0000000..dcca99f --- /dev/null +++ b/worker/test/migrations.test.ts @@ -0,0 +1,128 @@ +// A fresh install and an upgraded one have to arrive at the same schema, and the +// upgrade has to keep its data. Runs the real bundled statements against SQLite. +// Run with `bun test worker/test/migrations.test.ts`. + +import { test, expect } from 'bun:test'; +import { Database } from 'bun:sqlite'; +import { MIGRATIONS } from '../src/platform/db/migrations.gen'; + +function apply(db: Database, upTo: number) { + for (const m of MIGRATIONS.slice(0, upTo)) { + for (const stmt of m.statements) db.run(stmt); + } +} + +// Ordered so two databases built by different routes compare directly. +function schemaOf(db: Database): string { + const rows = db + .query<{ type: string; name: string; sql: string | null }, []>( + `SELECT type, name, sql FROM sqlite_master + WHERE name NOT LIKE 'sqlite_%' ORDER BY type, name` + ) + .all(); + return rows.map((r) => `${r.type} ${r.name}\n${r.sql ?? ''}`).join('\n---\n'); +} + +function seed(db: Database) { + db.run(`INSERT INTO users (id, email, name, role, created_at, updated_at) + VALUES ('usr_1', 'a@b.c', 'A', 'owner', 1, 1)`); + db.run(`INSERT INTO projects (id, name, created_by, created_at, updated_at) + VALUES ('prj_alpha', 'Alpha', 'usr_1', 100, 100), + ('prj_beta', 'Beta', 'usr_1', 200, 200)`); + db.run(`INSERT INTO project_variables (id, project_id, key, unit, created_at, updated_at, last_seen) + VALUES ('var_1', 'prj_alpha', 'temp', 'C', 100, 100, 500), + ('var_2', 'prj_alpha', 'humidity', NULL, 100, 100, 600), + ('var_3', 'prj_beta', 'temp', NULL, 200, 200, NULL)`); +} + +test('fresh and upgraded instances reach an identical schema', () => { + const upgraded = new Database(':memory:'); + apply(upgraded, 1); + seed(upgraded); + apply(upgraded, MIGRATIONS.length); + + const fresh = new Database(':memory:'); + apply(fresh, MIGRATIONS.length); + + expect(schemaOf(upgraded)).toBe(schemaOf(fresh)); +}); + +test('the upgrade runs clean against an instance with no rows', () => { + const db = new Database(':memory:'); + apply(db, 1); + expect(() => apply(db, MIGRATIONS.length)).not.toThrow(); + expect(db.query(`SELECT COUNT(*) AS n FROM devices`).get()).toEqual({ n: 0 }); +}); + +test('every project gets exactly one default device', () => { + const db = new Database(':memory:'); + apply(db, 1); + seed(db); + apply(db, MIGRATIONS.length); + + const devices = db + .query<{ id: string; project_id: string; name: string; is_default: number }, []>( + `SELECT id, project_id, name, is_default FROM devices ORDER BY project_id` + ) + .all(); + expect(devices).toEqual([ + { id: 'dev_alpha', project_id: 'prj_alpha', name: 'Default', is_default: 1 }, + { id: 'dev_beta', project_id: 'prj_beta', name: 'Default', is_default: 1 }, + ]); +}); + +test('a second default device cannot be inserted', () => { + const db = new Database(':memory:'); + apply(db, 1); + seed(db); + apply(db, MIGRATIONS.length); + expect(() => + db.run(`INSERT INTO devices (id, project_id, name, is_default, created_at) + VALUES ('dev_x', 'prj_alpha', 'Second', 1, 1)`) + ).toThrow(); +}); + +test('existing variables survive and land on their default device', () => { + const db = new Database(':memory:'); + apply(db, 1); + seed(db); + apply(db, MIGRATIONS.length); + + const rows = db + .query<{ id: string; project_id: string; device_id: string; key: string; unit: string | null; last_seen: number | null }, []>( + `SELECT id, project_id, device_id, key, unit, last_seen FROM project_variables ORDER BY id` + ) + .all(); + expect(rows).toEqual([ + { id: 'var_1', project_id: 'prj_alpha', device_id: 'dev_alpha', key: 'temp', unit: 'C', last_seen: 500 }, + { id: 'var_2', project_id: 'prj_alpha', device_id: 'dev_alpha', key: 'humidity', unit: null, last_seen: 600 }, + { id: 'var_3', project_id: 'prj_beta', device_id: 'dev_beta', key: 'temp', unit: null, last_seen: null }, + ]); +}); + +test('the same key is now allowed once per device', () => { + const db = new Database(':memory:'); + apply(db, 1); + seed(db); + apply(db, MIGRATIONS.length); + + db.run(`INSERT INTO devices (id, project_id, name, created_at) VALUES ('dev_two', 'prj_alpha', 'Shed', 1)`); + expect(() => + db.run(`INSERT INTO project_variables (id, project_id, device_id, key, created_at, updated_at) + VALUES ('var_4', 'prj_alpha', 'dev_two', 'temp', 1, 1)`) + ).not.toThrow(); + expect(() => + db.run(`INSERT INTO project_variables (id, project_id, device_id, key, created_at, updated_at) + VALUES ('var_5', 'prj_alpha', 'dev_alpha', 'temp', 1, 1)`) + ).toThrow(); +}); + +// The baseline is all CREATE ... IF NOT EXISTS, including the index whose columns +// 0002 changes. Replaying it must not put the old shape back. +test('replaying the baseline over a migrated database changes nothing', () => { + const db = new Database(':memory:'); + apply(db, MIGRATIONS.length); + const before = schemaOf(db); + apply(db, 1); + expect(schemaOf(db)).toBe(before); +}); From 9846b93dfee80e85fe9106d9081fb84aa2155ea5 Mon Sep 17 00:00:00 2001 From: Arjun Krishna Date: Sat, 22 Aug 2026 17:47:43 +0530 Subject: [PATCH 09/34] feat(worker): attribute incoming telemetry to a device MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Boards identify themselves with an X-Nodrix-Device header; the value maps to a device row, created on first sight. Anything that names no device lands on the project's default, so an existing board or a plain curl keeps working untouched. The reported key is untrusted input — normalised, length-capped, and bounded at 100 devices per project so a board that reports a fresh key every boot can't grow the table without limit. It is kept apart from the device id so renaming stays free and a MAC never surfaces in anything a user reads. Device creation reads back after inserting rather than trusting the id it generated: two isolates racing on the same first boot both insert, one loses the conflict, and returning its own id would attribute telemetry to a row that does not exist. createProject now writes the project and its default device in one batch. The migration only covers projects that already existed, so without this a project created after upgrading would have no default device at all. --- worker/src/domains/devices/service.ts | 119 ++++++++++++++++++ worker/src/domains/projects/service.ts | 14 ++- worker/src/domains/telemetry/telemetry.ts | 14 ++- worker/src/domains/telemetry/variables.ts | 21 ++-- worker/src/domains/variables/service.ts | 11 +- worker/src/platform/db/migrations.gen.ts | 3 +- .../platform/db/migrations/0002_devices.sql | 6 + .../platform/durable-objects/project-do.ts | 7 +- worker/src/platform/lib/ids.ts | 2 + 9 files changed, 172 insertions(+), 25 deletions(-) create mode 100644 worker/src/domains/devices/service.ts diff --git a/worker/src/domains/devices/service.ts b/worker/src/domains/devices/service.ts new file mode 100644 index 0000000..bf74a80 --- /dev/null +++ b/worker/src/domains/devices/service.ts @@ -0,0 +1,119 @@ +import type { Env } from '../../env'; +import { newId } from '../../platform/lib/ids'; + +export type DeviceSummary = { + id: string; + name: string; + chip: string | null; + firmware_version: string | null; + is_default: number; + first_seen: number | null; + last_seen: number | null; +}; + +// Bounds device growth from a board that reports a fresh key every boot. +const MAX_DEVICES_PER_PROJECT = 100; +const MAX_DEVICE_KEY_LEN = 64; + +// Device ids change only when a device is forgotten, so an isolate can hold the +// project -> device mapping for as long as it lives. +const resolved = new Map(); + +export function forgetCachedDevice(projectId: string, deviceKey?: string | null) { + resolved.delete(cacheKey(projectId, deviceKey ?? null)); + if (deviceKey) return; + for (const k of resolved.keys()) if (k.startsWith(`${projectId}:`)) resolved.delete(k); +} + +function cacheKey(projectId: string, deviceKey: string | null): string { + return `${projectId}:${deviceKey ?? ''}`; +} + +// A board picks its own key, so it has to be treated as untrusted input. +export function normaliseDeviceKey(raw: string | null | undefined): string | null { + if (typeof raw !== 'string') return null; + const trimmed = raw.trim(); + if (!trimmed || trimmed.length > MAX_DEVICE_KEY_LEN) return null; + return /^[A-Za-z0-9:._-]+$/.test(trimmed) ? trimmed : null; +} + +export async function defaultDeviceId(env: Env, projectId: string): Promise { + const cached = resolved.get(cacheKey(projectId, null)); + if (cached) return cached; + const row = await env.DB + .prepare(`SELECT id FROM devices WHERE project_id = ? AND is_default = 1`) + .bind(projectId) + .first<{ id: string }>(); + if (row) resolved.set(cacheKey(projectId, null), row.id); + return row?.id ?? null; +} + +// Every project needs one, including projects created after the upgrade — a +// migration alone would only cover the ones that already existed. +export function createDefaultDevice(env: Env, projectId: string, now: number) { + return env.DB + .prepare( + `INSERT INTO devices (id, project_id, name, is_default, created_at) + VALUES (?, ?, 'Default', 1, ?)` + ) + .bind(newId('device'), projectId, now); +} + +// Maps what a board calls itself to a device row, creating it on first sight. +// An unnamed board lands on the default device. +export async function resolveDevice( + env: Env, + projectId: string, + deviceKey: string | null, + now: number +): Promise { + if (!deviceKey) return defaultDeviceId(env, projectId); + + const cached = resolved.get(cacheKey(projectId, deviceKey)); + if (cached) return cached; + + const existing = await env.DB + .prepare(`SELECT id FROM devices WHERE project_id = ? AND device_key = ?`) + .bind(projectId, deviceKey) + .first<{ id: string }>(); + if (existing) { + resolved.set(cacheKey(projectId, deviceKey), existing.id); + return existing.id; + } + + const count = await env.DB + .prepare(`SELECT COUNT(*) AS n FROM devices WHERE project_id = ?`) + .bind(projectId) + .first<{ n: number }>(); + if ((count?.n ?? 0) >= MAX_DEVICES_PER_PROJECT) return defaultDeviceId(env, projectId); + + const id = newId('device'); + await env.DB + .prepare( + `INSERT INTO devices (id, project_id, name, device_key, first_seen, last_seen, created_at) + VALUES (?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(project_id, device_key) DO NOTHING` + ) + .bind(id, projectId, deviceKey, deviceKey, now, now, now) + .run(); + + // A concurrent isolate may have won the insert, so read back rather than + // assuming the id we generated is the one that stuck. + const settled = await env.DB + .prepare(`SELECT id FROM devices WHERE project_id = ? AND device_key = ?`) + .bind(projectId, deviceKey) + .first<{ id: string }>(); + if (settled) resolved.set(cacheKey(projectId, deviceKey), settled.id); + return settled?.id ?? null; +} + +export async function listDevices(env: Env, projectId: string): Promise { + const rows = await env.DB + .prepare( + `SELECT id, name, chip, firmware_version, is_default, first_seen, last_seen + FROM devices WHERE project_id = ? ORDER BY is_default DESC, name ASC` + ) + .bind(projectId) + .all(); + return rows.results; +} diff --git a/worker/src/domains/projects/service.ts b/worker/src/domains/projects/service.ts index d54c8b5..5d184c0 100644 --- a/worker/src/domains/projects/service.ts +++ b/worker/src/domains/projects/service.ts @@ -1,6 +1,7 @@ import type { Env } from '../../env'; import { type Actor, isInstanceAdmin, ServiceError } from '../../platform/lib/service'; import { newId } from '../../platform/lib/ids'; +import { createDefaultDevice } from '../devices/service'; import { recordAudit } from '../../platform/lib/audit'; import { buildUpdate, chunk, inClause, MAX_BOUND_PARAMS } from '../../platform/lib/sql'; @@ -103,12 +104,13 @@ export async function createProject( const id = newId('project'); const now = Math.floor(Date.now() / 1000); - await env.DB - .prepare( - `INSERT INTO projects (id, name, created_by, created_at, updated_at) VALUES (?, ?, ?, ?, ?)` - ) - .bind(id, name, actor.userId, now, now) - .run(); + // One batch so a project can never exist without its default device. + await env.DB.batch([ + env.DB + .prepare(`INSERT INTO projects (id, name, created_by, created_at, updated_at) VALUES (?, ?, ?, ?, ?)`) + .bind(id, name, actor.userId, now, now), + createDefaultDevice(env, id, now), + ]); await recordAudit(env, { projectId: id, diff --git a/worker/src/domains/telemetry/telemetry.ts b/worker/src/domains/telemetry/telemetry.ts index a641dd7..180494e 100644 --- a/worker/src/domains/telemetry/telemetry.ts +++ b/worker/src/domains/telemetry/telemetry.ts @@ -4,6 +4,7 @@ import { requireProjectToken, type ProjectTokenContextVars } from '../../platfor import { projectStub } from '../../platform/durable-objects/stubs'; import { parseTelemetryBody, MAX_POINTS, MAX_KEY_LEN, MAX_STRING_VALUE } from './validate'; import { upsertVariables } from './variables'; +import { normaliseDeviceKey, resolveDevice } from '../devices/service'; const telemetry = new Hono<{ Bindings: Env; Variables: ProjectTokenContextVars }>(); @@ -40,14 +41,19 @@ telemetry.post('/', async (c) => { const points = parsed.points; const { project_id } = c.get('projectToken'); + const now = Math.floor(Date.now() / 1000); + const deviceKey = normaliseDeviceKey(c.req.header('x-nodrix-device')); + const deviceId = await resolveDevice(c.env, project_id, deviceKey, now); + const stub = projectStub(c.env, project_id); await stub.ingest(project_id, points); // Auto-create new variables + bump last_seen off the response path (best-effort). - const now = Math.floor(Date.now() / 1000); - c.executionCtx.waitUntil( - upsertVariables(c.env, project_id, points.map((p) => p.variable), now) - ); + if (deviceId) { + c.executionCtx.waitUntil( + upsertVariables(c.env, project_id, deviceId, points.map((p) => p.variable), now) + ); + } return c.body(null, 204); }); diff --git a/worker/src/domains/telemetry/variables.ts b/worker/src/domains/telemetry/variables.ts index 9434a9f..4bd8351 100644 --- a/worker/src/domains/telemetry/variables.ts +++ b/worker/src/domains/telemetry/variables.ts @@ -12,8 +12,8 @@ const MAX_VARIABLES_PER_PROJECT = 250; const LAST_SEEN_THROTTLE_MS = 60_000; const lastSeenWrites = new Map(); -function dueForLastSeen(projectId: string, key: string, nowMs: number): boolean { - const k = `${projectId}:${key}`; +function dueForLastSeen(deviceId: string, key: string, nowMs: number): boolean { + const k = `${deviceId}:${key}`; const prev = lastSeenWrites.get(k); if (prev !== undefined && nowMs - prev < LAST_SEEN_THROTTLE_MS) return false; lastSeenWrites.set(k, nowMs); @@ -27,11 +27,12 @@ function dueForLastSeen(projectId: string, key: string, nowMs: number): boolean export async function upsertVariables( env: Env, projectId: string, + deviceId: string, keys: string[], now: number ): Promise { const nowMs = Date.now(); - const due = [...new Set(keys)].filter((key) => dueForLastSeen(projectId, key, nowMs)); + const due = [...new Set(keys)].filter((key) => dueForLastSeen(deviceId, key, nowMs)); if (due.length === 0) return; try { // Existing keys only refresh last_seen; new keys count against the cap. Chunk the @@ -40,8 +41,8 @@ export async function upsertVariables( for (const part of chunk(due, MAX_BOUND_PARAMS - 1)) { const placeholders = part.map(() => '?').join(','); const rows = await env.DB - .prepare(`SELECT key FROM project_variables WHERE project_id = ? AND key IN (${placeholders})`) - .bind(projectId, ...part) + .prepare(`SELECT key FROM project_variables WHERE device_id = ? AND key IN (${placeholders})`) + .bind(deviceId, ...part) .all<{ key: string }>(); for (const r of rows.results) existing.add(r.key); } @@ -61,15 +62,15 @@ export async function upsertVariables( keysToWrite.map((key) => env.DB .prepare( - `INSERT INTO project_variables (id, project_id, key, created_at, updated_at, last_seen) - VALUES (?, ?, ?, ?, ?, ?) - ON CONFLICT(project_id, key) DO UPDATE SET last_seen = excluded.last_seen` + `INSERT INTO project_variables (id, project_id, device_id, key, created_at, updated_at, last_seen) + VALUES (?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(project_id, device_id, key) DO UPDATE SET last_seen = excluded.last_seen` ) - .bind(newId('variable'), projectId, key, now, now, now) + .bind(newId('variable'), projectId, deviceId, key, now, now, now) ) ); } catch { // Best-effort — never fail telemetry on it. Clear stamps so a failed write retries next ingest. - for (const key of due) lastSeenWrites.delete(`${projectId}:${key}`); + for (const key of due) lastSeenWrites.delete(`${deviceId}:${key}`); } } diff --git a/worker/src/domains/variables/service.ts b/worker/src/domains/variables/service.ts index d313ab5..3d0e39b 100644 --- a/worker/src/domains/variables/service.ts +++ b/worker/src/domains/variables/service.ts @@ -4,6 +4,7 @@ import { recordAudit } from '../../platform/lib/audit'; import { projectStub } from '../../platform/durable-objects/stubs'; import { type Actor, ServiceError } from '../../platform/lib/service'; import { assertProjectAccess } from '../projects/service'; +import { defaultDeviceId } from '../devices/service'; export type VariableSummary = { id: string; @@ -68,13 +69,17 @@ export async function createVariable( const id = newId('variable'); const now = Math.floor(Date.now() / 1000); + // Hand-declared variables belong to the default device; a board that reports + // the same key under its own identity gets its own row. + const deviceId = await defaultDeviceId(env, projectId); + if (!deviceId) throw new ServiceError('not_found', 'project has no default device', 'no_default_device'); try { await env.DB .prepare( - `INSERT INTO project_variables (id, project_id, key, unit, created_at, updated_at) - VALUES (?, ?, ?, ?, ?, ?)` + `INSERT INTO project_variables (id, project_id, device_id, key, unit, created_at, updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?)` ) - .bind(id, projectId, key, input.unit ?? null, now, now) + .bind(id, projectId, deviceId, key, input.unit ?? null, now, now) .run(); } catch { throw new ServiceError('conflict', 'a variable with this key already exists', 'duplicate_key'); diff --git a/worker/src/platform/db/migrations.gen.ts b/worker/src/platform/db/migrations.gen.ts index 9fa0820..8988702 100644 --- a/worker/src/platform/db/migrations.gen.ts +++ b/worker/src/platform/db/migrations.gen.ts @@ -51,8 +51,9 @@ export const MIGRATIONS: Migration[] = [ { "name": "0002_devices", "statements": [ - "CREATE TABLE IF NOT EXISTS devices (\n id TEXT PRIMARY KEY,\n project_id TEXT NOT NULL REFERENCES projects(id) ON DELETE CASCADE,\n name TEXT NOT NULL,\n chip TEXT,\n firmware_version TEXT,\n is_default INTEGER NOT NULL DEFAULT 0,\n first_seen INTEGER,\n last_seen INTEGER,\n created_at INTEGER NOT NULL\n)", + "CREATE TABLE IF NOT EXISTS devices (\n id TEXT PRIMARY KEY,\n project_id TEXT NOT NULL REFERENCES projects(id) ON DELETE CASCADE,\n name TEXT NOT NULL,\n device_key TEXT,\n chip TEXT,\n firmware_version TEXT,\n is_default INTEGER NOT NULL DEFAULT 0,\n first_seen INTEGER,\n last_seen INTEGER,\n created_at INTEGER NOT NULL\n)", "CREATE INDEX IF NOT EXISTS idx_devices_project ON devices(project_id)", + "CREATE UNIQUE INDEX IF NOT EXISTS idx_devices_key\n ON devices(project_id, device_key) WHERE device_key IS NOT NULL", "CREATE UNIQUE INDEX IF NOT EXISTS idx_devices_default\n ON devices(project_id) WHERE is_default = 1", "INSERT INTO devices (id, project_id, name, is_default, created_at)\nSELECT 'dev_' || substr(p.id, 5), p.id, 'Default', 1, p.created_at\nFROM projects p\nWHERE NOT EXISTS (SELECT 1 FROM devices d WHERE d.project_id = p.id AND d.is_default = 1)", "CREATE TABLE project_variables_new (\n id TEXT PRIMARY KEY,\n project_id TEXT NOT NULL REFERENCES projects(id) ON DELETE CASCADE,\n device_id TEXT NOT NULL REFERENCES devices(id) ON DELETE CASCADE,\n key TEXT NOT NULL,\n unit TEXT,\n created_at INTEGER NOT NULL,\n updated_at INTEGER NOT NULL,\n last_seen INTEGER\n)", diff --git a/worker/src/platform/db/migrations/0002_devices.sql b/worker/src/platform/db/migrations/0002_devices.sql index 1478cc6..7f23664 100644 --- a/worker/src/platform/db/migrations/0002_devices.sql +++ b/worker/src/platform/db/migrations/0002_devices.sql @@ -3,6 +3,7 @@ CREATE TABLE IF NOT EXISTS devices ( id TEXT PRIMARY KEY, project_id TEXT NOT NULL REFERENCES projects(id) ON DELETE CASCADE, name TEXT NOT NULL, + device_key TEXT, chip TEXT, firmware_version TEXT, is_default INTEGER NOT NULL DEFAULT 0, @@ -12,6 +13,11 @@ CREATE TABLE IF NOT EXISTS devices ( ); CREATE INDEX IF NOT EXISTS idx_devices_project ON devices(project_id); +-- What the board calls itself, kept apart from the id so renaming is free and a +-- MAC never leaks into anything a user reads. NULL on the default device. +CREATE UNIQUE INDEX IF NOT EXISTS idx_devices_key + ON devices(project_id, device_key) WHERE device_key IS NOT NULL; + -- Exactly one default per project, enforced rather than assumed. CREATE UNIQUE INDEX IF NOT EXISTS idx_devices_default ON devices(project_id) WHERE is_default = 1; diff --git a/worker/src/platform/durable-objects/project-do.ts b/worker/src/platform/durable-objects/project-do.ts index bdf95c9..945f856 100644 --- a/worker/src/platform/durable-objects/project-do.ts +++ b/worker/src/platform/durable-objects/project-do.ts @@ -10,6 +10,7 @@ import { toCompactSeries, type CompactSeries } from '../lib/series'; import { chunk, MAX_BOUND_PARAMS } from '../lib/sql'; import { parseDeviceMessage } from '../../domains/telemetry/ws-protocol'; import { upsertVariables } from '../../domains/telemetry/variables'; +import { defaultDeviceId } from '../../domains/devices/service'; import { migrateSchema, type SchemaStep } from './schema'; // Project Durable Object (one per project id, SQLite-backed): latest variable @@ -515,7 +516,11 @@ export class ProjectDO extends DurableObject { const pid = this.projectId(); await this.ingest(pid, msg.points); const now = Math.floor(Date.now() / 1000); - this.ctx.waitUntil(upsertVariables(this.env, pid, msg.points.map((p) => p.variable), now)); + this.ctx.waitUntil( + defaultDeviceId(this.env, pid).then((deviceId) => + deviceId ? upsertVariables(this.env, pid, deviceId, msg.points.map((p) => p.variable), now) : undefined + ) + ); return; } case 'event': diff --git a/worker/src/platform/lib/ids.ts b/worker/src/platform/lib/ids.ts index 7ca5a2d..0f4761c 100644 --- a/worker/src/platform/lib/ids.ts +++ b/worker/src/platform/lib/ids.ts @@ -7,6 +7,7 @@ import { nanoid } from 'nanoid'; // tok_xxx token ctl_xxx control write // aut_xxx automation itg_xxx integration // wid_xxx widget instance (inside a dashboard layout) +// dev_xxx device // dly_xxx delay continuation (pending automation resume) const PREFIXES = { @@ -20,6 +21,7 @@ const PREFIXES = { integration: 'itg', widget: 'wid', delay: 'dly', + device: 'dev', } as const; export type IdKind = keyof typeof PREFIXES; From b1e458d4c83f9754d0f2b80fce11f7b4c852a7ad Mon Sep 17 00:00:00 2001 From: Arjun Krishna Date: Sat, 22 Aug 2026 19:23:03 +0530 Subject: [PATCH 10/34] feat(do): key project storage by device MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit latest_state is rebuilt on (device_id, variable), ring_buffer and pending_control gain the column, and the ring index follows. '' marks the project's default device and every pre-devices row backfills to it. NULL cannot serve there: SQLite treats NULLs as distinct in a unique key, so the upsert would never match and each write would append a duplicate row instead of updating one. pending_control does use NULL, where it means a control write with no target that still broadcasts. R2 keeps its shape — the device goes in the NDJSON row, not the key path, so an hour isn't fragmented into one small object per device. Rows written before this read back as the default device. Series reads take an optional device filter; omitting it queries the whole project, which is what dashboards still do. --- .../platform/durable-objects/project-do.ts | 125 ++++++++++++------ 1 file changed, 81 insertions(+), 44 deletions(-) diff --git a/worker/src/platform/durable-objects/project-do.ts b/worker/src/platform/durable-objects/project-do.ts index 945f856..3b0b823 100644 --- a/worker/src/platform/durable-objects/project-do.ts +++ b/worker/src/platform/durable-objects/project-do.ts @@ -18,8 +18,8 @@ import { migrateSchema, type SchemaStep } from './schema'; const RING_BUFFER_MAX_ROWS = 1000; const RING_BUFFER_MAX_AGE_SECONDS = 60 * 60; // 1 hour -// Multi-row inserts bind 3 columns/row; this keeps a chunk under MAX_BOUND_PARAMS. -const ROWS_PER_3COL_INSERT = Math.floor(MAX_BOUND_PARAMS / 3); +// Multi-row inserts bind 4 columns/row; this keeps a chunk under MAX_BOUND_PARAMS. +const ROWS_PER_4COL_INSERT = Math.floor(MAX_BOUND_PARAMS / 4); // Eviction (age + overflow DELETE + COUNT) runs at most this often instead of on // every ingest — a single cheap last_evict_at lookup gates the actual work. const EVICT_INTERVAL_SECONDS = 30; @@ -40,6 +40,7 @@ export type IngestResult = { }; export type LatestStateRow = { + device_id: string; variable: string; value: unknown; received_at: number; @@ -105,6 +106,32 @@ const SCHEMA: SchemaStep[] = [ ); `); }, + // '' is the default device. NULL can't serve: SQLite treats NULLs as distinct + // in a unique key, so the upsert would duplicate on every write. + (sql) => { + sql.exec(` + CREATE TABLE latest_state_new ( + device_id TEXT NOT NULL DEFAULT '', + variable TEXT NOT NULL, + value TEXT NOT NULL, + received_at INTEGER NOT NULL, + PRIMARY KEY (device_id, variable) + ); + `); + sql.exec(` + INSERT INTO latest_state_new (device_id, variable, value, received_at) + SELECT '', variable, value, received_at FROM latest_state; + `); + sql.exec(`DROP TABLE latest_state;`); + sql.exec(`ALTER TABLE latest_state_new RENAME TO latest_state;`); + + sql.exec(`ALTER TABLE ring_buffer ADD COLUMN device_id TEXT NOT NULL DEFAULT '';`); + sql.exec(`DROP INDEX IF EXISTS idx_ring_buffer_var_ts;`); + sql.exec(`CREATE INDEX IF NOT EXISTS idx_ring_buffer_dev_var_ts ON ring_buffer(device_id, variable, ts);`); + + // NULL device still broadcasts. + sql.exec(`ALTER TABLE pending_control ADD COLUMN device_id TEXT;`); + }, ]; export class ProjectDO extends DurableObject { @@ -132,7 +159,7 @@ export class ProjectDO extends DurableObject { ); } - async ingest(projectId: string, points: IngestPoint[], _deviceTs?: number): Promise { + async ingest(projectId: string, points: IngestPoint[], deviceId = ''): Promise { const receivedAt = Math.floor(Date.now() / 1000); // Persist project_id once for the R2 key (idFromName doesn't round-trip cheaply). @@ -152,7 +179,8 @@ export class ProjectDO extends DurableObject { const placeholders = part.map(() => '?').join(','); const rows = this.sql .exec<{ variable: string; value: string }>( - `SELECT variable, value FROM latest_state WHERE variable IN (${placeholders})`, + `SELECT variable, value FROM latest_state WHERE device_id = ? AND variable IN (${placeholders})`, + deviceId, ...part ) .toArray(); @@ -163,28 +191,28 @@ export class ProjectDO extends DurableObject { // can't have two VALUES rows hit the same conflict target). const latestByVar = new Map(); for (const p of points) latestByVar.set(p.variable, p.value); - for (const part of chunk([...latestByVar], ROWS_PER_3COL_INSERT)) { - const rows = part.map(() => '(?, ?, ?)').join(', '); + for (const part of chunk([...latestByVar], ROWS_PER_4COL_INSERT)) { + const rows = part.map(() => '(?, ?, ?, ?)').join(', '); const binds: unknown[] = []; for (const [variable, value] of part) { - binds.push(variable, JSON.stringify(value), receivedAt); + binds.push(deviceId, variable, JSON.stringify(value), receivedAt); } this.sql.exec( - `INSERT INTO latest_state (variable, value, received_at) + `INSERT INTO latest_state (device_id, variable, value, received_at) VALUES ${rows} - ON CONFLICT(variable) DO UPDATE SET value = excluded.value, received_at = excluded.received_at`, + ON CONFLICT(device_id, variable) DO UPDATE SET value = excluded.value, received_at = excluded.received_at`, ...binds ); } // ring_buffer: append every point. - for (const part of chunk(points, ROWS_PER_3COL_INSERT)) { - const rows = part.map(() => '(?, ?, ?)').join(', '); + for (const part of chunk(points, ROWS_PER_4COL_INSERT)) { + const rows = part.map(() => '(?, ?, ?, ?)').join(', '); const binds: unknown[] = []; for (const p of part) { - binds.push(receivedAt, p.variable, JSON.stringify(p.value)); + binds.push(receivedAt, deviceId, p.variable, JSON.stringify(p.value)); } - this.sql.exec(`INSERT INTO ring_buffer (ts, variable, value) VALUES ${rows}`, ...binds); + this.sql.exec(`INSERT INTO ring_buffer (ts, device_id, variable, value) VALUES ${rows}`, ...binds); } // Eviction is independent of the flush cursor (copy-not-move). @@ -338,11 +366,13 @@ export class ProjectDO extends DurableObject { async getLatestState(): Promise { const rows = this.sql - .exec<{ variable: string; value: string; received_at: number }>( - `SELECT variable, value, received_at FROM latest_state ORDER BY variable ASC` + .exec<{ device_id: string; variable: string; value: string; received_at: number }>( + `SELECT device_id, variable, value, received_at FROM latest_state + ORDER BY device_id ASC, variable ASC` ) .toArray(); return rows.map((r) => ({ + device_id: r.device_id, variable: r.variable, value: safeParse(r.value), received_at: r.received_at, @@ -354,18 +384,21 @@ export class ProjectDO extends DurableObject { async getSeriesForVariables( variables: string[], sinceTs: number | null, - cap?: number + cap?: number, + deviceId?: string | null ): Promise { if (variables.length === 0) return {}; const cutoff = sinceTs ?? 0; const placeholders = variables.map(() => '?').join(','); + const scoped = deviceId != null; const rows = this.sql .exec<{ ts: number; variable: string; value: string }>( `SELECT ts, variable, value FROM ring_buffer - WHERE variable IN (${placeholders}) AND ts >= ? + WHERE variable IN (${placeholders}) AND ts >= ?${scoped ? ' AND device_id = ?' : ''} ORDER BY ts ASC`, ...variables, - cutoff + cutoff, + ...(scoped ? [deviceId] : []) ) .toArray(); return toCompactSeries( @@ -374,26 +407,24 @@ export class ProjectDO extends DurableObject { ); } - async getSeries(variable: string | null, sinceTs: number | null): Promise { + async getSeries( + variable: string | null, + sinceTs: number | null, + deviceId?: string | null + ): Promise { const cutoff = sinceTs ?? 0; - const rows = variable - ? this.sql - .exec<{ ts: number; variable: string; value: string }>( - `SELECT ts, variable, value FROM ring_buffer - WHERE variable = ? AND ts >= ? - ORDER BY ts ASC`, - variable, - cutoff - ) - .toArray() - : this.sql - .exec<{ ts: number; variable: string; value: string }>( - `SELECT ts, variable, value FROM ring_buffer - WHERE ts >= ? - ORDER BY ts ASC`, - cutoff - ) - .toArray(); + const where = ['ts >= ?']; + const binds: unknown[] = [cutoff]; + if (variable) { where.push('variable = ?'); binds.push(variable); } + if (deviceId != null) { where.push('device_id = ?'); binds.push(deviceId); } + const rows = this.sql + .exec<{ ts: number; variable: string; value: string }>( + `SELECT ts, variable, value FROM ring_buffer + WHERE ${where.join(' AND ')} + ORDER BY ts ASC`, + ...binds + ) + .toArray(); return rows.map((r) => ({ ts: r.ts, variable: r.variable, value: safeParse(r.value) })); } @@ -456,10 +487,16 @@ export class ProjectDO extends DurableObject { // Drops hot state for a variable. Cold R2 history (partitioned by project+hour) // is left in place — orphaned but harmless. - async deleteVariable(variable: string): Promise { - this.sql.exec(`DELETE FROM latest_state WHERE variable = ?`, variable); - this.sql.exec(`DELETE FROM ring_buffer WHERE variable = ?`, variable); - this.sql.exec(`DELETE FROM pending_control WHERE variable = ?`, variable); + async deleteVariable(variable: string, deviceId?: string | null): Promise { + if (deviceId == null) { + this.sql.exec(`DELETE FROM latest_state WHERE variable = ?`, variable); + this.sql.exec(`DELETE FROM ring_buffer WHERE variable = ?`, variable); + this.sql.exec(`DELETE FROM pending_control WHERE variable = ?`, variable); + return; + } + this.sql.exec(`DELETE FROM latest_state WHERE variable = ? AND device_id = ?`, variable, deviceId); + this.sql.exec(`DELETE FROM ring_buffer WHERE variable = ? AND device_id = ?`, variable, deviceId); + this.sql.exec(`DELETE FROM pending_control WHERE variable = ? AND device_id IS ?`, variable, deviceId); } async flushNow(): Promise { @@ -614,8 +651,8 @@ export class ProjectDO extends DurableObject { private async runFlush(): Promise { const cursor = this.getFlushCursor(); const rows = this.sql - .exec<{ rowid: number; ts: number; variable: string; value: string }>( - `SELECT rowid, ts, variable, value FROM ring_buffer WHERE rowid > ? ORDER BY rowid ASC`, + .exec<{ rowid: number; ts: number; device_id: string; variable: string; value: string }>( + `SELECT rowid, ts, device_id, variable, value FROM ring_buffer WHERE rowid > ? ORDER BY rowid ASC`, cursor ) .toArray(); @@ -639,7 +676,7 @@ export class ProjectDO extends DurableObject { const key = `telemetry/${projectId}/${bucket}/r-${lastRowid.toString().padStart(12, '0')}.ndjson`; const body = bucketRows .map((r) => - JSON.stringify({ ts: r.ts, variable: r.variable, value: safeParse(r.value) }) + JSON.stringify({ ts: r.ts, device: r.device_id || null, variable: r.variable, value: safeParse(r.value) }) ) .join('\n') + '\n'; From 54c5073980c614f40e470c62bee8dde1655a1af3 Mon Sep 17 00:00:00 2001 From: Arjun Krishna Date: Sat, 22 Aug 2026 19:30:10 +0530 Subject: [PATCH 11/34] feat(worker): make devices addressable end to end MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ingest now carries the device through to Durable Object storage, and /state groups by device rather than returning a flat key map. Those had to change together: two boards reporting the same key into a flat map means the second silently overwrites the first, which is the whole point of having devices. resolveDevice returns both ids it is asked for — the D1 id that owns the variable rows, and the storage id the DO keys by, which is '' for the default device. Callers get what they need without either of them knowing the rule. Adds list, rename and forget. The default device can't be forgotten; without one, telemetry that names no device has nowhere to land. Forgetting drops D1 rows and DO state but leaves R2 history, which is only safe because the device is named in each row rather than in the key path. --- worker/src/domains/devices/routes.ts | 58 +++++++++++++ worker/src/domains/devices/service.ts | 84 +++++++++++++++---- worker/src/domains/telemetry/telemetry.ts | 8 +- worker/src/domains/variables/service.ts | 31 +++++-- .../platform/durable-objects/project-do.ts | 6 ++ worker/src/platform/lib/audit.ts | 1 + worker/src/routes.ts | 2 + 7 files changed, 160 insertions(+), 30 deletions(-) create mode 100644 worker/src/domains/devices/routes.ts diff --git a/worker/src/domains/devices/routes.ts b/worker/src/domains/devices/routes.ts new file mode 100644 index 0000000..dd4fffa --- /dev/null +++ b/worker/src/domains/devices/routes.ts @@ -0,0 +1,58 @@ +import { Hono } from 'hono'; +import type { Env } from '../../env'; +import { requireSession } from '../../platform/middleware/require-session'; +import { resolveProject, type ProjectContextVars } from '../../platform/middleware/resolve-project'; +import { recordAudit } from '../../platform/lib/audit'; +import { listDevices, renameDevice, forgetDevice } from './service'; +import { actorFromSession, serviceErrorResponse } from '../../platform/lib/service'; + +const devices = new Hono<{ Bindings: Env; Variables: ProjectContextVars }>(); + +devices.use('*', requireSession); +devices.use('*', resolveProject); + +devices.get('/', async (c) => { + const project = c.get('project'); + return c.json({ devices: await listDevices(c.env, project.id) }); +}); + +devices.patch('/:id', async (c) => { + const project = c.get('project'); + const id = c.req.param('id'); + const body = await c.req.json<{ name?: string }>(); + try { + const device = await renameDevice(c.env, project.id, id, body.name ?? ''); + await recordAudit(c.env, { + projectId: project.id, + userId: c.get('user').id, + action: 'device.rename', + targetType: 'device', + targetId: id, + metadata: { name: device.name }, + }); + return c.json({ device }); + } catch (e) { + return serviceErrorResponse(c, e); + } +}); + +devices.delete('/:id', async (c) => { + const project = c.get('project'); + const id = c.req.param('id'); + try { + await forgetDevice(c.env, project.id, id); + await recordAudit(c.env, { + projectId: project.id, + userId: c.get('user').id, + action: 'device.forget', + targetType: 'device', + targetId: id, + metadata: { source: actorFromSession(c.get('user')).source }, + }); + return c.body(null, 204); + } catch (e) { + return serviceErrorResponse(c, e); + } +}); + +export default devices; diff --git a/worker/src/domains/devices/service.ts b/worker/src/domains/devices/service.ts index bf74a80..1a6dd65 100644 --- a/worker/src/domains/devices/service.ts +++ b/worker/src/domains/devices/service.ts @@ -1,5 +1,11 @@ import type { Env } from '../../env'; import { newId } from '../../platform/lib/ids'; +import { ServiceError } from '../../platform/lib/service'; +import { projectStub } from '../../platform/durable-objects/stubs'; + +// storageId is '' for the default device — the DO keys rows by it, so pre-devices +// rows need no backfill. +export type ResolvedDevice = { id: string; storageId: string }; export type DeviceSummary = { id: string; @@ -11,12 +17,9 @@ export type DeviceSummary = { last_seen: number | null; }; -// Bounds device growth from a board that reports a fresh key every boot. const MAX_DEVICES_PER_PROJECT = 100; const MAX_DEVICE_KEY_LEN = 64; -// Device ids change only when a device is forgotten, so an isolate can hold the -// project -> device mapping for as long as it lives. const resolved = new Map(); export function forgetCachedDevice(projectId: string, deviceKey?: string | null) { @@ -29,7 +32,6 @@ function cacheKey(projectId: string, deviceKey: string | null): string { return `${projectId}:${deviceKey ?? ''}`; } -// A board picks its own key, so it has to be treated as untrusted input. export function normaliseDeviceKey(raw: string | null | undefined): string | null { if (typeof raw !== 'string') return null; const trimmed = raw.trim(); @@ -48,8 +50,7 @@ export async function defaultDeviceId(env: Env, projectId: string): Promise { - if (!deviceKey) return defaultDeviceId(env, projectId); +): Promise { + if (!deviceKey) { + const id = await defaultDeviceId(env, projectId); + return id ? { id, storageId: '' } : null; + } const cached = resolved.get(cacheKey(projectId, deviceKey)); - if (cached) return cached; + if (cached) return { id: cached, storageId: cached }; const existing = await env.DB .prepare(`SELECT id FROM devices WHERE project_id = ? AND device_key = ?`) @@ -78,14 +81,17 @@ export async function resolveDevice( .first<{ id: string }>(); if (existing) { resolved.set(cacheKey(projectId, deviceKey), existing.id); - return existing.id; + return { id: existing.id, storageId: existing.id }; } const count = await env.DB .prepare(`SELECT COUNT(*) AS n FROM devices WHERE project_id = ?`) .bind(projectId) .first<{ n: number }>(); - if ((count?.n ?? 0) >= MAX_DEVICES_PER_PROJECT) return defaultDeviceId(env, projectId); + if ((count?.n ?? 0) >= MAX_DEVICES_PER_PROJECT) { + const id = await defaultDeviceId(env, projectId); + return id ? { id, storageId: '' } : null; + } const id = newId('device'); await env.DB @@ -97,14 +103,14 @@ export async function resolveDevice( .bind(id, projectId, deviceKey, deviceKey, now, now, now) .run(); - // A concurrent isolate may have won the insert, so read back rather than - // assuming the id we generated is the one that stuck. + // A concurrent isolate may have won the insert, so read back the id that stuck. const settled = await env.DB .prepare(`SELECT id FROM devices WHERE project_id = ? AND device_key = ?`) .bind(projectId, deviceKey) .first<{ id: string }>(); - if (settled) resolved.set(cacheKey(projectId, deviceKey), settled.id); - return settled?.id ?? null; + if (!settled) return null; + resolved.set(cacheKey(projectId, deviceKey), settled.id); + return { id: settled.id, storageId: settled.id }; } export async function listDevices(env: Env, projectId: string): Promise { @@ -117,3 +123,47 @@ export async function listDevices(env: Env, projectId: string): Promise(); return rows.results; } + +const MAX_DEVICE_NAME_LEN = 60; + +export async function renameDevice( + env: Env, + projectId: string, + id: string, + rawName: string +): Promise { + const name = rawName.trim().slice(0, MAX_DEVICE_NAME_LEN); + if (!name) throw new ServiceError('bad_request', 'name is required', 'missing_name'); + const res = await env.DB + .prepare(`UPDATE devices SET name = ? WHERE id = ? AND project_id = ?`) + .bind(name, id, projectId) + .run(); + if (res.meta.changes === 0) throw new ServiceError('not_found', 'no such device', 'unknown_device'); + const row = await env.DB + .prepare( + `SELECT id, name, chip, firmware_version, is_default, first_seen, last_seen + FROM devices WHERE id = ?` + ) + .bind(id) + .first(); + return row!; +} + +// R2 history stays readable without the row — the device is named in each row. +export async function forgetDevice(env: Env, projectId: string, id: string): Promise { + const row = await env.DB + .prepare(`SELECT is_default, device_key FROM devices WHERE id = ? AND project_id = ?`) + .bind(id, projectId) + .first<{ is_default: number; device_key: string | null }>(); + if (!row) throw new ServiceError('not_found', 'no such device', 'unknown_device'); + if (row.is_default) { + throw new ServiceError('conflict', 'the default device cannot be forgotten', 'default_device'); + } + + await env.DB.batch([ + env.DB.prepare(`DELETE FROM project_variables WHERE device_id = ?`).bind(id), + env.DB.prepare(`DELETE FROM devices WHERE id = ? AND project_id = ?`).bind(id, projectId), + ]); + forgetCachedDevice(projectId, row.device_key); + await projectStub(env, projectId).deleteDevice(id); +} diff --git a/worker/src/domains/telemetry/telemetry.ts b/worker/src/domains/telemetry/telemetry.ts index 180494e..3ec6130 100644 --- a/worker/src/domains/telemetry/telemetry.ts +++ b/worker/src/domains/telemetry/telemetry.ts @@ -43,15 +43,15 @@ telemetry.post('/', async (c) => { const { project_id } = c.get('projectToken'); const now = Math.floor(Date.now() / 1000); const deviceKey = normaliseDeviceKey(c.req.header('x-nodrix-device')); - const deviceId = await resolveDevice(c.env, project_id, deviceKey, now); + const device = await resolveDevice(c.env, project_id, deviceKey, now); const stub = projectStub(c.env, project_id); - await stub.ingest(project_id, points); + await stub.ingest(project_id, points, device?.storageId ?? ''); // Auto-create new variables + bump last_seen off the response path (best-effort). - if (deviceId) { + if (device) { c.executionCtx.waitUntil( - upsertVariables(c.env, project_id, deviceId, points.map((p) => p.variable), now) + upsertVariables(c.env, project_id, device.id, points.map((p) => p.variable), now) ); } diff --git a/worker/src/domains/variables/service.ts b/worker/src/domains/variables/service.ts index 3d0e39b..ccb9751 100644 --- a/worker/src/domains/variables/service.ts +++ b/worker/src/domains/variables/service.ts @@ -4,7 +4,7 @@ import { recordAudit } from '../../platform/lib/audit'; import { projectStub } from '../../platform/durable-objects/stubs'; import { type Actor, ServiceError } from '../../platform/lib/service'; import { assertProjectAccess } from '../projects/service'; -import { defaultDeviceId } from '../devices/service'; +import { defaultDeviceId, listDevices } from '../devices/service'; export type VariableSummary = { id: string; @@ -28,13 +28,27 @@ export async function listVariables(env: Env, projectId: string): Promise }; -// Latest value of every variable. Mirrors GET /v1/projects/:proj/state. -export async function getState(env: Env, projectId: string): Promise> { - const latest = await projectStub(env, projectId).getLatestState(); - const out: Record = {}; - for (const r of latest) out[r.variable] = { value: r.value, received_at: r.received_at }; - return out; +// Mirrors GET /v1/projects/:proj/state. Grouped by device because two of them +// may report the same key, and a flat map would drop one. +export async function getState(env: Env, projectId: string): Promise { + const [latest, devices] = await Promise.all([ + projectStub(env, projectId).getLatestState(), + listDevices(env, projectId), + ]); + const fallback = devices.find((d) => d.is_default)?.id; + const byDevice = new Map(); + for (const d of devices) byDevice.set(d.id, { id: d.id, name: d.name, variables: {} }); + + for (const r of latest) { + // The DO marks the default device '' so it never had to backfill. + const id = r.device_id || fallback; + if (!id) continue; + const bucket = byDevice.get(id); + if (bucket) bucket.variables[r.variable] = { value: r.value, received_at: r.received_at }; + } + return [...byDevice.values()]; } // Recent points from the Project DO ring buffer (never R2). Mirrors @@ -69,8 +83,7 @@ export async function createVariable( const id = newId('variable'); const now = Math.floor(Date.now() / 1000); - // Hand-declared variables belong to the default device; a board that reports - // the same key under its own identity gets its own row. + // Hand-declared variables belong to the default device. const deviceId = await defaultDeviceId(env, projectId); if (!deviceId) throw new ServiceError('not_found', 'project has no default device', 'no_default_device'); try { diff --git a/worker/src/platform/durable-objects/project-do.ts b/worker/src/platform/durable-objects/project-do.ts index 3b0b823..e994147 100644 --- a/worker/src/platform/durable-objects/project-do.ts +++ b/worker/src/platform/durable-objects/project-do.ts @@ -499,6 +499,12 @@ export class ProjectDO extends DurableObject { this.sql.exec(`DELETE FROM pending_control WHERE variable = ? AND device_id IS ?`, variable, deviceId); } + async deleteDevice(deviceId: string): Promise { + this.sql.exec(`DELETE FROM latest_state WHERE device_id = ?`, deviceId); + this.sql.exec(`DELETE FROM ring_buffer WHERE device_id = ?`, deviceId); + this.sql.exec(`DELETE FROM pending_control WHERE device_id = ?`, deviceId); + } + async flushNow(): Promise { return this.runFlush(); } diff --git a/worker/src/platform/lib/audit.ts b/worker/src/platform/lib/audit.ts index 86108e8..c1405d9 100644 --- a/worker/src/platform/lib/audit.ts +++ b/worker/src/platform/lib/audit.ts @@ -21,6 +21,7 @@ export async function isAuditEnabled(env: Env): Promise { export type AuditTargetType = | 'project' | 'variable' + | 'device' | 'project_token' | 'dashboard' | 'token' diff --git a/worker/src/routes.ts b/worker/src/routes.ts index cdc2913..07e452b 100644 --- a/worker/src/routes.ts +++ b/worker/src/routes.ts @@ -18,6 +18,7 @@ import publicInvite from './domains/identity/public-invite'; // projects / variables / dashboards / automations / integrations import projects from './domains/projects/routes'; import variables from './domains/variables/routes'; +import devicesRouter from './domains/devices/routes'; import { readList, readState, readSeries } from './domains/variables/read'; import dashboards from './domains/dashboards/routes'; import publicDashboards from './domains/dashboards/public'; @@ -67,6 +68,7 @@ export function registerRoutes(app: App): void { app.route('/v1/admin/invites', invitesRouter); app.route('/v1/admin/projects', projects); app.route('/v1/admin/projects/:proj/variables', variables); + app.route('/v1/admin/projects/:proj/devices', devicesRouter); app.route('/v1/admin/projects/:proj/dashboards', dashboards); app.route('/v1/admin/projects/:proj/automations', automations); app.route('/v1/admin/projects/:proj/integrations', integrations); From 45585f63b44c805cd95dc2086cad502f937e84b0 Mon Sep 17 00:00:00 2001 From: Arjun Krishna Date: Sat, 22 Aug 2026 19:41:09 +0530 Subject: [PATCH 12/34] feat(worker): let WebSocket boards identify themselves Adds a hello frame carrying the device key and optional chip and firmware. An all-WS board never sends an HTTP header, so without this it had no way to say which board it was and everything landed on the default device. The socket's device is held in serializeAttachment rather than memory: a hibernated Durable Object keeps the attachment but loses anything in an isolate, so an idle board that wakes hours later stays attributed. Connect and hello split the pending-control flush. A socket has to catch up the moment it connects, but its identity isn't known until hello arrives, so it starts as the default device and takes broadcasts and default-device writes, then re-scopes and drains its own queue on hello. The two queries can't overlap, and a hello that fails to resolve returns rather than falling through to the default. Control writes can now target one device; a null target still broadcasts. --- worker/src/domains/devices/service.ts | 20 +++++ worker/src/domains/telemetry/ws-protocol.ts | 8 ++ .../platform/durable-objects/project-do.ts | 73 ++++++++++++------- worker/test/ws-protocol.test.ts | 17 +++++ 4 files changed, 90 insertions(+), 28 deletions(-) diff --git a/worker/src/domains/devices/service.ts b/worker/src/domains/devices/service.ts index 1a6dd65..abb9d7e 100644 --- a/worker/src/domains/devices/service.ts +++ b/worker/src/domains/devices/service.ts @@ -167,3 +167,23 @@ export async function forgetDevice(env: Env, projectId: string, id: string): Pro forgetCachedDevice(projectId, row.device_key); await projectStub(env, projectId).deleteDevice(id); } + +export async function recordDeviceSeen( + env: Env, + id: string, + chip?: string, + firmware?: string +): Promise { + const now = Math.floor(Date.now() / 1000); + await env.DB + .prepare( + `UPDATE devices + SET last_seen = ?, + first_seen = COALESCE(first_seen, ?), + chip = COALESCE(?, chip), + firmware_version = COALESCE(?, firmware_version) + WHERE id = ?` + ) + .bind(now, now, chip ?? null, firmware ?? null, id) + .run(); +} diff --git a/worker/src/domains/telemetry/ws-protocol.ts b/worker/src/domains/telemetry/ws-protocol.ts index 66ee2c1..22c1300 100644 --- a/worker/src/domains/telemetry/ws-protocol.ts +++ b/worker/src/domains/telemetry/ws-protocol.ts @@ -5,6 +5,7 @@ import { parseTelemetryBody } from './validate'; import type { IngestPoint } from '../../platform/durable-objects/project-do'; export type DeviceMessage = + | { kind: 'hello'; device: string; chip?: string; firmware?: string } | { kind: 'ack'; ids: string[] } | { kind: 'telemetry'; points: IngestPoint[] } | { kind: 'event'; event: string; payload?: Record } @@ -24,6 +25,13 @@ export function parseDeviceMessage(raw: string): DeviceMessage { const m = msg as Record; switch (m.type) { + case 'hello': { + const device = typeof m.device === 'string' ? m.device.trim() : ''; + if (!device) return { kind: 'ignore' }; + const chip = typeof m.chip === 'string' ? m.chip.slice(0, 32) : undefined; + const firmware = typeof m.firmware === 'string' ? m.firmware.slice(0, 32) : undefined; + return { kind: 'hello', device, ...(chip ? { chip } : {}), ...(firmware ? { firmware } : {}) }; + } case 'ack': { const ids = Array.isArray(m.ids) ? m.ids.filter((x): x is string => typeof x === 'string') : []; return { kind: 'ack', ids }; diff --git a/worker/src/platform/durable-objects/project-do.ts b/worker/src/platform/durable-objects/project-do.ts index e994147..cac20c3 100644 --- a/worker/src/platform/durable-objects/project-do.ts +++ b/worker/src/platform/durable-objects/project-do.ts @@ -10,7 +10,7 @@ import { toCompactSeries, type CompactSeries } from '../lib/series'; import { chunk, MAX_BOUND_PARAMS } from '../lib/sql'; import { parseDeviceMessage } from '../../domains/telemetry/ws-protocol'; import { upsertVariables } from '../../domains/telemetry/variables'; -import { defaultDeviceId } from '../../domains/devices/service'; +import { defaultDeviceId, normaliseDeviceKey, resolveDevice, recordDeviceSeen } from '../../domains/devices/service'; import { migrateSchema, type SchemaStep } from './schema'; // Project Durable Object (one per project id, SQLite-backed): latest variable @@ -440,21 +440,23 @@ export class ProjectDO extends DurableObject { this.sql.exec(`DELETE FROM subscriptions WHERE dashboard_id = ?`, dashboardId); } - async addControl(id: string, variable: string, value: unknown): Promise { + async addControl(id: string, variable: string, value: unknown, deviceId: string | null = null): Promise { const now = Math.floor(Date.now() / 1000); this.sql.exec( - `INSERT INTO pending_control (id, variable, value, created_at, delivered_at) - VALUES (?, ?, ?, ?, NULL)`, + `INSERT INTO pending_control (id, variable, value, created_at, delivered_at, device_id) + VALUES (?, ?, ?, ?, NULL, ?)`, id, variable, JSON.stringify(value), - now + now, + deviceId ); - // Push to any connected hardware WS clients. If offline, the write stays in - // pending_control and is flushed on next connect. + // Push to connected hardware. If offline, the write stays in pending_control + // and is flushed on next connect. const payload = JSON.stringify({ type: 'control', id, variable, value }); for (const ws of this.ctx.getWebSockets()) { + if (deviceId !== null && this.deviceOf(ws) !== deviceId) continue; try { ws.send(payload); } catch { /* dead socket; ignore */ } } @@ -499,6 +501,28 @@ export class ProjectDO extends DurableObject { this.sql.exec(`DELETE FROM pending_control WHERE variable = ? AND device_id IS ?`, variable, deviceId); } + // Survives hibernation, unlike anything held in memory. + private deviceOf(ws: WebSocket): string { + const att = ws.deserializeAttachment() as { device?: string } | null; + return typeof att?.device === 'string' ? att.device : ''; + } + + private sendPending(ws: WebSocket, where: string, ...binds: unknown[]): void { + const rows = this.sql + .exec<{ id: string; variable: string; value: string }>( + `SELECT id, variable, value FROM pending_control + WHERE delivered_at IS NULL AND ${where} + ORDER BY created_at ASC`, + ...binds + ) + .toArray(); + for (const cmd of rows) { + try { + ws.send(JSON.stringify({ type: 'control', id: cmd.id, variable: cmd.variable, value: safeParse(cmd.value) })); + } catch { /* dead socket; ignore */ } + } + } + async deleteDevice(deviceId: string): Promise { this.sql.exec(`DELETE FROM latest_state WHERE device_id = ?`, deviceId); this.sql.exec(`DELETE FROM ring_buffer WHERE device_id = ?`, deviceId); @@ -522,26 +546,9 @@ export class ProjectDO extends DurableObject { const server = pair[1] as WebSocket; this.ctx.acceptWebSocket(server); - // Flush any pending (undelivered) control writes on connect so a device - // that missed messages while offline catches up immediately. The device - // acks them via `{type:'ack', ids:[...]}` once processed. - const pending = this.sql - .exec<{ id: string; variable: string; value: string }>( - `SELECT id, variable, value FROM pending_control - WHERE delivered_at IS NULL - ORDER BY created_at ASC` - ) - .toArray(); - for (const cmd of pending) { - try { - server.send(JSON.stringify({ - type: 'control', - id: cmd.id, - variable: cmd.variable, - value: safeParse(cmd.value), - })); - } catch { /* ignore */ } - } + // Until a hello arrives the socket counts as the default device. + server.serializeAttachment({ device: '' }); + this.sendPending(server, `(device_id IS NULL OR device_id = '')`); return new Response(null, { status: 101, webSocket: client }); } @@ -552,12 +559,22 @@ export class ProjectDO extends DurableObject { const raw = typeof message === 'string' ? message : new TextDecoder().decode(message); const msg = parseDeviceMessage(raw); switch (msg.kind) { + case 'hello': { + const pid = this.projectId(); + const key = normaliseDeviceKey(msg.device); + const device = key ? await resolveDevice(this.env, pid, key, Math.floor(Date.now() / 1000)) : null; + if (!device || !device.storageId) return; + ws.serializeAttachment({ device: device.storageId }); + this.ctx.waitUntil(recordDeviceSeen(this.env, device.id, msg.chip, msg.firmware)); + this.sendPending(ws, `device_id = ?`, device.storageId); + return; + } case 'ack': if (msg.ids.length > 0) await this.ackControl(msg.ids); return; case 'telemetry': { const pid = this.projectId(); - await this.ingest(pid, msg.points); + await this.ingest(pid, msg.points, this.deviceOf(ws)); const now = Math.floor(Date.now() / 1000); this.ctx.waitUntil( defaultDeviceId(this.env, pid).then((deviceId) => diff --git a/worker/test/ws-protocol.test.ts b/worker/test/ws-protocol.test.ts index c74301c..2820d71 100644 --- a/worker/test/ws-protocol.test.ts +++ b/worker/test/ws-protocol.test.ts @@ -71,7 +71,24 @@ test('event with no name is an error frame', () => { }); test('unknown type is ignored', () => { + expect(parseDeviceMessage(JSON.stringify({ type: 'wat' }))).toEqual({ kind: 'ignore' }); +}); + +test('hello carries the device and its optional metadata', () => { + expect(parseDeviceMessage(JSON.stringify({ type: 'hello', device: 'a4:cf:12:00' }))) + .toEqual({ kind: 'hello', device: 'a4:cf:12:00' }); + expect(parseDeviceMessage(JSON.stringify({ type: 'hello', device: 'b1', chip: 'esp32s3', firmware: '1.4.0' }))) + .toEqual({ kind: 'hello', device: 'b1', chip: 'esp32s3', firmware: '1.4.0' }); +}); + +test('hello without a device is ignored', () => { expect(parseDeviceMessage(JSON.stringify({ type: 'hello' }))).toEqual({ kind: 'ignore' }); + expect(parseDeviceMessage(JSON.stringify({ type: 'hello', device: ' ' }))).toEqual({ kind: 'ignore' }); +}); + +test('hello metadata is length-capped', () => { + const m = parseDeviceMessage(JSON.stringify({ type: 'hello', device: 'b1', chip: 'x'.repeat(80) })); + expect(m.kind === 'hello' && m.chip?.length).toBe(32); }); test('non-JSON is ignored', () => { From 317c912fbde6d06d9e0fda803ad8ff384181d146 Mon Sep 17 00:00:00 2001 From: Arjun Krishna Date: Sat, 22 Aug 2026 19:52:12 +0530 Subject: [PATCH 13/34] feat(web): list, rename and forget devices MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Devices becomes the first tab under Device and the console moves alongside it. Renaming matters more than it looks: a board identifies itself by something MAC-derived, and without a name that string would surface in every variable a user reads. The default device is badged and has no Forget control, mirroring the server rule rather than trusting the client to know it. The forget confirmation spells out what happens, because none of it is guessable: variables and recent history go, archived telemetry stays, and the board reappears if it ever reports again. Focus on the rename input is set from a function ref. A template ref declared inside v-for collects into an array — the compiler decides that by lexical position, so the v-if narrowing it to one row makes no difference — and calling focus() on the array would have thrown. --- web/src/pages/project/device/DeviceHub.vue | 11 +- web/src/pages/project/device/DevicesList.vue | 129 +++++++++++++++++++ web/src/router.ts | 3 +- web/src/stores/project.ts | 32 +++++ web/src/types.ts | 10 ++ 5 files changed, 182 insertions(+), 3 deletions(-) create mode 100644 web/src/pages/project/device/DevicesList.vue diff --git a/web/src/pages/project/device/DeviceHub.vue b/web/src/pages/project/device/DeviceHub.vue index 9dde30f..8905d53 100644 --- a/web/src/pages/project/device/DeviceHub.vue +++ b/web/src/pages/project/device/DeviceHub.vue @@ -9,7 +9,8 @@ const route = useRoute(); const proj = computed(() => project.currentProjectId ?? ''); const tabs = computed(() => [ - { name: 'serial-console', label: 'Console', to: `/p/${proj.value}/device` }, + { name: 'devices', label: 'Devices', to: `/p/${proj.value}/device`, count: project.devices.length }, + { name: 'serial-console', label: 'Console', to: `/p/${proj.value}/device/console` }, ]); @@ -32,7 +33,13 @@ const tabs = computed(() => [ :class="route.name === t.name ? 'border-accent-600 text-accent-700 dark:text-accent-400' : 'border-transparent text-neutral-500 hover:text-neutral-900 dark:text-neutral-400 dark:hover:text-neutral-100'" - >{{ t.label }} + > + {{ t.label }} + {{ t.count }} + diff --git a/web/src/pages/project/device/DevicesList.vue b/web/src/pages/project/device/DevicesList.vue new file mode 100644 index 0000000..30b18a0 --- /dev/null +++ b/web/src/pages/project/device/DevicesList.vue @@ -0,0 +1,129 @@ + + + diff --git a/web/src/router.ts b/web/src/router.ts index 98ecc10..95db78c 100644 --- a/web/src/router.ts +++ b/web/src/router.ts @@ -47,7 +47,8 @@ const routes: RouteRecordRaw[] = [ path: 'device', component: () => import('./pages/project/device/DeviceHub.vue'), children: [ - { path: '', name: 'serial-console', component: () => import('./pages/project/device/SerialConsole.vue'), meta: { title: 'Console' } }, + { path: '', name: 'devices', component: () => import('./pages/project/device/DevicesList.vue'), meta: { title: 'Devices' } }, + { path: 'console', name: 'serial-console', component: () => import('./pages/project/device/SerialConsole.vue'), meta: { title: 'Console' } }, ], }, // Automations + Integrations live under one hub with two tabs. diff --git a/web/src/stores/project.ts b/web/src/stores/project.ts index 3774ae9..65fc1b7 100644 --- a/web/src/stores/project.ts +++ b/web/src/stores/project.ts @@ -8,6 +8,7 @@ import type { AutomationGraph, Dashboard, DashboardMeta, + Device, Variable, ProjectToken, ProjectTokenWithSecret, @@ -23,6 +24,7 @@ export const useProjectStore = defineStore('project', () => { const currentProjectId = ref(null); const variables = ref([]); const projectTokens = ref([]); + const devices = ref([]); const dashboards = ref([]); const tokens = ref([]); const automations = ref([]); @@ -42,6 +44,7 @@ export const useProjectStore = defineStore('project', () => { automations.value = []; integrations.value = []; projectTokens.value = []; + devices.value = []; } currentProjectId.value = projectId; await Promise.all([loadVariables(), loadDashboards()]); @@ -64,6 +67,31 @@ export const useProjectStore = defineStore('project', () => { variables.value = data.variables; } + async function loadDevices(): Promise { + if (!currentProjectId.value) return; + const data = await api.get<{ devices: Device[] }>( + `/v1/admin/projects/${currentProjectId.value}/devices` + ); + devices.value = data.devices; + } + + async function renameDevice(id: string, name: string): Promise { + const pid = requireProjectId(); + const { device } = await api.patch<{ device: Device }>( + `/v1/admin/projects/${pid}/devices/${id}`, + { name } + ); + devices.value = devices.value.map((d) => (d.id === id ? device : d)); + } + + async function forgetDevice(id: string): Promise { + const pid = requireProjectId(); + await api.del(`/v1/admin/projects/${pid}/devices/${id}`); + devices.value = devices.value.filter((d) => d.id !== id); + // Its variables went with it. + await loadVariables(); + } + async function createVariable(input: { key: string; unit?: string | null }): Promise { const pid = requireProjectId(); const v = await api.post( @@ -370,6 +398,7 @@ export const useProjectStore = defineStore('project', () => { return { currentProjectId, variables, + devices, projectTokens, dashboards, tokens, @@ -377,6 +406,9 @@ export const useProjectStore = defineStore('project', () => { integrations, pendingAutomation, switchTo, + loadDevices, + renameDevice, + forgetDevice, loadVariables, createVariable, updateVariable, diff --git a/web/src/types.ts b/web/src/types.ts index a7897fa..5cb570c 100644 --- a/web/src/types.ts +++ b/web/src/types.ts @@ -70,6 +70,16 @@ export type Variable = { last_seen: number | null; }; +export type Device = { + id: string; + name: string; + chip: string | null; + firmware_version: string | null; + is_default: number; + first_seen: number | null; + last_seen: number | null; +}; + export type ProjectToken = { id: string; name?: string | null; From 246264b43fbfd5715ec938fd27a53cdebf9891b5 Mon Sep 17 00:00:00 2001 From: Arjun Krishna Date: Sat, 22 Aug 2026 20:35:46 +0530 Subject: [PATCH 14/34] feat(worker): scope automations and control delivery to a device MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Variable triggers take an optional device, defaulting to any. Both readings are real — one rule per place, or one rule for every place — and neither was expressible before. An automation now stays on the device that fired it: set_variable targets it and condition reads come from its state, so a rule about one greenhouse can't switch the fan in another. Pending control was returned in full to any polling board, so a second device executed writes meant for the first. Listing and acking are both scoped now — a device sees broadcasts and its own, and can't consume another's queue. Triggers name devices by their D1 id while storage calls the default device '', so the id is mapped before comparing. Without that a trigger pinned to the default device matches nothing and the automation silently never fires. --- shared/blocks/index.ts | 1 + shared/blocks/triggers.ts | 6 +++ .../project/automations/AutomationEditor.vue | 1 + .../project/automations/NodeInspector.vue | 12 ++++++ web/src/types.ts | 1 + worker/src/domains/telemetry/control.ts | 17 ++++++++- .../platform/durable-objects/project-do.ts | 37 +++++++++++++------ worker/src/platform/engine/types.ts | 2 + 8 files changed, 63 insertions(+), 14 deletions(-) diff --git a/shared/blocks/index.ts b/shared/blocks/index.ts index 981b588..54f7924 100644 --- a/shared/blocks/index.ts +++ b/shared/blocks/index.ts @@ -26,6 +26,7 @@ export type BlockFieldType = | 'number' | 'boolean' | 'variable' + | 'device' | 'integration' | 'time' | 'weekdays'; diff --git a/shared/blocks/triggers.ts b/shared/blocks/triggers.ts index 788915c..4416573 100644 --- a/shared/blocks/triggers.ts +++ b/shared/blocks/triggers.ts @@ -13,6 +13,12 @@ export const TRIGGER_CATALOG = [ ports: { out: ['out'] }, fields: [ { key: 'variable', label: 'Variable', type: 'variable', required: true }, + { + key: 'device', + label: 'Device', + type: 'device', + hint: 'Leave as any device to fire whichever board reports it.', + }, { key: 'operator', label: 'Condition', diff --git a/web/src/pages/project/automations/AutomationEditor.vue b/web/src/pages/project/automations/AutomationEditor.vue index c2498ba..6a7006d 100644 --- a/web/src/pages/project/automations/AutomationEditor.vue +++ b/web/src/pages/project/automations/AutomationEditor.vue @@ -107,6 +107,7 @@ async function init() { if (!project.currentProjectId) return; await Promise.all([ project.variables.length ? Promise.resolve() : project.loadVariables(), + project.devices.length ? Promise.resolve() : project.loadDevices(), project.loadIntegrations(), ]); diff --git a/web/src/pages/project/automations/NodeInspector.vue b/web/src/pages/project/automations/NodeInspector.vue index 034fbee..a195c2c 100644 --- a/web/src/pages/project/automations/NodeInspector.vue +++ b/web/src/pages/project/automations/NodeInspector.vue @@ -17,6 +17,10 @@ const WEEKDAYS = [ ]; const variableOptions = computed(() => project.variables.map((v) => ({ value: v.key, label: v.key }))); +const deviceOptions = computed(() => [ + { value: '', label: 'Any device' }, + ...project.devices.map((d) => ({ value: d.id, label: d.name })), +]); const integrationOptions = computed(() => project.integrations.map((i) => ({ value: i.id, label: i.name, hint: connSpec(i.kind).label })) ); @@ -97,6 +101,14 @@ watch(() => cfg.value['operation'], () => { if (isCallIntegration.value) seedDef size="sm" class="mt-1 w-full" /> + =' | '<=' | '==' | '!=' | 'changed' export type VariableTriggerConfig = { variable: string; // variable key + device?: string | null; // device id; absent/empty = any device operator: VariableOperator; value?: number | string | boolean; // omitted for 'changed' mode?: 'edge' | 'always'; // edge = fire once on entry (default) diff --git a/worker/src/domains/telemetry/control.ts b/worker/src/domains/telemetry/control.ts index 80a8b9b..b3c36ff 100644 --- a/worker/src/domains/telemetry/control.ts +++ b/worker/src/domains/telemetry/control.ts @@ -3,6 +3,7 @@ import type { Env } from '../../env'; import { requireProjectToken, type ProjectTokenContextVars } from '../../platform/middleware/require-project-token'; import { lookupProjectToken, touchTokenLastUsed } from '../../platform/lib/tokens'; import { projectStub } from '../../platform/durable-objects/stubs'; +import { normaliseDeviceKey, resolveDevice } from '../devices/service'; const control = new Hono<{ Bindings: Env; Variables: ProjectTokenContextVars }>(); @@ -12,8 +13,14 @@ control.use('*', requireProjectToken); // Pending cloud->hardware variable writes for the authenticated project. control.get('/', async (c) => { const { project_id } = c.get('projectToken'); + const device = await resolveDevice( + c.env, + project_id, + normaliseDeviceKey(c.req.header('x-nodrix-device')), + Math.floor(Date.now() / 1000) + ); const stub = projectStub(c.env, project_id); - const pending = await stub.listPendingControl(); + const pending = await stub.listPendingControl(device?.storageId ?? ''); return c.json({ control: pending }); }); @@ -24,8 +31,14 @@ control.post('/ack', async (c) => { if (ids.length === 0) return c.json({ acked: 0 }); const { project_id } = c.get('projectToken'); + const device = await resolveDevice( + c.env, + project_id, + normaliseDeviceKey(c.req.header('x-nodrix-device')), + Math.floor(Date.now() / 1000) + ); const stub = projectStub(c.env, project_id); - const result = await stub.ackControl(ids); + const result = await stub.ackControl(ids, device?.storageId ?? ''); return c.json(result); }); diff --git a/worker/src/platform/durable-objects/project-do.ts b/worker/src/platform/durable-objects/project-do.ts index cac20c3..50a5827 100644 --- a/worker/src/platform/durable-objects/project-do.ts +++ b/worker/src/platform/durable-objects/project-do.ts @@ -221,7 +221,7 @@ export class ProjectDO extends DurableObject { // Fire-and-forget; failures never block ingest — the device already got its 204. this.notifyDashboards(points, receivedAt); - this.evaluateVariableTriggers(projectId, points, prev, receivedAt); + this.evaluateVariableTriggers(projectId, points, prev, receivedAt, deviceId); return { receivedAt, count: points.length }; } @@ -230,7 +230,8 @@ export class ProjectDO extends DurableObject { projectId: string, points: IngestPoint[], prev: Map, - ts: number + ts: number, + deviceId: string ): void { this.ctx.waitUntil( (async () => { @@ -238,17 +239,24 @@ export class ProjectDO extends DurableObject { const autos = await this.getVariableAutomations(projectId); if (autos.length === 0) return; + // Triggers name devices by their D1 id; storage calls the default one ''. + const triggerDevice = deviceId || (await defaultDeviceId(this.env, projectId)) || ''; + + // Both stay on the device that triggered: an automation that reacts to one + // greenhouse shouldn't read or switch another's. const setVariable = (variable: string, value: unknown): Promise => - this.addControl(newId('control'), variable, value); - // Condition nodes read live values; serve them from this DO's own state. + this.addControl(newId('control'), variable, value, deviceId); const getVariable = async (variable: string): Promise => - (await this.getLatestState()).find((r) => r.variable === variable)?.value; + (await this.getLatestState()) + .find((r) => r.device_id === deviceId && r.variable === variable)?.value; for (const a of autos) { for (const node of triggerNodes(toGraph(a))) { if (node.kind !== 'variable') continue; const cfg = node.config as VariableTriggerConfig; + if (cfg.device && cfg.device !== triggerDevice) continue; + const point = points.find((p) => p.variable === cfg.variable); if (!point) continue; if (!matchVariableCondition(cfg, point.value, prev.get(cfg.variable))) continue; @@ -259,6 +267,7 @@ export class ProjectDO extends DurableObject { ts, variable: cfg.variable, value: point.value, + device: triggerDevice, depth: 0, entryNodeId: node.id, }; @@ -463,26 +472,30 @@ export class ProjectDO extends DurableObject { this.notifyDashboards([{ variable, value: value as IngestPoint['value'] }], now); } - async listPendingControl(): Promise> { + // A device sees broadcasts and its own writes, never another device's. + async listPendingControl(deviceId = ''): Promise> { const rows = this.sql .exec<{ id: string; variable: string; value: string }>( `SELECT id, variable, value FROM pending_control - WHERE delivered_at IS NULL - ORDER BY created_at ASC` + WHERE delivered_at IS NULL AND (device_id IS NULL OR device_id = ?) + ORDER BY created_at ASC`, + deviceId ) .toArray(); return rows.map((r) => ({ id: r.id, variable: r.variable, value: safeParse(r.value) })); } - async ackControl(ids: string[]): Promise<{ acked: number }> { + async ackControl(ids: string[], deviceId = ''): Promise<{ acked: number }> { if (ids.length === 0) return { acked: 0 }; const now = Math.floor(Date.now() / 1000); const placeholders = ids.map(() => '?').join(','); const cursor = this.sql.exec( `UPDATE pending_control SET delivered_at = ? - WHERE id IN (${placeholders}) AND delivered_at IS NULL`, + WHERE id IN (${placeholders}) AND delivered_at IS NULL + AND (device_id IS NULL OR device_id = ?)`, now, - ...ids + ...ids, + deviceId ); return { acked: cursor.rowsWritten }; } @@ -570,7 +583,7 @@ export class ProjectDO extends DurableObject { return; } case 'ack': - if (msg.ids.length > 0) await this.ackControl(msg.ids); + if (msg.ids.length > 0) await this.ackControl(msg.ids, this.deviceOf(ws)); return; case 'telemetry': { const pid = this.projectId(); diff --git a/worker/src/platform/engine/types.ts b/worker/src/platform/engine/types.ts index d6216c6..32a8f10 100644 --- a/worker/src/platform/engine/types.ts +++ b/worker/src/platform/engine/types.ts @@ -7,6 +7,7 @@ export type VariableOperator = '>' | '<' | '>=' | '<=' | '==' | '!=' | 'changed' export type VariableTriggerConfig = { variable: string; // variable key + device?: string | null; // device id; absent/null = any device operator: VariableOperator; value?: number | string | boolean; // omitted for 'changed' mode?: 'edge' | 'always'; // edge (default): fire once on entry @@ -49,6 +50,7 @@ export type AutomationContext = { ts: number; // unix seconds variable?: string; value?: unknown; + device?: string; // storage id of the device that triggered event?: string; payload?: Record; depth: number; // emit_event recursion depth From fc63db9ba327d58af49a1b12de210e532cd757ab Mon Sep 17 00:00:00 2001 From: Arjun Krishna Date: Sat, 22 Aug 2026 20:41:21 +0530 Subject: [PATCH 15/34] feat(web): flash a board from the browser MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Drives esptool-js through the port handoff the console already owns, so the monitor tears down cleanly and comes back when the write finishes. Flasher progress is pushed into the same console buffer. It arrives out of band — the port is speaking binary to the ROM loader at the time — and without it the console simply goes dead for twenty seconds mid-session. after() resets the board once writing completes; skipping it leaves the chip in the ROM bootloader until it is physically unplugged, which reads as a bricked board. The transport is disconnected in a finally so a failed flash doesn't leave the port held for the rest of the session. esptool-js is 106 kB, so it stays in the lazily loaded Flash chunk rather than the main bundle. --- web/package.json | 1 + web/src/composables/useEspFlasher.ts | 79 +++++++++++++++ web/src/pages/project/device/DeviceHub.vue | 1 + web/src/pages/project/device/FlashPanel.vue | 103 ++++++++++++++++++++ web/src/router.ts | 1 + 5 files changed, 185 insertions(+) create mode 100644 web/src/composables/useEspFlasher.ts create mode 100644 web/src/pages/project/device/FlashPanel.vue diff --git a/web/package.json b/web/package.json index 80abfed..1e0d201 100644 --- a/web/package.json +++ b/web/package.json @@ -15,6 +15,7 @@ "@nodrix/widgets-shared": "*", "@vue-flow/core": "^1.48.2", "better-auth": "^1.6.11", + "esptool-js": "^0.6.1", "grid-layout-plus": "^1.1.1", "pinia": "^2.3.0", "reka-ui": "^2.0.2", diff --git a/web/src/composables/useEspFlasher.ts b/web/src/composables/useEspFlasher.ts new file mode 100644 index 0000000..27c1eda --- /dev/null +++ b/web/src/composables/useEspFlasher.ts @@ -0,0 +1,79 @@ +import { ref } from 'vue'; +import { ESPLoader, Transport } from 'esptool-js'; +import { useSerialPort, emit } from './useSerialPort'; + +export type FlashPart = { data: Uint8Array; address: number }; + +export type FlashPhase = 'idle' | 'connecting' | 'writing' | 'done' | 'failed'; + +// main() handshakes with the ROM loader at 115200, then negotiates up to this. +const WRITE_BAUD = 460800; + +const phase = ref('idle'); +const progress = ref(0); +const chip = ref(null); +const error = ref(null); + +// esptool-js reports out of band — none of this arrives over the port, which is +// speaking binary to the ROM loader at the time. +const terminal = { + clean: () => {}, + write: (data: string) => { if (data.trim()) emit('flash', data.trim()); }, + writeLine: (data: string) => { if (data.trim()) emit('flash', data.trim()); }, +}; + +export function useEspFlasher() { + const { claim } = useSerialPort(); + + async function flash(parts: FlashPart[]): Promise { + if (!parts.length) throw new Error('Nothing to flash'); + phase.value = 'connecting'; + progress.value = 0; + error.value = null; + + const total = parts.reduce((n, p) => n + p.data.length, 0); + const written = new Map(); + + try { + await claim('flash', async (port) => { + const transport = new Transport(port, false); + const loader = new ESPLoader({ + transport, + baudrate: WRITE_BAUD, + terminal, + }); + try { + chip.value = await loader.main(); + phase.value = 'writing'; + await loader.writeFlash({ + fileArray: parts, + flashMode: 'keep', + flashFreq: 'keep', + flashSize: 'keep', + eraseAll: false, + compress: true, + reportProgress: (fileIndex, bytes) => { + written.set(fileIndex, bytes); + const done = [...written.values()].reduce((n, v) => n + v, 0); + progress.value = total ? Math.min(1, done / total) : 0; + }, + }); + // Without this the board sits in the ROM loader until it's unplugged. + await loader.after(); + } finally { + await transport.disconnect(); + } + }); + phase.value = 'done'; + progress.value = 1; + return true; + } catch (e) { + error.value = (e as Error).message; + emit('flash', `Failed: ${error.value}`); + phase.value = 'failed'; + return false; + } + } + + return { flash, phase, progress, chip, error }; +} diff --git a/web/src/pages/project/device/DeviceHub.vue b/web/src/pages/project/device/DeviceHub.vue index 8905d53..276c72a 100644 --- a/web/src/pages/project/device/DeviceHub.vue +++ b/web/src/pages/project/device/DeviceHub.vue @@ -10,6 +10,7 @@ const proj = computed(() => project.currentProjectId ?? ''); const tabs = computed(() => [ { name: 'devices', label: 'Devices', to: `/p/${proj.value}/device`, count: project.devices.length }, + { name: 'device-flash', label: 'Flash', to: `/p/${proj.value}/device/flash` }, { name: 'serial-console', label: 'Console', to: `/p/${proj.value}/device/console` }, ]); diff --git a/web/src/pages/project/device/FlashPanel.vue b/web/src/pages/project/device/FlashPanel.vue new file mode 100644 index 0000000..5d6689c --- /dev/null +++ b/web/src/pages/project/device/FlashPanel.vue @@ -0,0 +1,103 @@ + + + diff --git a/web/src/router.ts b/web/src/router.ts index 95db78c..ee74acd 100644 --- a/web/src/router.ts +++ b/web/src/router.ts @@ -48,6 +48,7 @@ const routes: RouteRecordRaw[] = [ component: () => import('./pages/project/device/DeviceHub.vue'), children: [ { path: '', name: 'devices', component: () => import('./pages/project/device/DevicesList.vue'), meta: { title: 'Devices' } }, + { path: 'flash', name: 'device-flash', component: () => import('./pages/project/device/FlashPanel.vue'), meta: { title: 'Flash' } }, { path: 'console', name: 'serial-console', component: () => import('./pages/project/device/SerialConsole.vue'), meta: { title: 'Console' } }, ], }, From 9c76c95ce6b95cade8f0840ac4370a99198116f1 Mon Sep 17 00:00:00 2001 From: Arjun Krishna Date: Sat, 22 Aug 2026 20:46:54 +0530 Subject: [PATCH 16/34] feat(worker): flash published SDK examples through the browser MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit GitHub's API sends CORS headers but release assets do not — they redirect to a host that sends none — so the browser can read the catalogue and cannot read the binary. The worker proxies both, which also keeps every browser off the unauthenticated GitHub rate limit. The download endpoint takes a tag and a filename, never a URL, and builds the target from a fixed repo. Both parts are charset-checked and '..' is rejected on its own, since the charset allows dots and v1..2 would otherwise climb out of the release path. The catalogue caches in KV behind an ETag. No published release reads as an empty catalogue rather than an error, and the panel falls back to picking a local .bin, which is what it did before. --- web/src/pages/project/device/FlashPanel.vue | 69 +++++++++++++++-- web/src/types.ts | 3 + worker/src/domains/firmware/routes.ts | 23 ++++++ worker/src/domains/firmware/service.ts | 85 +++++++++++++++++++++ worker/src/routes.ts | 2 + worker/test/firmware-url.test.ts | 39 ++++++++++ 6 files changed, 214 insertions(+), 7 deletions(-) create mode 100644 worker/src/domains/firmware/routes.ts create mode 100644 worker/src/domains/firmware/service.ts create mode 100644 worker/test/firmware-url.test.ts diff --git a/web/src/pages/project/device/FlashPanel.vue b/web/src/pages/project/device/FlashPanel.vue index 5d6689c..6f90667 100644 --- a/web/src/pages/project/device/FlashPanel.vue +++ b/web/src/pages/project/device/FlashPanel.vue @@ -1,8 +1,10 @@ diff --git a/web/src/pages/project/device/FirmwarePanel.vue b/web/src/pages/project/device/FirmwarePanel.vue new file mode 100644 index 0000000..5670737 --- /dev/null +++ b/web/src/pages/project/device/FirmwarePanel.vue @@ -0,0 +1,189 @@ + + + diff --git a/web/src/router.ts b/web/src/router.ts index ee74acd..40e7fd7 100644 --- a/web/src/router.ts +++ b/web/src/router.ts @@ -48,6 +48,7 @@ const routes: RouteRecordRaw[] = [ component: () => import('./pages/project/device/DeviceHub.vue'), children: [ { path: '', name: 'devices', component: () => import('./pages/project/device/DevicesList.vue'), meta: { title: 'Devices' } }, + { path: 'firmware', name: 'device-firmware', component: () => import('./pages/project/device/FirmwarePanel.vue'), meta: { title: 'Firmware' } }, { path: 'flash', name: 'device-flash', component: () => import('./pages/project/device/FlashPanel.vue'), meta: { title: 'Flash' } }, { path: 'console', name: 'serial-console', component: () => import('./pages/project/device/SerialConsole.vue'), meta: { title: 'Console' } }, ], diff --git a/web/src/stores/project.ts b/web/src/stores/project.ts index 65fc1b7..0e320ce 100644 --- a/web/src/stores/project.ts +++ b/web/src/stores/project.ts @@ -9,6 +9,7 @@ import type { Dashboard, DashboardMeta, Device, + Firmware, Variable, ProjectToken, ProjectTokenWithSecret, @@ -25,6 +26,7 @@ export const useProjectStore = defineStore('project', () => { const variables = ref([]); const projectTokens = ref([]); const devices = ref([]); + const firmware = ref([]); const dashboards = ref([]); const tokens = ref([]); const automations = ref([]); @@ -45,6 +47,7 @@ export const useProjectStore = defineStore('project', () => { integrations.value = []; projectTokens.value = []; devices.value = []; + firmware.value = []; } currentProjectId.value = projectId; await Promise.all([loadVariables(), loadDashboards()]); @@ -75,6 +78,46 @@ export const useProjectStore = defineStore('project', () => { devices.value = data.devices; } + async function loadFirmware(): Promise { + if (!currentProjectId.value) return; + const data = await api.get<{ firmware: Firmware[] }>( + `/v1/admin/projects/${currentProjectId.value}/firmware` + ); + firmware.value = data.firmware; + } + + async function uploadFirmware(input: { version: string; target?: string; notes?: string; body: ArrayBuffer }): Promise { + const pid = requireProjectId(); + const q = new URLSearchParams({ version: input.version }); + if (input.target) q.set('target', input.target); + if (input.notes) q.set('notes', input.notes); + // Raw body, so this bypasses the JSON api helper. + const res = await fetch(`/v1/admin/projects/${pid}/firmware?${q}`, { + method: 'POST', + credentials: 'include', + headers: { 'Content-Type': 'application/octet-stream' }, + body: input.body, + }); + if (!res.ok) { + const err = await res.json().catch(() => ({})) as { error?: string }; + throw new Error(err.error ?? `Upload failed (${res.status})`); + } + await loadFirmware(); + } + + async function deleteFirmware(id: string): Promise { + const pid = requireProjectId(); + await api.del(`/v1/admin/projects/${pid}/firmware/${id}`); + firmware.value = firmware.value.filter((f) => f.id !== id); + await loadDevices(); + } + + async function assignFirmware(deviceId: string, firmwareId: string | null): Promise { + const pid = requireProjectId(); + await api.put(`/v1/admin/projects/${pid}/firmware/assign/${deviceId}`, { firmware_id: firmwareId }); + await loadDevices(); + } + async function renameDevice(id: string, name: string): Promise { const pid = requireProjectId(); const { device } = await api.patch<{ device: Device }>( @@ -399,6 +442,7 @@ export const useProjectStore = defineStore('project', () => { currentProjectId, variables, devices, + firmware, projectTokens, dashboards, tokens, @@ -407,6 +451,10 @@ export const useProjectStore = defineStore('project', () => { pendingAutomation, switchTo, loadDevices, + loadFirmware, + uploadFirmware, + deleteFirmware, + assignFirmware, renameDevice, forgetDevice, loadVariables, diff --git a/web/src/types.ts b/web/src/types.ts index 08b4077..1b0ab67 100644 --- a/web/src/types.ts +++ b/web/src/types.ts @@ -78,6 +78,18 @@ export type Device = { is_default: number; first_seen: number | null; last_seen: number | null; + desired_firmware_id: string | null; + ota_status: string | null; +}; + +export type Firmware = { + id: string; + version: string; + target: string | null; + size: number; + sha256: string; + notes: string | null; + created_at: number; }; export type FirmwareEntry = { example: string; target: string; file: string; size: number }; diff --git a/worker/src/domains/devices/service.ts b/worker/src/domains/devices/service.ts index ba4d1b8..c64f5fd 100644 --- a/worker/src/domains/devices/service.ts +++ b/worker/src/domains/devices/service.ts @@ -15,6 +15,8 @@ export type DeviceSummary = { is_default: number; first_seen: number | null; last_seen: number | null; + desired_firmware_id: string | null; + ota_status: string | null; }; const MAX_DEVICES_PER_PROJECT = 100; @@ -126,7 +128,8 @@ export async function storageIdOf(env: Env, projectId: string, deviceId: string) export async function listDevices(env: Env, projectId: string): Promise { const rows = await env.DB .prepare( - `SELECT id, name, chip, firmware_version, is_default, first_seen, last_seen + `SELECT id, name, chip, firmware_version, is_default, first_seen, last_seen, + desired_firmware_id, ota_status FROM devices WHERE project_id = ? ORDER BY is_default DESC, name ASC` ) .bind(projectId) @@ -151,7 +154,8 @@ export async function renameDevice( if (res.meta.changes === 0) throw new ServiceError('not_found', 'no such device', 'unknown_device'); const row = await env.DB .prepare( - `SELECT id, name, chip, firmware_version, is_default, first_seen, last_seen + `SELECT id, name, chip, firmware_version, is_default, first_seen, last_seen, + desired_firmware_id, ota_status FROM devices WHERE id = ?` ) .bind(id) From 1063926cfd3fb1cc3113665ca05a03a5c2202b21 Mon Sep 17 00:00:00 2001 From: Arjun Krishna Date: Sat, 22 Aug 2026 21:04:15 +0530 Subject: [PATCH 19/34] feat(web): edit sketches in the browser A CodeMirror editor with C++ highlighting, seeded from the SDK examples and kept per project in localStorage. Example sources are read at the same release tag the binaries were built from, not the default branch, so what's on screen is what a published image was compiled from. raw.githubusercontent sends CORS headers, so unlike release assets this needs no proxy. The compile box explains why a browser can't run a C++ toolchain and what to do instead, rather than offering a button that does nothing. CodeMirror and esptool-js are both excluded from the service worker precache. Workbox globs every chunk, so ~630 kB of toolchain was being downloaded on install by people who may never open either tab. --- web/package.json | 5 + web/src/components/CodeEditor.vue | 46 ++++++++ web/src/pages/project/device/CodePanel.vue | 127 +++++++++++++++++++++ web/src/pages/project/device/DeviceHub.vue | 1 + web/src/router.ts | 1 + web/vite.config.ts | 3 +- 6 files changed, 182 insertions(+), 1 deletion(-) create mode 100644 web/src/components/CodeEditor.vue create mode 100644 web/src/pages/project/device/CodePanel.vue diff --git a/web/package.json b/web/package.json index 1e0d201..efe66d4 100644 --- a/web/package.json +++ b/web/package.json @@ -10,11 +10,16 @@ "typecheck": "vue-tsc --noEmit" }, "dependencies": { + "@codemirror/lang-cpp": "^6.0.3", + "@codemirror/state": "^6.7.1", + "@codemirror/theme-one-dark": "^6.1.3", + "@codemirror/view": "^6.43.9", "@nodrix/blocks-shared": "*", "@nodrix/integrations-shared": "*", "@nodrix/widgets-shared": "*", "@vue-flow/core": "^1.48.2", "better-auth": "^1.6.11", + "codemirror": "^6.0.2", "esptool-js": "^0.6.1", "grid-layout-plus": "^1.1.1", "pinia": "^2.3.0", diff --git a/web/src/components/CodeEditor.vue b/web/src/components/CodeEditor.vue new file mode 100644 index 0000000..c840916 --- /dev/null +++ b/web/src/components/CodeEditor.vue @@ -0,0 +1,46 @@ + + + diff --git a/web/src/pages/project/device/CodePanel.vue b/web/src/pages/project/device/CodePanel.vue new file mode 100644 index 0000000..a56af3a --- /dev/null +++ b/web/src/pages/project/device/CodePanel.vue @@ -0,0 +1,127 @@ + + + diff --git a/web/src/pages/project/device/DeviceHub.vue b/web/src/pages/project/device/DeviceHub.vue index b83621b..fcae4b1 100644 --- a/web/src/pages/project/device/DeviceHub.vue +++ b/web/src/pages/project/device/DeviceHub.vue @@ -10,6 +10,7 @@ const proj = computed(() => project.currentProjectId ?? ''); const tabs = computed(() => [ { name: 'devices', label: 'Devices', to: `/p/${proj.value}/device`, count: project.devices.length }, + { name: 'device-code', label: 'Code', to: `/p/${proj.value}/device/code` }, { name: 'device-flash', label: 'Flash', to: `/p/${proj.value}/device/flash` }, { name: 'device-firmware', label: 'Firmware', to: `/p/${proj.value}/device/firmware` }, { name: 'serial-console', label: 'Console', to: `/p/${proj.value}/device/console` }, diff --git a/web/src/router.ts b/web/src/router.ts index 40e7fd7..ba830f2 100644 --- a/web/src/router.ts +++ b/web/src/router.ts @@ -49,6 +49,7 @@ const routes: RouteRecordRaw[] = [ children: [ { path: '', name: 'devices', component: () => import('./pages/project/device/DevicesList.vue'), meta: { title: 'Devices' } }, { path: 'firmware', name: 'device-firmware', component: () => import('./pages/project/device/FirmwarePanel.vue'), meta: { title: 'Firmware' } }, + { path: 'code', name: 'device-code', component: () => import('./pages/project/device/CodePanel.vue'), meta: { title: 'Code' } }, { path: 'flash', name: 'device-flash', component: () => import('./pages/project/device/FlashPanel.vue'), meta: { title: 'Flash' } }, { path: 'console', name: 'serial-console', component: () => import('./pages/project/device/SerialConsole.vue'), meta: { title: 'Console' } }, ], diff --git a/web/vite.config.ts b/web/vite.config.ts index ac92a60..71db64a 100644 --- a/web/vite.config.ts +++ b/web/vite.config.ts @@ -23,7 +23,8 @@ export default defineConfig({ workbox: { globPatterns: ['**/*.{js,css,html,svg,png,ico,woff2}'], // og.png is only for social scrapers — no need to precache it offline. - globIgnores: ['**/og.png'], + // ~630 kB between them, and both need hardware or the network anyway. + globIgnores: ['**/og.png', '**/FlashPanel-*.js', '**/CodePanel-*.js'], navigateFallback: '/index.html', // Never hijack worker-rendered routes with the SPA fallback. navigateFallbackDenylist: [/^\/v1/, /^\/ws/, /^\/authorize/, /^\/\.well-known\//], From cb84f38f5274733e8074e2edb41572cd7c572f69 Mon Sep 17 00:00:00 2001 From: Arjun Krishna Date: Sat, 22 Aug 2026 21:09:48 +0530 Subject: [PATCH 20/34] fix(worker): stop device work leaking data and objects MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Deleting a variable cleared its hot state for every device in the project, not the one that owned it. Five sensors reporting temp from five places meant removing one wiped all five — the exact case device scoping exists for. Deleting a project wiped telemetry/ but not firmware/, leaving images in R2 that nothing could reach and nothing would ever remove. --- worker/src/domains/variables/routes.ts | 9 ++++--- .../platform/durable-objects/project-do.ts | 25 +++++++++---------- 2 files changed, 18 insertions(+), 16 deletions(-) diff --git a/worker/src/domains/variables/routes.ts b/worker/src/domains/variables/routes.ts index 2f046ae..400f417 100644 --- a/worker/src/domains/variables/routes.ts +++ b/worker/src/domains/variables/routes.ts @@ -6,6 +6,7 @@ import { newId } from '../../platform/lib/ids'; import { generateToken } from '../../platform/lib/tokens'; import { recordAudit } from '../../platform/lib/audit'; import { projectStub } from '../../platform/durable-objects/stubs'; +import { storageIdOf } from '../devices/service'; import { createVariable, updateVariable, listVariables } from './service'; import { actorFromSession, serviceErrorResponse } from '../../platform/lib/service'; @@ -59,13 +60,15 @@ variables.delete('/:id', async (c) => { const id = c.req.param('id'); const v = await c.env.DB - .prepare(`SELECT key FROM project_variables WHERE id = ? AND project_id = ?`) + .prepare(`SELECT key, device_id FROM project_variables WHERE id = ? AND project_id = ?`) .bind(id, project.id) - .first<{ key: string }>(); + .first<{ key: string; device_id: string }>(); if (!v) return c.json({ error: 'not_found' }, 404); try { - await projectStub(c.env, project.id).deleteVariable(v.key); + // Scoped, or deleting one device's copy would clear the key on all of them. + const storageId = await storageIdOf(c.env, project.id, v.device_id); + await projectStub(c.env, project.id).deleteVariable(v.key, storageId); } catch (e) { console.error('variable hot-state delete failed', id, e); } diff --git a/worker/src/platform/durable-objects/project-do.ts b/worker/src/platform/durable-objects/project-do.ts index 535aeb0..fbb47b5 100644 --- a/worker/src/platform/durable-objects/project-do.ts +++ b/worker/src/platform/durable-objects/project-do.ts @@ -631,22 +631,21 @@ export class ProjectDO extends DurableObject { // Same as close: nothing to do. } - // Wipes all data owned by this project — DO SQLite + R2 telemetry history. + // Wipes all data owned by this project — DO SQLite + everything it owns in R2. async destroy(): Promise { const projectId = this.projectId(); - // Delete every R2 object under telemetry/{projectId}/ (paginated). - let cursor: string | undefined; - do { - const list = await this.env.R2.list({ - prefix: `telemetry/${projectId}/`, - ...(cursor ? { cursor } : {}), - }); - if (list.objects.length > 0) { - await this.env.R2.delete(list.objects.map((o) => o.key)); - } - cursor = list.truncated ? list.cursor : undefined; - } while (cursor); + // A prefix added elsewhere and not listed here leaks objects nothing reaches. + for (const prefix of [`telemetry/${projectId}/`, `firmware/${projectId}/`]) { + let cursor: string | undefined; + do { + const list = await this.env.R2.list({ prefix, ...(cursor ? { cursor } : {}) }); + if (list.objects.length > 0) { + await this.env.R2.delete(list.objects.map((o) => o.key)); + } + cursor = list.truncated ? list.cursor : undefined; + } while (cursor); + } // Cancel any scheduled flush + wipe SQLite storage entirely. await this.ctx.storage.deleteAlarm(); From fb52966baecce31745eccd1fd03bb1a256f7db0a Mon Sep 17 00:00:00 2001 From: Arjun Krishna Date: Sat, 22 Aug 2026 21:13:20 +0530 Subject: [PATCH 21/34] feat(worker): export a project's data MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The migration to 2.0 runs once, unattended, on instances nobody can observe. Every other safeguard — batched statements, transaction-wrapped Durable Object steps — lowers the chance of failure without putting a floor under it. This is the floor: a copy taken before upgrading, and something to compare against afterwards. NDJSON, one typed record per line, streamed from an async generator so a project carrying a year of telemetry never sits in memory. No secrets leave: token hashes, sealed integration config and dashboard share tokens are all omitted. That makes the file safe to hand to someone, and means it restores data rather than credentials. --- web/src/pages/Projects.vue | 12 +++++ worker/src/domains/projects/export.ts | 74 +++++++++++++++++++++++++++ worker/src/domains/projects/routes.ts | 24 +++++++++ 3 files changed, 110 insertions(+) create mode 100644 worker/src/domains/projects/export.ts diff --git a/web/src/pages/Projects.vue b/web/src/pages/Projects.vue index 33daeda..9ed813a 100644 --- a/web/src/pages/Projects.vue +++ b/web/src/pages/Projects.vue @@ -105,6 +105,13 @@ async function removeProject(p: Project) { if (editing.value?.id === p.id) editing.value = null; } +// Navigation, not fetch — a Blob would defeat the streaming. +function exportProject(p: Project, event: Event) { + event.stopPropagation(); + openMenuFor.value = null; + window.location.href = `/v1/admin/projects/${p.id}/export`; +} + function deleteFromMenu(p: Project, event: Event) { event.stopPropagation(); openMenuFor.value = null; @@ -228,6 +235,11 @@ watch( class="block w-full px-3 py-1.5 text-left text-xs hover:bg-neutral-100 dark:hover:bg-neutral-800" @click="startEdit(p, $event)" >Edit project + + +

+ Builds on your machine via the nodrix agent. A first build installs the toolchain and takes minutes. +

+ + +

{{ buildError }}

+ 
{{ buildLog.join('\n') }}
diff --git a/worker/src/domains/firmware/agent.ts b/worker/src/domains/firmware/agent.ts new file mode 100644 index 0000000..51e2a62 --- /dev/null +++ b/worker/src/domains/firmware/agent.ts @@ -0,0 +1,56 @@ +import { Hono, type Context } from 'hono'; +import type { Env } from '../../env'; +import { requireSession } from '../../platform/middleware/require-session'; +import { resolveProject, type ProjectContextVars } from '../../platform/middleware/resolve-project'; +import { lookupUserToken, touchTokenLastUsed } from '../../platform/lib/tokens'; +import { projectStub } from '../../platform/durable-objects/stubs'; + +const MAX_SKETCH_BYTES = 256 * 1024; +const SAFE_FQBN = /^[A-Za-z0-9_.:-]{1,120}$/; + +// A build runs toolchain work on somebody's physical machine, so not members. +export async function agentWsHandler(c: Context<{ Bindings: Env }>): Promise { + const token = c.req.header('authorization')?.replace(/^Bearer\s+/i, '').trim(); + if (!token) return c.text('unauthorized', 401); + if (c.req.header('upgrade') !== 'websocket') return c.text('expected websocket', 426); + + const row = await lookupUserToken(c.env, token); + if (!row || row.scope !== 'admin') return c.text('unauthorized', 401); + if (row.role !== 'owner' && row.role !== 'admin') return c.text('forbidden', 403); + + const projectId = c.req.query('project') ?? row.project_id; + if (!projectId) return c.text('project required', 400); + if (row.project_id && row.project_id !== projectId) return c.text('forbidden', 403); + + c.executionCtx.waitUntil(touchTokenLastUsed(c.env, 'user', row.id)); + const stub = projectStub(c.env, projectId); + await stub.setProjectId(projectId); + return stub.fetch( + new Request(c.req.raw, { headers: { ...Object.fromEntries(c.req.raw.headers), 'x-nodrix-role': 'agent' } }) + ); +} + +const build = new Hono<{ Bindings: Env; Variables: ProjectContextVars }>(); + +build.use('*', requireSession); +build.use('*', resolveProject); + +build.post('/', async (c) => { + const user = c.get('user'); + if (user.role !== 'owner' && user.role !== 'admin') return c.json({ error: 'forbidden' }, 403); + + const body = await c.req.json<{ fqbn?: string; sketch?: string }>(); + const fqbn = (body.fqbn ?? '').trim(); + const sketch = body.sketch ?? ''; + if (!SAFE_FQBN.test(fqbn)) return c.json({ error: 'invalid_fqbn' }, 400); + if (!sketch.trim()) return c.json({ error: 'empty_sketch' }, 400); + if (new TextEncoder().encode(sketch).length > MAX_SKETCH_BYTES) { + return c.json({ error: 'sketch_too_large' }, 413); + } + + // Held open until the agent answers — no wall-clock limit while a client waits. + const result = await projectStub(c.env, c.get('project').id).requestBuild(fqbn, sketch); + return c.json(result, result.ok ? 200 : 409); +}); + +export default build; diff --git a/worker/src/platform/durable-objects/project-do.ts b/worker/src/platform/durable-objects/project-do.ts index 4d5383f..c08d573 100644 --- a/worker/src/platform/durable-objects/project-do.ts +++ b/worker/src/platform/durable-objects/project-do.ts @@ -30,6 +30,8 @@ const FLUSH_INTERVAL_MS = 60_000; const HIGH_WATER_MARK_ROWS = 500; // Roughly a megabyte a call, and a boot-looping board would pull it forever. const OTA_DOWNLOADS_PER_HOUR = 6; +// A cold toolchain install can take minutes; a warm build is seconds. +const BUILD_TIMEOUT_MS = 5 * 60_000; // Variable-trigger automations are cached in DO SQLite so high-rate telemetry // doesn't read D1 per point. Refreshed when stale or on invalidateAutomations(). const AUTO_CACHE_TTL_MS = 30_000; @@ -57,6 +59,10 @@ export type SeriesRow = { value: unknown; }; +export type BuildOutcome = + | { ok: true; binary: string; log: string[] } + | { ok: false; error: string; log: string[] }; + export type FlushResult = { flushed: number; keys: string[]; @@ -66,6 +72,9 @@ export type FlushResult = { export class ProjectDO extends DurableObject { private sql: SqlStorage; + // Held only while a browser waits on the response, so hibernation can't strand one. + private pendingBuilds = new Map void; log: string[] }>(); + private projectId(): string { // Stored on first ingest; used as the R2 key prefix. const row = this.sql @@ -447,6 +456,58 @@ export class ProjectDO extends DurableObject { this.sql.exec(`DELETE FROM pending_control WHERE variable = ? AND device_id IS ?`, variable, deviceId); } + private isAgent(ws: WebSocket): boolean { + return (ws.deserializeAttachment() as { role?: string } | null)?.role === 'agent'; + } + + // Nothing is queued for an agent that isn't connected. + async requestBuild(fqbn: string, sketch: string): Promise { + const agent = this.ctx.getWebSockets().find((ws) => this.isAgent(ws)); + if (!agent) return { ok: false, error: 'no_agent', log: [] }; + + const id = newId('build'); + const log: string[] = []; + return new Promise((resolve) => { + this.pendingBuilds.set(id, { resolve, log }); + setTimeout(() => { + const pending = this.pendingBuilds.get(id); + if (!pending) return; + this.pendingBuilds.delete(id); + pending.resolve({ ok: false, error: 'timed out', log: pending.log }); + }, BUILD_TIMEOUT_MS); + try { + agent.send(JSON.stringify({ type: 'build', id, fqbn, sketch })); + } catch { + this.pendingBuilds.delete(id); + resolve({ ok: false, error: 'agent went away', log: [] }); + } + }); + } + + private handleAgentFrame(raw: string): void { + let msg: Record; + try { + msg = JSON.parse(raw) as Record; + } catch { + return; + } + const build = typeof msg['build'] === 'string' ? msg['build'] : ''; + const pending = this.pendingBuilds.get(build); + if (!pending) return; + + if (msg['type'] === 'log' && typeof msg['line'] === 'string') { + if (pending.log.length < 500) pending.log.push(msg['line']); + return; + } + if (msg['type'] !== 'result') return; + this.pendingBuilds.delete(build); + pending.resolve( + msg['ok'] === true && typeof msg['binary'] === 'string' + ? { ok: true, binary: msg['binary'], log: pending.log } + : { ok: false, error: String(msg['error'] ?? 'build failed'), log: pending.log } + ); + } + // Survives hibernation, unlike anything held in memory. private deviceOf(ws: WebSocket): string { const att = ws.deserializeAttachment() as { device?: string } | null; @@ -518,6 +579,11 @@ export class ProjectDO extends DurableObject { const server = pair[1] as WebSocket; this.ctx.acceptWebSocket(server); + if (request.headers.get('x-nodrix-role') === 'agent') { + server.serializeAttachment({ role: 'agent' }); + return new Response(null, { status: 101, webSocket: client }); + } + // Until a hello arrives the socket counts as the default device. server.serializeAttachment({ device: '' }); this.sendPending(server, `(device_id IS NULL OR device_id = '')`); @@ -529,6 +595,7 @@ export class ProjectDO extends DurableObject { // parser. Invalid input → error frame; unknown/garbage → dropped. override async webSocketMessage(ws: WebSocket, message: string | ArrayBuffer): Promise { const raw = typeof message === 'string' ? message : new TextDecoder().decode(message); + if (this.isAgent(ws)) return this.handleAgentFrame(raw); const msg = parseDeviceMessage(raw); switch (msg.kind) { case 'hello': { diff --git a/worker/src/platform/lib/ids.ts b/worker/src/platform/lib/ids.ts index 79f4f0e..e0f2d08 100644 --- a/worker/src/platform/lib/ids.ts +++ b/worker/src/platform/lib/ids.ts @@ -8,6 +8,7 @@ import { nanoid } from 'nanoid'; // aut_xxx automation itg_xxx integration // wid_xxx widget instance (inside a dashboard layout) // dev_xxx device fwr_xxx firmware image +// bld_xxx agent build // dly_xxx delay continuation (pending automation resume) const PREFIXES = { @@ -23,6 +24,7 @@ const PREFIXES = { delay: 'dly', device: 'dev', firmware: 'fwr', + build: 'bld', } as const; export type IdKind = keyof typeof PREFIXES; diff --git a/worker/src/routes.ts b/worker/src/routes.ts index 7d07a79..7e1a49e 100644 --- a/worker/src/routes.ts +++ b/worker/src/routes.ts @@ -21,6 +21,7 @@ import variables from './domains/variables/routes'; import devicesRouter from './domains/devices/routes'; import firmware from './domains/firmware/routes'; import firmwareAdmin from './domains/firmware/admin'; +import build, { agentWsHandler } from './domains/firmware/agent'; import otaDevice from './domains/firmware/device'; import { readList, readState, readSeries } from './domains/variables/read'; import dashboards from './domains/dashboards/routes'; @@ -75,6 +76,8 @@ export function registerRoutes(app: App): void { app.route('/v1/admin/firmware', firmware); app.route('/v1/admin/projects/:proj/firmware', firmwareAdmin); app.route('/v1/ota', otaDevice); + app.route('/v1/admin/projects/:proj/build', build); + app.get('/v1/agent/ws', agentWsHandler); app.route('/v1/admin/projects/:proj/dashboards', dashboards); app.route('/v1/admin/projects/:proj/automations', automations); app.route('/v1/admin/projects/:proj/integrations', integrations); From d7923f7bc07951568c6e2491521858e58ca48ddc Mon Sep 17 00:00:00 2001 From: Arjun Krishna Date: Sat, 22 Aug 2026 22:16:58 +0530 Subject: [PATCH 32/34] fix(devices): keep last_seen current on every device-facing path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit recordDeviceSeen had one caller — the WebSocket hello handler — so HTTP-mode boards never updated last_seen and the online indicator read them as never seen. On an upgraded instance every device is the default device, so that is every device. touchDevice writes at most once a minute per device and is called from HTTP ingest, the control poll, the OTA check and image download, and WS telemetry (hello alone left a long-lived socket looking stale). The WS path also attributed variables to the default device regardless of which device the socket belonged to, so a named board's telemetry landed under its own id while its variable rows were created elsewhere. --- worker/src/domains/devices/service.ts | 24 +++++++++ worker/src/domains/firmware/device.ts | 9 ++-- worker/src/domains/telemetry/control.ts | 3 +- worker/src/domains/telemetry/telemetry.ts | 7 ++- .../platform/durable-objects/project-do.ts | 16 ++++-- worker/test/device-seen.test.ts | 51 +++++++++++++++++++ 6 files changed, 101 insertions(+), 9 deletions(-) create mode 100644 worker/test/device-seen.test.ts diff --git a/worker/src/domains/devices/service.ts b/worker/src/domains/devices/service.ts index c64f5fd..bb8ff74 100644 --- a/worker/src/domains/devices/service.ts +++ b/worker/src/domains/devices/service.ts @@ -182,6 +182,30 @@ export async function forgetDevice(env: Env, projectId: string, id: string): Pro await projectStub(env, projectId).deleteDevice(id); } +const SEEN_THROTTLE_MS = 60_000; +const seenWrites = new Map(); + +// Ingest is hot and last_seen only needs to be roughly right. +export async function touchDevice(env: Env, id: string): Promise { + const nowMs = Date.now(); + const prev = seenWrites.get(id); + if (prev !== undefined && nowMs - prev < SEEN_THROTTLE_MS) return; + seenWrites.set(id, nowMs); + if (seenWrites.size > 10_000) { + const cutoff = nowMs - SEEN_THROTTLE_MS; + for (const [k, t] of seenWrites) if (t < cutoff) seenWrites.delete(k); + } + const now = Math.floor(nowMs / 1000); + try { + await env.DB + .prepare(`UPDATE devices SET last_seen = ?, first_seen = COALESCE(first_seen, ?) WHERE id = ?`) + .bind(now, now, id) + .run(); + } catch { + seenWrites.delete(id); + } +} + export async function recordDeviceSeen( env: Env, id: string, diff --git a/worker/src/domains/firmware/device.ts b/worker/src/domains/firmware/device.ts index 26e47e1..d811f37 100644 --- a/worker/src/domains/firmware/device.ts +++ b/worker/src/domains/firmware/device.ts @@ -1,22 +1,25 @@ -import { Hono } from 'hono'; +import { Hono, type Context } from 'hono'; import type { Env } from '../../env'; import { requireProjectToken, type ProjectTokenContextVars } from '../../platform/middleware/require-project-token'; -import { normaliseDeviceKey, resolveDevice } from '../devices/service'; +import { normaliseDeviceKey, resolveDevice, touchDevice } from '../devices/service'; import { offerFor, openImage } from './ota'; import { projectStub } from '../../platform/durable-objects/stubs'; import { storageIdOf } from '../devices/service'; +type OtaContext = Context<{ Bindings: Env; Variables: ProjectTokenContextVars }>; + const ota = new Hono<{ Bindings: Env; Variables: ProjectTokenContextVars }>(); ota.use('*', requireProjectToken); -async function deviceIdFor(c: { env: Env; req: { header: (n: string) => string | undefined } }, projectId: string) { +async function deviceIdFor(c: OtaContext, projectId: string) { const device = await resolveDevice( c.env, projectId, normaliseDeviceKey(c.req.header('x-nodrix-device')), Math.floor(Date.now() / 1000) ); + if (device) c.executionCtx.waitUntil(touchDevice(c.env, device.id)); return device?.id ?? null; } diff --git a/worker/src/domains/telemetry/control.ts b/worker/src/domains/telemetry/control.ts index b3c36ff..3bf6200 100644 --- a/worker/src/domains/telemetry/control.ts +++ b/worker/src/domains/telemetry/control.ts @@ -3,7 +3,7 @@ import type { Env } from '../../env'; import { requireProjectToken, type ProjectTokenContextVars } from '../../platform/middleware/require-project-token'; import { lookupProjectToken, touchTokenLastUsed } from '../../platform/lib/tokens'; import { projectStub } from '../../platform/durable-objects/stubs'; -import { normaliseDeviceKey, resolveDevice } from '../devices/service'; +import { normaliseDeviceKey, resolveDevice, touchDevice } from '../devices/service'; const control = new Hono<{ Bindings: Env; Variables: ProjectTokenContextVars }>(); @@ -19,6 +19,7 @@ control.get('/', async (c) => { normaliseDeviceKey(c.req.header('x-nodrix-device')), Math.floor(Date.now() / 1000) ); + if (device) c.executionCtx.waitUntil(touchDevice(c.env, device.id)); const stub = projectStub(c.env, project_id); const pending = await stub.listPendingControl(device?.storageId ?? ''); return c.json({ control: pending }); diff --git a/worker/src/domains/telemetry/telemetry.ts b/worker/src/domains/telemetry/telemetry.ts index 3ec6130..b5a429c 100644 --- a/worker/src/domains/telemetry/telemetry.ts +++ b/worker/src/domains/telemetry/telemetry.ts @@ -4,7 +4,7 @@ import { requireProjectToken, type ProjectTokenContextVars } from '../../platfor import { projectStub } from '../../platform/durable-objects/stubs'; import { parseTelemetryBody, MAX_POINTS, MAX_KEY_LEN, MAX_STRING_VALUE } from './validate'; import { upsertVariables } from './variables'; -import { normaliseDeviceKey, resolveDevice } from '../devices/service'; +import { normaliseDeviceKey, resolveDevice, touchDevice } from '../devices/service'; const telemetry = new Hono<{ Bindings: Env; Variables: ProjectTokenContextVars }>(); @@ -51,7 +51,10 @@ telemetry.post('/', async (c) => { // Auto-create new variables + bump last_seen off the response path (best-effort). if (device) { c.executionCtx.waitUntil( - upsertVariables(c.env, project_id, device.id, points.map((p) => p.variable), now) + Promise.all([ + upsertVariables(c.env, project_id, device.id, points.map((p) => p.variable), now), + touchDevice(c.env, device.id), + ]) ); } diff --git a/worker/src/platform/durable-objects/project-do.ts b/worker/src/platform/durable-objects/project-do.ts index c08d573..117dc1c 100644 --- a/worker/src/platform/durable-objects/project-do.ts +++ b/worker/src/platform/durable-objects/project-do.ts @@ -10,7 +10,7 @@ import { toCompactSeries, type CompactSeries } from '../lib/series'; import { chunk, MAX_BOUND_PARAMS } from '../lib/sql'; import { parseDeviceMessage } from '../../domains/telemetry/ws-protocol'; import { upsertVariables } from '../../domains/telemetry/variables'; -import { defaultDeviceId, normaliseDeviceKey, resolveDevice, recordDeviceSeen } from '../../domains/devices/service'; +import { defaultDeviceId, normaliseDeviceKey, resolveDevice, recordDeviceSeen, touchDevice } from '../../domains/devices/service'; import { reconcile } from '../../domains/firmware/ota'; import { migrateSchema } from './schema'; import { PROJECT_SCHEMA } from './project-schema'; @@ -514,6 +514,11 @@ export class ProjectDO extends DurableObject { return typeof att?.device === 'string' ? att.device : ''; } + // The attachment holds a storage id; '' has to become a real D1 id. + private async d1DeviceId(ws: WebSocket, projectId: string): Promise { + return this.deviceOf(ws) || (await defaultDeviceId(this.env, projectId)); + } + private sendPending(ws: WebSocket, where: string, ...binds: unknown[]): void { const rows = this.sql .exec<{ id: string; variable: string; value: string }>( @@ -619,8 +624,13 @@ export class ProjectDO extends DurableObject { await this.ingest(pid, msg.points, this.deviceOf(ws)); const now = Math.floor(Date.now() / 1000); this.ctx.waitUntil( - defaultDeviceId(this.env, pid).then((deviceId) => - deviceId ? upsertVariables(this.env, pid, deviceId, msg.points.map((p) => p.variable), now) : undefined + this.d1DeviceId(ws, pid).then((id) => + id + ? Promise.all([ + upsertVariables(this.env, pid, id, msg.points.map((p) => p.variable), now), + touchDevice(this.env, id), + ]) + : undefined ) ); return; diff --git a/worker/test/device-seen.test.ts b/worker/test/device-seen.test.ts new file mode 100644 index 0000000..825fd35 --- /dev/null +++ b/worker/test/device-seen.test.ts @@ -0,0 +1,51 @@ +// A throttle that never lets go is the same bug as no write at all. +// Run with `bun test worker/test/device-seen.test.ts`. + +import { test, expect } from 'bun:test'; +import { touchDevice } from '../src/domains/devices/service'; +import type { Env } from '../src/env'; + +function fakeEnv(fail = false) { + const writes: string[] = []; + const env = { + DB: { + prepare() { + return { + bind(..._args: unknown[]) { + return { + run() { + if (fail) throw new Error('d1 down'); + writes.push('write'); + return Promise.resolve(); + }, + }; + }, + }; + }, + }, + } as unknown as Env; + return { env, writes }; +} + +test('writes once and then throttles the same device', async () => { + const { env, writes } = fakeEnv(); + await touchDevice(env, 'dev_throttle_a'); + await touchDevice(env, 'dev_throttle_a'); + await touchDevice(env, 'dev_throttle_a'); + expect(writes.length).toBe(1); +}); + +test('throttles per device, not globally', async () => { + const { env, writes } = fakeEnv(); + await touchDevice(env, 'dev_throttle_b'); + await touchDevice(env, 'dev_throttle_c'); + expect(writes.length).toBe(2); +}); + +test('a failed write retries on the next call', async () => { + const failing = fakeEnv(true); + await touchDevice(failing.env, 'dev_throttle_d'); + const ok = fakeEnv(); + await touchDevice(ok.env, 'dev_throttle_d'); + expect(ok.writes.length).toBe(1); +}); From e381d5d525ae47076ea90f5cc4381790884a2b24 Mon Sep 17 00:00:00 2001 From: Arjun Krishna Date: Sat, 22 Aug 2026 22:17:12 +0530 Subject: [PATCH 33/34] fix(build): keep compiled artifacts out of the project DO requestBuild carried the whole binary back as base64 through an object that is also running ingest, the ring buffer, R2 flushes and automation evaluation for every board in the project. The DO now carries job control only. The agent PUTs to /v1/agent/artifact which streams straight into R2, then reports the result, so ok always means the object is collectable; the browser fetches it once and the route deletes it. builds/ joins the project-delete prefix loop, and a sweep on each new build drops anything a timed-out request left behind. --- web/src/api.ts | 13 ++++ web/src/pages/project/device/CodePanel.vue | 13 ++-- worker/src/domains/firmware/agent.ts | 69 ++++++++++++++++++- .../platform/durable-objects/project-do.ts | 10 +-- worker/src/routes.ts | 3 +- 5 files changed, 94 insertions(+), 14 deletions(-) diff --git a/web/src/api.ts b/web/src/api.ts index 5e9e189..5a9028f 100644 --- a/web/src/api.ts +++ b/web/src/api.ts @@ -47,6 +47,18 @@ async function request(method: string, path: string, body?: unknown): Promise } } +async function requestBytes(path: string): Promise { + progress.start(); + try { + const res = await fetch(path, { credentials: 'include' }); + if (res.status === 401) unauthorizedHandler?.(); + if (!res.ok) throw new ApiError(res.status, await res.json().catch(() => null)); + return await res.arrayBuffer(); + } finally { + progress.done(); + } +} + // Collapse concurrent identical GETs into one network request — e.g. two // components mounting at once both calling the same loader. The promise is shared // while in flight and dropped as soon as it settles, so this is a dedup of @@ -63,6 +75,7 @@ function getDeduped(path: string): Promise { export const api = { get: (path: string) => getDeduped(path), + bytes: (path: string) => requestBytes(path), post: (path: string, body?: unknown) => request('POST', path, body), put: (path: string, body?: unknown) => request('PUT', path, body), patch: (path: string, body?: unknown) => request('PATCH', path, body), diff --git a/web/src/pages/project/device/CodePanel.vue b/web/src/pages/project/device/CodePanel.vue index 59fc82c..dfbbf8c 100644 --- a/web/src/pages/project/device/CodePanel.vue +++ b/web/src/pages/project/device/CodePanel.vue @@ -44,24 +44,23 @@ const building = ref(false); const buildLog = ref([]); const buildError = ref(''); -type BuildResult = { ok: boolean; binary?: string; error?: string; log?: string[] }; +type BuildResult = { ok: boolean; build?: string; error?: string; log?: string[] }; async function compileAndFlash() { building.value = true; buildLog.value = []; buildError.value = ''; try { - const res = await api.post( - `/v1/admin/projects/${project.currentProjectId}/build`, - { fqbn: fqbn.value, sketch: code.value } - ); + const base = `/v1/admin/projects/${project.currentProjectId}/build`; + const res = await api.post(base, { fqbn: fqbn.value, sketch: code.value }); buildLog.value = res.log ?? []; - if (!res.ok || !res.binary) { + if (!res.ok || !res.build) { buildError.value = res.error ?? 'Build failed'; return; } if (!port.value && !(await request())) return; - const bytes = Uint8Array.from(atob(res.binary), (ch) => ch.charCodeAt(0)); + // The image is served once and deleted, so ask for it only after a port is open. + const bytes = new Uint8Array(await api.bytes(`${base}/${res.build}/artifact`)); const ok = await flash([{ data: bytes, address: 0x10000 }]); if (ok) toast.success('Flashed — the board is restarting'); } catch (e) { diff --git a/worker/src/domains/firmware/agent.ts b/worker/src/domains/firmware/agent.ts index 51e2a62..7d91e62 100644 --- a/worker/src/domains/firmware/agent.ts +++ b/worker/src/domains/firmware/agent.ts @@ -6,13 +6,28 @@ import { lookupUserToken, touchTokenLastUsed } from '../../platform/lib/tokens'; import { projectStub } from '../../platform/durable-objects/stubs'; const MAX_SKETCH_BYTES = 256 * 1024; +const MAX_ARTIFACT_BYTES = 8 * 1024 * 1024; const SAFE_FQBN = /^[A-Za-z0-9_.:-]{1,120}$/; +const SAFE_BUILD_ID = /^bld_[A-Za-z0-9_-]{12}$/; + +const ARTIFACT_MAX_AGE_MS = 60 * 60 * 1000; + +function artifactKey(projectId: string, buildId: string): string { + return `builds/${projectId}/${buildId}.bin`; +} + +// A build that outran its request still uploads, and nothing will collect it. +async function pruneArtifacts(env: Env, projectId: string): Promise { + const cutoff = Date.now() - ARTIFACT_MAX_AGE_MS; + const list = await env.R2.list({ prefix: `builds/${projectId}/` }); + const stale = list.objects.filter((o) => o.uploaded.getTime() < cutoff).map((o) => o.key); + if (stale.length > 0) await env.R2.delete(stale); +} // A build runs toolchain work on somebody's physical machine, so not members. -export async function agentWsHandler(c: Context<{ Bindings: Env }>): Promise { +async function authoriseAgent(c: Context<{ Bindings: Env }>): Promise { const token = c.req.header('authorization')?.replace(/^Bearer\s+/i, '').trim(); if (!token) return c.text('unauthorized', 401); - if (c.req.header('upgrade') !== 'websocket') return c.text('expected websocket', 426); const row = await lookupUserToken(c.env, token); if (!row || row.scope !== 'admin') return c.text('unauthorized', 401); @@ -23,6 +38,14 @@ export async function agentWsHandler(c: Context<{ Bindings: Env }>): Promise): Promise { + if (c.req.header('upgrade') !== 'websocket') return c.text('expected websocket', 426); + const projectId = await authoriseAgent(c); + if (projectId instanceof Response) return projectId; + const stub = projectStub(c.env, projectId); await stub.setProjectId(projectId); return stub.fetch( @@ -30,6 +53,25 @@ export async function agentWsHandler(c: Context<{ Bindings: Env }>): Promise): Promise { + const projectId = await authoriseAgent(c); + if (projectId instanceof Response) return projectId; + + const build = c.req.param('build') ?? ''; + if (!SAFE_BUILD_ID.test(build)) return c.text('invalid build id', 400); + + const len = Number(c.req.header('content-length') ?? '0'); + if (!Number.isFinite(len) || len <= 0) return c.text('length required', 411); + if (len > MAX_ARTIFACT_BYTES) return c.text('artifact too large', 413); + if (!c.req.raw.body) return c.text('empty body', 400); + + await c.env.R2.put(artifactKey(projectId, build), c.req.raw.body, { + httpMetadata: { contentType: 'application/octet-stream' }, + }); + return c.body(null, 204); +} + const build = new Hono<{ Bindings: Env; Variables: ProjectContextVars }>(); build.use('*', requireSession); @@ -48,9 +90,32 @@ build.post('/', async (c) => { return c.json({ error: 'sketch_too_large' }, 413); } + c.executionCtx.waitUntil(pruneArtifacts(c.env, c.get('project').id).catch(() => {})); + // Held open until the agent answers — no wall-clock limit while a client waits. const result = await projectStub(c.env, c.get('project').id).requestBuild(fqbn, sketch); return c.json(result, result.ok ? 200 : 409); }); +// One-shot: the browser flashes from memory and a rebuild is cheap. +build.get('/:id/artifact', async (c) => { + const user = c.get('user'); + if (user.role !== 'owner' && user.role !== 'admin') return c.json({ error: 'forbidden' }, 403); + + const id = c.req.param('id'); + if (!SAFE_BUILD_ID.test(id)) return c.json({ error: 'invalid_build' }, 400); + + const key = artifactKey(c.get('project').id, id); + const object = await c.env.R2.get(key); + if (!object) return c.json({ error: 'not_found' }, 404); + + c.executionCtx.waitUntil(c.env.R2.delete(key)); + return new Response(object.body, { + headers: { + 'Content-Type': 'application/octet-stream', + 'Content-Length': String(object.size), + }, + }); +}); + export default build; diff --git a/worker/src/platform/durable-objects/project-do.ts b/worker/src/platform/durable-objects/project-do.ts index 117dc1c..8956779 100644 --- a/worker/src/platform/durable-objects/project-do.ts +++ b/worker/src/platform/durable-objects/project-do.ts @@ -59,8 +59,9 @@ export type SeriesRow = { value: unknown; }; +// The artifact goes to R2 on its own route; only its id crosses the DO. export type BuildOutcome = - | { ok: true; binary: string; log: string[] } + | { ok: true; build: string; log: string[] } | { ok: false; error: string; log: string[] }; export type FlushResult = { @@ -501,9 +502,10 @@ export class ProjectDO extends DurableObject { } if (msg['type'] !== 'result') return; this.pendingBuilds.delete(build); + // The agent uploads before it reports, so ok means the object is already there. pending.resolve( - msg['ok'] === true && typeof msg['binary'] === 'string' - ? { ok: true, binary: msg['binary'], log: pending.log } + msg['ok'] === true + ? { ok: true, build, log: pending.log } : { ok: false, error: String(msg['error'] ?? 'build failed'), log: pending.log } ); } @@ -662,7 +664,7 @@ export class ProjectDO extends DurableObject { const projectId = this.projectId(); // A prefix added elsewhere and not listed here leaks objects nothing reaches. - for (const prefix of [`telemetry/${projectId}/`, `firmware/${projectId}/`]) { + for (const prefix of [`telemetry/${projectId}/`, `firmware/${projectId}/`, `builds/${projectId}/`]) { let cursor: string | undefined; do { const list = await this.env.R2.list({ prefix, ...(cursor ? { cursor } : {}) }); diff --git a/worker/src/routes.ts b/worker/src/routes.ts index 7e1a49e..91ec116 100644 --- a/worker/src/routes.ts +++ b/worker/src/routes.ts @@ -21,7 +21,7 @@ import variables from './domains/variables/routes'; import devicesRouter from './domains/devices/routes'; import firmware from './domains/firmware/routes'; import firmwareAdmin from './domains/firmware/admin'; -import build, { agentWsHandler } from './domains/firmware/agent'; +import build, { agentWsHandler, agentArtifactHandler } from './domains/firmware/agent'; import otaDevice from './domains/firmware/device'; import { readList, readState, readSeries } from './domains/variables/read'; import dashboards from './domains/dashboards/routes'; @@ -78,6 +78,7 @@ export function registerRoutes(app: App): void { app.route('/v1/ota', otaDevice); app.route('/v1/admin/projects/:proj/build', build); app.get('/v1/agent/ws', agentWsHandler); + app.put('/v1/agent/artifact/:build', agentArtifactHandler); app.route('/v1/admin/projects/:proj/dashboards', dashboards); app.route('/v1/admin/projects/:proj/automations', automations); app.route('/v1/admin/projects/:proj/integrations', integrations); From 3e01297d0b0540efd474e61addaa19049bd7a9ef Mon Sep 17 00:00:00 2001 From: Arjun Krishna Date: Sat, 22 Aug 2026 22:27:41 +0530 Subject: [PATCH 34/34] fix(ota): reconcile with boards that were not listening when firmware shipped MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An HTTP-mode board also never reported the version it runs, so offerFor compared against NULL, offered the update, and would have gone on offering it after it succeeded — boot, flash, restart, repeat, bounded only by the hourly download quota. The check route now reads X-Nodrix-Firmware and X-Nodrix-Chip and feeds them to the same recordDeviceSeen and reconcile pair the hello frame uses. offerFor takes the reported version and prefers it over the stored one, so the answer is right in the same request instead of one round trip later. --- worker/src/domains/firmware/device.ts | 22 +++++++++++---- worker/src/domains/firmware/ota.ts | 11 ++++++-- worker/test/ota-offer.test.ts | 40 +++++++++++++++++++++++++++ 3 files changed, 65 insertions(+), 8 deletions(-) create mode 100644 worker/test/ota-offer.test.ts diff --git a/worker/src/domains/firmware/device.ts b/worker/src/domains/firmware/device.ts index d811f37..273c977 100644 --- a/worker/src/domains/firmware/device.ts +++ b/worker/src/domains/firmware/device.ts @@ -1,8 +1,8 @@ import { Hono, type Context } from 'hono'; import type { Env } from '../../env'; import { requireProjectToken, type ProjectTokenContextVars } from '../../platform/middleware/require-project-token'; -import { normaliseDeviceKey, resolveDevice, touchDevice } from '../devices/service'; -import { offerFor, openImage } from './ota'; +import { normaliseDeviceKey, recordDeviceSeen, resolveDevice, touchDevice } from '../devices/service'; +import { offerFor, openImage, reconcile } from './ota'; import { projectStub } from '../../platform/durable-objects/stubs'; import { storageIdOf } from '../devices/service'; @@ -12,23 +12,33 @@ const ota = new Hono<{ Bindings: Env; Variables: ProjectTokenContextVars }>(); ota.use('*', requireProjectToken); -async function deviceIdFor(c: OtaContext, projectId: string) { +async function deviceIdFor(c: OtaContext, projectId: string, seen = true) { const device = await resolveDevice( c.env, projectId, normaliseDeviceKey(c.req.header('x-nodrix-device')), Math.floor(Date.now() / 1000) ); - if (device) c.executionCtx.waitUntil(touchDevice(c.env, device.id)); + if (device && seen) c.executionCtx.waitUntil(touchDevice(c.env, device.id)); return device?.id ?? null; } // null means the board is already where it should be. ota.get('/', async (c) => { const { project_id } = c.get('projectToken'); - const deviceId = await deviceIdFor(c, project_id); + // An HTTP-mode board has no hello frame, so this is where it reports what it + // runs — without it a finished update is offered again forever. + const firmware = c.req.header('x-nodrix-firmware') ?? null; + const deviceId = await deviceIdFor(c, project_id, !firmware); if (!deviceId) return c.json({ update: null }); - return c.json({ update: await offerFor(c.env, project_id, deviceId) }); + + if (firmware) { + c.executionCtx.waitUntil( + recordDeviceSeen(c.env, deviceId, c.req.header('x-nodrix-chip'), firmware) + .then(() => reconcile(c.env, deviceId, firmware)) + ); + } + return c.json({ update: await offerFor(c.env, project_id, deviceId, firmware) }); }); ota.get('/image', async (c) => { diff --git a/worker/src/domains/firmware/ota.ts b/worker/src/domains/firmware/ota.ts index 80d064e..ea67f6e 100644 --- a/worker/src/domains/firmware/ota.ts +++ b/worker/src/domains/firmware/ota.ts @@ -142,7 +142,14 @@ export async function assignFirmware( export type UpdateOffer = { version: string; size: number; sha256: string; url: string } | null; // Reported vs desired is the whole reconciliation; there is no job to track. -export async function offerFor(env: Env, projectId: string, deviceId: string): Promise { +// A version the board claims in this request beats the stored one, which may not +// have caught up with the update it just applied. +export async function offerFor( + env: Env, + projectId: string, + deviceId: string, + reported?: string | null +): Promise { const row = await env.DB .prepare( `SELECT f.version AS version, f.size AS size, f.sha256 AS sha256, d.firmware_version AS current @@ -151,7 +158,7 @@ export async function offerFor(env: Env, projectId: string, deviceId: string): P ) .bind(deviceId, projectId) .first<{ version: string; size: number; sha256: string; current: string | null }>(); - if (!row || row.current === row.version) return null; + if (!row || (reported ?? row.current) === row.version) return null; return { version: row.version, size: row.size, sha256: row.sha256, url: '/v1/ota/image' }; } diff --git a/worker/test/ota-offer.test.ts b/worker/test/ota-offer.test.ts new file mode 100644 index 0000000..128e90b --- /dev/null +++ b/worker/test/ota-offer.test.ts @@ -0,0 +1,40 @@ +// Offering an update the board already applied is a reboot loop, not a retry. +// Run with `bun test worker/test/ota-offer.test.ts`. + +import { test, expect } from 'bun:test'; +import { offerFor } from '../src/domains/firmware/ota'; +import type { Env } from '../src/env'; + +function envWith(row: Record | null) { + return { + DB: { + prepare: () => ({ bind: () => ({ first: () => Promise.resolve(row) }) }), + }, + } as unknown as Env; +} + +const desired = { version: '1.2.0', size: 100, sha256: 'abc', current: null }; + +test('offers when the board runs something else', async () => { + const offer = await offerFor(envWith({ ...desired, current: '1.1.0' }), 'prj_a', 'dev_a'); + expect(offer?.version).toBe('1.2.0'); +}); + +test('offers nothing once the stored version matches', async () => { + const offer = await offerFor(envWith({ ...desired, current: '1.2.0' }), 'prj_a', 'dev_a'); + expect(offer).toBeNull(); +}); + +test('a version reported in the request beats the stored one', async () => { + const env = envWith({ ...desired, current: '1.1.0' }); + expect(await offerFor(env, 'prj_a', 'dev_a', '1.2.0')).toBeNull(); +}); + +test('a stale stored version does not suppress a real update', async () => { + const env = envWith({ ...desired, current: '1.2.0' }); + expect((await offerFor(env, 'prj_a', 'dev_a', '1.1.0'))?.version).toBe('1.2.0'); +}); + +test('offers nothing with no desired firmware', async () => { + expect(await offerFor(envWith(null), 'prj_a', 'dev_a')).toBeNull(); +});