From 9269adb4e08aeb27cf6d59f2190633613dbec6e7 Mon Sep 17 00:00:00 2001 From: Connor Hicks Date: Thu, 1 Oct 2026 17:05:54 -0400 Subject: [PATCH 1/4] Keep TUI responsive during syntax, LSP, and filesystem work --- aspec/architecture/lsp-engine.md | 12 + aspec/uxui/interface.md | 4 +- .../0016-tui-responsiveness-guide.md | 56 ++ aspec/work-items/0016-tui-responsiveness.md | 130 +++++ docs/03-using-the-tui.md | 12 +- docs/06-lsp-integration.md | 10 +- docs/09-syntax-highlighting-and-languages.md | 12 +- examples/syntax_performance.rs | 56 ++ scripts/tui_responsiveness.py | 251 +++++++++ src/bin/mock_lsp_server.rs | 54 +- src/commands/chord_engine/mod.rs | 6 +- src/commands/chord_engine/resolver.rs | 32 +- src/commands/diagnostics.rs | 69 +++ src/commands/lsp_engine/engine.rs | 254 ++++++++- src/commands/lsp_engine/mod.rs | 2 +- src/commands/lsp_engine/transport.rs | 249 ++++++-- src/commands/mod.rs | 2 + src/commands/syntax_engine/merge.rs | 82 ++- src/commands/syntax_engine/mod.rs | 189 ++++++- .../syntax_engine/tree_sitter_parse.rs | 97 +++- src/data/state.rs | 25 +- src/frontend/tui/app.rs | 530 ++++++++++++------ src/frontend/tui/background_chord.rs | 80 +++ src/frontend/tui/editor_pane.rs | 53 +- src/frontend/tui/fs_watcher.rs | 378 ++++++++++++- src/frontend/tui/mod.rs | 2 + src/frontend/tui/title_bar.rs | 2 +- src/frontend/tui/tree_pane.rs | 18 +- src/frontend/tui/tui_frontend.rs | 187 +++++- tests/lsp_integration.rs | 95 ++++ 30 files changed, 2617 insertions(+), 332 deletions(-) create mode 100644 aspec/work-items/0016-tui-responsiveness-guide.md create mode 100644 aspec/work-items/0016-tui-responsiveness.md create mode 100644 examples/syntax_performance.rs create mode 100644 scripts/tui_responsiveness.py create mode 100644 src/commands/diagnostics.rs create mode 100644 src/frontend/tui/background_chord.rs diff --git a/aspec/architecture/lsp-engine.md b/aspec/architecture/lsp-engine.md index 0dd6153..b3f5b85 100644 --- a/aspec/architecture/lsp-engine.md +++ b/aspec/architecture/lsp-engine.md @@ -249,3 +249,15 @@ The ChordEngine's resolver stage calls `LspEngine::document_symbols()` and `LspE ### Data Layer (Layer 0) LSPEngine reads from `data::lsp::registry` for server definitions and `data::lsp::types` for shared type definitions. It does not write to Layer 0. + +## TUI request endpoints and response deadlines (WI-16) + +`LspEngine::request_client` snapshots ready, cloneable endpoints under a short engine lock. `LspClient` performs request I/O and response waits after that lock is released. A per-server document lock orders nonblocking `didOpen`/`didChange` notification and request enqueueing; it is released before waiting. Chord clients carry captured unsaved text and a cancellation token. Interactive clients defer new background semantic requests while an interactive operation is pending. + +`LspProvider` exposes the resolver's document-symbol and selection-range queries. The headless CLI still supplies `LspEngine`; the TUI supplies `LspClient`, or `NoLsp` for chords that do not require semantic queries. Non-LSP resolution therefore never acquires the global engine mutex. + +Each transport has one pipe writer and one pipe reader. Outbound messages use a bounded queue, and responses are routed by ID to deadline-aware waiters. Server requests are distinguished by their `method`, even when their numeric ID collides with a client request. Configuration requests receive default/null entries; supported registration/progress requests are acknowledged; unsupported methods receive a JSON-RPC error. Notifications are handled separately from request responses. + +`request_timeout` defaults to five seconds and is configurable through `with_request_timeout`; initialization uses `startup_timeout` instead. Timeouts, cancelled waiters, closed streams, and partial responses cannot cause an unbounded caller wait. Late responses are discarded when their request is no longer pending. Transport failure is reflected in engine status polling. The synchronous CLI error path kills a live failed process before reaping it, avoiding a second indefinite wait. + +The TUI restores terminal state through a scope guard before starting shutdown. Its overall shutdown wait is limited to two seconds; an independent control handle can terminate children without acquiring the engine mutex. Individual graceful handshakes retain their existing 1.5-second bound. Background scans, watcher teardown, and syntax jobs do not have to finish before terminal restoration. diff --git a/aspec/uxui/interface.md b/aspec/uxui/interface.md index a2c498c..f417984 100644 --- a/aspec/uxui/interface.md +++ b/aspec/uxui/interface.md @@ -35,14 +35,14 @@ Modes: Keybindings: - `Ctrl-E`: toggle between Edit mode and Chord mode -- `Ctrl-T`: toggle file tree pane focus. If opened with single file and no tree exists, initiates a directory scan and shows the tree. +- `Ctrl-T`: toggle file tree pane focus. If opened with single file and no tree exists, shows a loading tree and starts its scan and watcher setup in the background. - `Ctrl-C`: opens exit confirmation modal (press Ctrl-C again to exit, Esc to cancel) - `Ctrl-S`: save current file (Edit mode) - Arrow keys: navigate (both modes) - `Tab`: insert tab character (Edit mode only; not a keybinding) - `Enter`: execute chord (Chord mode) / open file from tree (tree focused) / newline (Edit mode) - `Backspace`: delete character (Edit mode) / delete last chord character (Chord mode) -- `Esc`: clear chord input and status message (Chord mode) +- `Esc`: cancel a pending chord, or clear chord input and status message (Chord mode); confirmation dialogs keep priority Removed keybindings: - No `h/j/k/l` navigation — arrow keys only diff --git a/aspec/work-items/0016-tui-responsiveness-guide.md b/aspec/work-items/0016-tui-responsiveness-guide.md new file mode 100644 index 0000000..a62713f --- /dev/null +++ b/aspec/work-items/0016-tui-responsiveness-guide.md @@ -0,0 +1,56 @@ +# WI-16 implementation and validation + +The approved [work item](0016-tui-responsiveness.md) is implemented. The original user-session stall has no trace, so these measurements demonstrate fixes for confirmed blocking paths rather than attribution of that particular stall. + +## Delivered behavior + +- Ctrl-T immediately shows a loading tree. Recursive scanning, watcher setup, and teardown run on a worker. Existing visibility is preserved, including generated directories; this is a background full scan rather than lazy directory loading. +- Watcher events use a bounded queue, early access-event filtering, per-path modification coalescing, and asynchronous reconciliation after overflow. UI batches stop after 64 events or approximately 5 ms; selection follows its path across updates. +- Local parsing, token merging, token counting, and nonblank line counting run on a separate latest-wins worker. Revision checks reject obsolete local and semantic results. Immutable token snapshots, visible-line token lookup, cached counts, and viewport-limited wrapping reduce rendering work. +- Tree-sitter token conversion indexes source lines once, with indexed Unicode character boundaries. Token merging uses a sorted interval sweep while preserving whole-token overlap suppression. +- LSP pipe readers and writers run independently. Requests have IDs, bounded outbound queues, cancellation, and deadlines (5 seconds normally; 30 seconds during initialization). Server requests cannot be confused with client responses, and late responses cannot satisfy another request. Global engine locks are released before waiting. +- Chords resolve on a worker using captured unsaved document contents. Results apply only to the matching buffer, generation, cursor, mode, focus, and operation. Esc cancels pending resolution after any active modal handles it. Non-LSP chords do not require the engine lock. Interactive requests defer new replaceable semantic requests. +- Terminal restoration precedes LSP shutdown. An independent child-process control path bounds the final shutdown wait to 2 seconds even if an engine lock is unavailable. + +## Reference measurements + +Linux x86_64, AMD EPYC 9V74, five visible logical CPUs with a four-CPU container quota, local filesystem, Rust 1.95. These are one-machine diagnostic results, not portable latency guarantees. + +The release PTY run used 40,001 files and 10,101 directories, a 10,000-line Rust buffer, a mock LSP that never answers semantic/symbol requests, and continuous filesystem creates/removes. It checks actual selection/cursor changes, cancellation status, terminal attributes, and alternate-screen restoration. + +| Measurement | Result | +| --- | ---: | +| First populated tree visible | 202.95 ms | +| Tree arrow-key updates, p95 | 18.91 ms | +| Typing updates, p95 | 16.65 ms | +| Maximum sampled navigation/typing update | 21.00 ms | +| Pending chord cancellation | 8.84 ms | +| Terminal restoration after final quit input | 0.26 ms | + +The input measurements pass the proposed reference targets of p95 below 100 ms and sampled maximum below 250 ms. First-tree completion includes the full asynchronous scan; input remains available while it loads. A smaller fixture also passed (navigation p95 6.14 ms, typing p95 5.30 ms). + +For the repeated Rust source fixture, the original debug parser took 0.20/5.10/19.96 seconds at 1k/5k/10k lines. The new debug parser took roughly 13–16/70–73/139–163 ms, providing a same-profile comparison. The new release parser took 2.4–3.1/12.6–13.7/25.9–29.2 ms. Release merging took 0.37 ms for 10k tokens per source and 4.61 ms for 50k tokens per source. Parsing remains a full snapshot parse; the improvement comes from indexing and asynchronous scheduling rather than incremental Tree-sitter parsing. + +## Reproduce and review + +```sh +cargo test --offline --locked +cargo clippy --offline --all-targets -- -D warnings +cargo fmt --check +cargo check --offline --no-default-features +cargo build --offline --release --features test-support --bins --examples +cargo run --offline --release --example syntax_performance +python3 scripts/tui_responsiveness.py +``` + +The full suite passed: 579 unit tests and 74 integration tests. Clippy, formatting, no-default-features compilation, and the release build passed. New coverage exercises Unicode/CRLF token conversion, interval-merge equivalence, stale worker results, non-LSP execution while the engine lock is held, silent/partial/late LSP responses, server-request ID collisions, unsaved documents, cancellation, queue overflow, desired-watch replacement, viewport wrapping, and distant scroll jumps. + +For real-project traces, start ANE with `ANE_TIMINGS=1`. The bounded background logger writes `/tmp/ane-timings-.jsonl` on this platform (the platform temporary directory elsewhere), recording operation durations, counts, profile, OS, and architecture without source text or project paths. The watcher ignores its own timing file. Samples can be dropped when the queue fills. Event-loop timing includes idle terminal polling; use operation-specific timings to identify expensive work. + +## Remaining checks and limits + +- Runtime validation here is Linux only. macOS/Windows watcher behavior, permission failures, and network filesystem responsiveness still need platform-specific review. +- Opening, saving, renaming, deleting, and individual external-buffer reloads still use synchronous filesystem operations. The filesystem batch budget is cooperative between events, so one slow file operation can exceed it. This change removes the identified recursive/tree, highlighting, LSP-wait, and queue-starvation paths; it does not make every filesystem operation asynchronous. +- Full tree snapshots still require memory proportional to project size and UI application of a completed snapshot. Visibility/exclusion policy is unchanged. Tree navigation may wait for initial population, though the editor loop remains responsive. +- LSP cancellation releases editor waiters and sends the protocol cancellation notification; the server controls whether its own computation stops. A blocked pipe worker may remain until its process is terminated. Shutdown restores the terminal independently. +- Existing source-size caps and shared synchronous syntax APIs remain. TUI metrics/highlighting may briefly show the previous completed revision while new results arrive. diff --git a/aspec/work-items/0016-tui-responsiveness.md b/aspec/work-items/0016-tui-responsiveness.md new file mode 100644 index 0000000..10ef53f --- /dev/null +++ b/aspec/work-items/0016-tui-responsiveness.md @@ -0,0 +1,130 @@ +# Work Item: Bug + Task + +Title: prevent blocking work from freezing the TUI +Issue: not yet filed + +## Summary: +Users report delays of 10–20 seconds around first opening the file tree with `Ctrl-T` and moving with arrow keys. Audit and remove blocking paths that can affect the entire TUI: synchronous highlighting, unbounded LSP waits, filesystem-event starvation, recursive tree initialization, and shutdown waiting on the LSP engine lock. + +This plan was approved and implemented. See [implementation and validation](0016-tui-responsiveness-guide.md) for the delivered behavior, measurements, and remaining platform checks. The exact cause of the reported stall has not been established by a trace of the user's session. Fix the confirmed defects while adding enough timing evidence to distinguish them in real workloads. + +### Findings and evidence + +- **Confirmed: synchronous highlighting with quadratic token conversion.** `SyntaxEngine::compute` parses on the calling thread. `tree_sitter_parse::byte_to_char_col` calls `content.lines().nth(line_num)` for each token boundary, repeatedly scanning the file prefix. Every text-changing key invokes `refresh_buffer_caches`, so this can freeze ordinary typing, opening, reloading, and chord execution. Synthetic Rust input in the existing debug build took 0.20 seconds at 1,000 lines, 5.10 seconds at 5,000 lines, and 19.96 seconds at 10,000 lines. Input was repeated `let value = 123;\n`; these are diagnostic measurements, not release-build latency guarantees. +- **Confirmed: unbounded LSP transport waits can reach the UI thread.** `LspTransport::send_request` synchronously reads until its response arrives, with no response deadline. The syntax worker holds `Mutex` across semantic-token requests; `execute_chord_input` acquires the same lock on the UI thread for every chord, including non-LSP chords. LSP-dependent resolution can also perform its own blocking requests. The existing 30-second startup timeout does not bound requests after startup. +- **Confirmed: exit depends on that engine lock.** `run` acquires the lock and shuts down servers before disabling raw mode and leaving the alternate screen. A background request that never returns can prevent terminal restoration, despite the existing bounded shutdown handshake. +- **Confirmed: filesystem-event processing has no fairness limit.** `event_loop` drains the watcher queue completely before drawing or reading keyboard input. Access events undergo path canonicalization before being ignored. Each inserted path scans and sorts the complete backing tree. Watching remains active when the tree is hidden. +- **Confirmed: first tree initialization blocks.** `FileTree::from_dir` scans all descendants, including `.git`, `target`, and dependency directories, although the initial view only displays depth-zero entries. Recursive watcher registration also blocks. A synthetic tree with 50,102 entries, 10,101 directories, and two visible rows took 232–310 ms for scanning, watcher registration, and startup-event path processing combined in this Linux environment. Watcher registration generated 10,101 access/open events. This alone did not reproduce the reported 10–20 second stall. +- **Confirmed: token merging is quadratic.** `merge::merge` checks each Tree-sitter token against LSP tokens. It runs in both the background worker and synchronous cache refresh. A non-overlapping synthetic case with 10,000 tokens from each source took about 400 ms in the debug build. +- **Additional recurring costs to measure:** every frame clones the complete syntax-token vector, scans it for each displayed editor line, recomputes nonblank line count, and wraps complete logical lines before clipping the viewport. Whole-buffer token counting also runs after each edit. Token-count initialization measured about 520 ms once; subsequent counts for 10,000 lines were about 12 ms, so it was not the dominant bottleneck in that probe. + +`Ctrl-T` and tree up/down do not directly invoke parsing or acquire the LSP engine lock. An unchanged file normally hits the Tree-sitter cache. A slow LSP semantic response alone therefore does not prove the tree stall; LSP indexing could instead amplify filesystem events or rendering work. Verify this distinction during profiling. + +## User Stories + +### User Story 1: Navigate and edit while background work runs +As a: user + +I want to: +Type, move the cursor, switch tree focus, and scroll while highlighting, language-server requests, and filesystem updates are running + +So I can: +Use the editor without multi-second pauses or queued keystrokes suddenly replaying after a stall + +### User Story 2: Recover from a slow or stuck language server +As a: user + +I want to: +Cancel an outstanding chord, continue using non-LSP commands, and quit even when a language server never responds + +So I can: +Keep working and return to a usable terminal without killing the editor externally + +### User Story 3: Open large project trees promptly +As a: user + +I want to: +Open the tree and navigate available rows while additional directories and watches load in the background + +So I can: +Browse a project without waiting for every generated or dependency file to be scanned + +## Implementation Details: + +### 1. Establish timing evidence and a responsiveness budget + +- Add opt-in diagnostic timings for event-loop iterations, filesystem batches, tree scan/watch setup, syntax parse/conversion/merge, token counting, rendering, chord resolution, and LSP queue/request duration. Include counts and operation names; avoid recording file contents or credentials. Write diagnostics outside watched project paths to avoid feedback loops. +- Record debug/release profile, OS, fixture size, active file size, and LSP state with benchmark results. Measure cold/warm runs and input-to-visible-update latency, not just worker execution time. +- Proposed review targets: on documented local release-build fixtures, input-to-visible-update p95 below 100 ms and no application-induced UI stall above 250 ms during background work. Give each filesystem batch a small elapsed-time budget (initial proposal: 5 ms) plus an event-count cap. These targets need a recorded baseline and reference hardware before becoming CI timing assertions. + +### 2. Make LSP requests bounded and keep waits off the UI thread + +- Give transport I/O a single owner with request IDs and response dispatch. Use a dedicated reader and deadline-aware request completion rather than checking a clock around a blocking `read_line`, which cannot interrupt a silent server. Handle server-initiated requests and notifications separately from responses. +- Introduce configurable request deadlines distinct from startup deadlines (initial proposal: 5 seconds per interactive request). Cancellation must release the request's waiters. Late responses must not satisfy another request; reconnect/restart must clear pending requests predictably. +- Do not hold a global engine mutex across network/process I/O. Route background semantic requests and interactive operations through an engine worker or per-server request queues. Give interactive work priority over replaceable highlighting requests. +- Resolve LSP-dependent chords asynchronously against a captured buffer revision. Apply the resulting action on the UI thread only if the buffer/revision and operation identity still match. Show pending/failure status and support `Esc` cancellation without blocking navigation. +- Remove the engine-lock dependency from non-LSP chord execution. Refactor resolver access to LSP behind a narrow provider interface or equivalent separation, preserving headless CLI behavior and existing error semantics. + +### 3. Fix highlighting complexity and schedule expensive refreshes + +- Build a line-offset/line-slice index once per source snapshot and pass it through token emission. Convert byte columns using the indexed line; retain UTF-8 character-column semantics. Reuse the index for multiline tokens instead of collecting the entire file's lines for each token. +- Replace the nested token-overlap scan with an indexed or sweep-based merge that exploits sorted line/column positions. Preserve the rule that any overlapping LSP token suppresses the entire Tree-sitter token, including nested/overlapping tokens within either source. +- Move parse, merge, and token counting off the event thread. Use bounded, latest-wins work keyed by buffer identity and revision; discard stale results after edits, reloads, path changes, and buffer switches. Keep the previous valid highlighting while a new revision computes. +- Share immutable token snapshots so drawing does not clone every token under a mutex. Index tokens by line for viewport rendering. Cache line count/nonblank line count by revision and avoid wrapping an entire very long logical line when only a small viewport is needed. +- Keep the existing debounced LSP behavior, but do not let a slow semantic request delay local highlighting or keyboard processing. Consider incremental Tree-sitter parsing after the indexed conversion and worker changes are correct; full reparsing must still satisfy responsiveness goals through asynchronous execution. + +### 4. Bound filesystem work and make tree opening asynchronous + +- Drop access events before canonicalization or tree/buffer work, preferably before enqueueing them. Normalize only paths needed for relevant create/remove/rename/write handling; retain atomic-save detection. +- Process a bounded batch per frame, prioritizing keyboard input and redraws. Coalesce redundant events by path while preserving rename pairs and remove/create ordering. Use a bounded queue or overflow flag with asynchronous reconciliation so an event storm cannot grow memory without limit or silently lose final state. +- Batch backing-tree updates instead of sorting the entire tree after each insertion. Preserve visible selection by path when updates insert/remove rows before it. +- Show a loading tree immediately and populate directory children lazily or scan into a background snapshot. Move recursive watch setup off the UI thread, too. Keep the UI navigable during startup and directory expansion; right-arrow expansion currently scans all backing entries and inserts children individually. +- Make exclusions an explicit policy rather than silently changing what users can browse. Review skipping generated/dependency/VCS internals for indexing/watching while retaining an explicit way to display them. Merely hiding rows does not eliminate scan/watch costs. +- Reconcile events and scan results across watcher setup so files created, removed, or renamed during initialization are not lost. Audit `FileTree::entries` and `files_only` consumers before switching to lazy loading; callers needing a complete file list must receive one through a separate background/indexing path. + +### 5. Guarantee terminal restoration and bounded shutdown + +- Restore raw-mode, mouse-capture, and alternate-screen state through a scope guard, including setup failures and error exits. Do not make restoration depend on obtaining the engine lock. +- Signal worker cancellation and stop pending request waits through an independently reachable control path. Bound total shutdown time (initial proposal: 2 seconds overall), including lock acquisition and all servers, then terminate remaining child processes. Preserve the existing graceful handshake where it fits within that overall deadline. +- Workers must not send late mutations into a closed session. Ensure cancellation and terminal restoration do not deadlock on request queues or callback locks. + +### Suggested delivery order + +1. Add reproducible fixtures/timing hooks and fix indexed token conversion and merging. +2. Bound LSP requests, decouple non-LSP chords, and make terminal restoration independent of engine locks. +3. Move syntax/chord work to cancellable workers with revision-safe results. +4. Bound/coalesce watcher events and move scan/watch setup off-thread. +5. Optimize viewport rendering and validate all responsiveness targets in release builds. + +Each step must include its regression coverage; performance optimization alone does not replace the scheduling and timeout fixes. + +## Edge Case Considerations: + +- Hung server, closed stdout, malformed/partial response, late response after timeout, server-initiated request, concurrent requests, and restart during an outstanding chord. A startup watchdog is insufficient once the server is running. +- User edits/switches/reloads/renames a buffer while a chord or syntax result is pending. Never apply an old action to newer text or another buffer; cancellation must not overwrite unrelated current status. +- Unicode, CRLF, empty/final lines, multiline tokens, nested highlights, and very long/minified lines. Keep byte/character/display-column semantics consistent. +- Filesystem storms from builds, dependency installs, Git operations, and LSP indexing. Linux access-event behavior is verified; macOS and Windows watcher semantics need independent checks. +- Atomic saves, rename pairs, queue overflow, permission errors, symlinks, directories disappearing mid-scan, and slow/network-mounted paths. Background operations still need bounded queues and cancellation even when underlying filesystem calls are slow. +- Hiding the tree must not disable active-file change detection. Retained tree watches must not starve the editor while hidden. +- Lazy loading must preserve empty-directory expansion state, selection, sorting, rename/delete/new-file behavior, and complete-file-list consumers. Review exclusion policy before changing default visibility. +- Quit during startup, parse, scan, semantic request, or chord resolution; failures entering alternate-screen/raw mode; multiple LSP servers. Terminal cleanup must be independent of worker success. + +## Test Considerations: + +- **Highlight correctness and scaling:** compare tokens before/after indexing for supported languages and Unicode/multiline cases. Benchmark repeated source at 1k/5k/10k lines and very long single-line input in release builds. Verify near-linear conversion scaling instead of the observed quadratic curve. Test overlap-merge equivalence for nested, same-line, and non-overlapping intervals. +- **Stale results:** controllable workers finish requests out of order; assert only the current buffer revision/operation can update tokens, counts, or chord actions. Verify bounded latest-wins queues during rapid typing. +- **LSP nonresponse:** extend `mock_lsp_server` to ignore a request, send a partial response, delay a response beyond its deadline, and issue a server request. Verify timeout/cancellation, correct response routing, and recovery without an uninterruptible read. +- **Whole-TUI integration:** while the mock semantic request is stalled, navigate both panes, type in Edit mode, execute a non-LSP chord, cancel an LSP chord, and quit. Assert the loop continues processing input and the terminal is restored within the shutdown budget. Use a PTY or an injectable terminal/input harness as appropriate. +- **Watcher fairness:** continuously produce events while injecting keyboard input; verify batch budgets, redraw progress, bounded backlog, early access-event filtering, and reconciliation after overflow. Check the final tree/disk-change state after create/remove/rename storms and atomic saves. +- **Large tree startup:** reuse a documented fixture with approximately 50k entries/10k directories and measure first `Ctrl-T`, warm reopen, expansion, and navigation during scan/watch setup. Verify creation/removal during initialization and explicit excluded-directory visibility behavior. +- **Rendering:** benchmark large token snapshots, many lines, and minified/very long lines with a fixed viewport. Verify work tracks visible content rather than repeatedly scanning all tokens or formatting offscreen wrapped rows. +- Prefer deterministic scheduling/deadline assertions in unit tests; keep machine-dependent latency thresholds in documented performance/integration runs. Record release-build measurements against the agreed reference workload before accepting the work item. +- Run the applicable existing tests, `cargo clippy --all-targets -- -D warnings`, and `cargo fmt --check`. Preserve headless chord behavior and file-tree/atomic-save regressions from WI-13/WI-14. + +## Codebase Integration: + +- Follow the strict Layer 0/1/2 dependency rules in `aspec/architecture/design.md`. Plain revision/result/tree state belongs in `src/data`; LSP transport, syntax computation, and chord resolution belong in `src/commands`; terminal scheduling, rendering, and application of completed actions belong in `src/frontend/tui`. +- Primary files: `src/frontend/tui/app.rs`, `fs_watcher.rs`, `editor_pane.rs`, `tree_pane.rs`, `title_bar.rs`; `src/commands/syntax_engine/{mod.rs,tree_sitter_parse.rs,merge.rs}`; `src/commands/lsp_engine/{engine.rs,transport.rs}`; `src/commands/chord_engine/resolver.rs`; `src/data/{state.rs,file_tree.rs,buffer.rs}`. +- Extend the existing syntax latest-wins mailbox and mock-LSP facilities where suitable; avoid introducing another unbounded queue or frontend dependency into shared command/data code. +- Preserve shared CLI/TUI resolver semantics through a narrow LSP access abstraction. The architectural promise that non-LSP chords run without waiting for LSP must hold in implementation. +- Update `aspec/architecture/lsp-engine.md` and relevant TUI documentation for deadlines, cancellation, loading behavior, diagnostics, and any approved exclusion policy. Coordinate tree changes with `0013-on-disk-updates.md` and `0014-filetree-enhancements.md`. diff --git a/docs/03-using-the-tui.md b/docs/03-using-the-tui.md index 37f4a51..6df3afd 100644 --- a/docs/03-using-the-tui.md +++ b/docs/03-using-the-tui.md @@ -25,7 +25,7 @@ ane starts in Chord mode. A command box at the bottom accepts chord input: - Short-form chords (4 lowercase characters) **auto-execute** when the combination is valid for the current cursor position -- no Enter required. - Arrow keys (when the chord input is empty) move the cursor in the editor. - Arrow keys (when chord input has text) move within the chord input string. -- Press **Esc** to clear the chord input. +- Press **Esc** to clear the chord input, or cancel a chord that is still running. ### Edit mode @@ -62,7 +62,7 @@ ane does not use `h`/`j`/`k`/`l` for navigation, `i` to enter insert mode, or `q ## File tree -The file tree appears on the left when you open a directory, or when you press `Ctrl-T`. +The file tree appears on the left when you open a directory, or when you press `Ctrl-T`. It shows a loading state while the directory scan and filesystem watches are prepared in the background; you can keep using the editor during this work. Generated, dependency, and hidden directories remain visible. - **Up/Down arrows** navigate the tree. - **Left/Right arrows** expand/collapse directories. @@ -93,7 +93,7 @@ Non-LSP chords (Line, Buffer, Delimiter) work immediately regardless of LSP stat ane merges two highlighting sources into a single pass: -1. **tree-sitter** — runs immediately on open, producing structural highlights (keywords, strings, comments, types) with no server required. +1. **tree-sitter** — runs in a local background worker, producing structural highlights (keywords, strings, comments, types) with no server required. New edits replace queued work and outdated results are discarded. 2. **LSP semantic tokens** — added once the language server is ready, layering type-aware colors on top (e.g. distinguishing a local variable from a function parameter, or a mutable binding from an immutable one). LSP tokens take precedence where they overlap, so colors become richer as the server initializes without any visible flash or re-render. @@ -101,3 +101,9 @@ LSP tokens take precedence where they overlap, so colors become richer as the se --- [<- Chord Examples](02-chord-examples.md) | [Next: Exec Mode ->](04-exec-mode.md) + +## Pending work and diagnostics + +Chords resolve in the background. A pending chord shows `chord running — Esc to cancel`; changing its buffer, cursor, mode, or tree focus cancels its result. Confirmation dialogs retain priority over chord cancellation. Non-LSP chords can run while a language server is busy. Quit restores the terminal before waiting for language-server cleanup. + +For diagnosing pauses, launch with `ANE_TIMINGS=1`. Numeric operation durations and counts are written asynchronously to the system temporary directory as `ane-timings-.jsonl`. The log records build profile, OS, and architecture, and excludes file contents, paths, and request parameters. Its queue is bounded, so measurements may be dropped under heavy load. The filesystem watcher ignores the session's diagnostic file to prevent feedback loops. See the [responsiveness validation guide](../aspec/work-items/0016-tui-responsiveness-guide.md) for reproducible benchmarks. diff --git a/docs/06-lsp-integration.md b/docs/06-lsp-integration.md index 175794e..93ae6ef 100644 --- a/docs/06-lsp-integration.md +++ b/docs/06-lsp-integration.md @@ -31,6 +31,14 @@ LSP-scoped chords wait for the server to reach the `Running` state before execut --- +## Request deadlines and cancellation + +After initialization, requests have a five-second response deadline by default; this is separate from the 30-second startup deadline. Library callers can configure it with `LspEngineConfig::with_request_timeout`. Dedicated pipe workers keep silent or partially responding servers from blocking the TUI. Late responses retain their original request IDs and cannot complete a newer request. + +In the TUI, running chords can be cancelled with Esc. A cancelled or timed-out chord reports its outcome without applying an outdated edit. Language-server requests use the captured unsaved buffer text. Local highlighting and non-LSP chords continue independently of semantic requests. + +--- + ## Supported languages | Language | Server | Detection | @@ -63,7 +71,7 @@ The status bar shows the current LSP state: When a chord targets an LSP scope (Function, Variable, Struct, Member): -- **TUI mode**: the chord waits for LSP readiness. If the server is still starting, the status bar shows progress. +- **TUI mode**: if the server is still starting, the status bar asks you to wait and retry the chord. Once ready, the chord resolves in a cancellable background worker. - **Exec mode**: the command blocks until the server is ready, then executes the chord. Non-LSP chords (Line, Buffer, Delimiter) bypass this gate entirely and execute immediately. diff --git a/docs/09-syntax-highlighting-and-languages.md b/docs/09-syntax-highlighting-and-languages.md index e7d23f5..a06f5bb 100644 --- a/docs/09-syntax-highlighting-and-languages.md +++ b/docs/09-syntax-highlighting-and-languages.md @@ -10,12 +10,12 @@ This guide explains how syntax highlighting works, which languages are supported ### Two-tier pipeline -When you open or edit a file, ane runs two highlighting passes in parallel: +When you open or edit a file in the TUI, ane schedules two independent highlighting workers: -1. **Tree-sitter (synchronous, ~2ms)** - - Parses the code structure immediately +1. **Tree-sitter (local background worker)** + - Parses the newest queued source snapshot - Highlights keywords, strings, types, comments, etc. - - Results appear instantly — no waiting for language servers + - Results appear without waiting for language servers; typing and navigation continue during parsing - Works the same way every time (structural, not semantic) 2. **LSP semantic tokens (asynchronous, 300ms+)** @@ -27,11 +27,11 @@ When you open or edit a file, ane runs two highlighting passes in parallel: ### Why two tiers? -Tree-sitter is fast and language-agnostic — it works offline and scales instantly to large files. LSP is slower but smarter — it uses your project's type information and compiler knowledge to color code more precisely. By running both, you get the best of both worlds: +Tree-sitter works offline. Token conversion uses indexed lines, and the TUI keeps previous highlights while a newer revision is computed. LSP is slower but smarter — it uses your project's type information and compiler knowledge to color code more precisely. By running both, you get the best of both worlds: - **Immediate visual feedback** via tree-sitter when you open a file - **Richer semantic coloring** via LSP once the server has time to analyze the code -- **No typing lag** — tree-sitter re-highlights at 2ms per keystroke, LSP is debounced +- **Responsive editing** — parsing, merging, and token counting run off the UI thread; outdated results are discarded and LSP requests are debounced --- diff --git a/examples/syntax_performance.rs b/examples/syntax_performance.rs new file mode 100644 index 0000000..2578029 --- /dev/null +++ b/examples/syntax_performance.rs @@ -0,0 +1,56 @@ +//! Reproduce WI-16's conversion/merge measurements without an LSP server. +use ane::commands::syntax_engine::{merge, tree_sitter_parse}; +use ane::data::lsp::types::{Language, SemanticToken}; +use std::hint::black_box; +use std::time::Instant; + +fn main() { + println!( + "os={} arch={} profile={}", + std::env::consts::OS, + std::env::consts::ARCH, + if cfg!(debug_assertions) { + "debug" + } else { + "release" + } + ); + for lines in [1_000, 5_000, 10_000] { + let source = "let value = 123;\n".repeat(lines); + for round in 1..=3 { + let start = Instant::now(); + let tokens = black_box(tree_sitter_parse::parse(Language::Rust, black_box(&source))); + println!( + "parse lines={lines} bytes={} tokens={} round={round} micros={}", + source.len(), + tokens.len(), + start.elapsed().as_micros() + ); + } + } + for count in [1_000, 5_000, 10_000, 50_000] { + let ts: Vec<_> = (0..count) + .map(|line| SemanticToken { + line, + start_col: 0, + length: 3, + token_type: "keyword".into(), + }) + .collect(); + let lsp: Vec<_> = (0..count) + .map(|line| SemanticToken { + line, + start_col: 10, + length: 3, + token_type: "variable".into(), + }) + .collect(); + let start = Instant::now(); + let result = black_box(merge::merge(black_box(&ts), black_box(&lsp))); + println!( + "merge tokens_per_source={count} result={} micros={}", + result.len(), + start.elapsed().as_micros() + ); + } +} diff --git a/scripts/tui_responsiveness.py b/scripts/tui_responsiveness.py new file mode 100644 index 0000000..1d84042 --- /dev/null +++ b/scripts/tui_responsiveness.py @@ -0,0 +1,251 @@ +#!/usr/bin/env python3 +"""WI-16 Linux/macOS PTY check; standard library only. + +Build: cargo build --release --features test-support --bins +Run: python3 scripts/tui_responsiveness.py --ane target/release/ane \ + --mock-server target/release/mock_lsp_server + +Creates a 50k-entry tree, stalls semantic/symbol LSP requests, produces a +filesystem storm, and measures input-to-visible-update latency. Uses a small +ANSI screen reader, rather than treating unrelated periodic output as an +acknowledgement of a key. This is a reference benchmark, not a CI timing test. +""" +import argparse +import codecs +import errno +import fcntl +import json +import os +from pathlib import Path +import platform +import pty +import re +import select +import shlex +import struct +import subprocess +import tempfile +import termios +import threading +import time +import unicodedata + + +class Screen: + def __init__(self, rows=24, cols=100): + self.rows, self.cols = rows, cols + self.cells = [[(" ", None) for _ in range(cols)] for _ in range(rows)] + self.row = self.col = 0 + self.bg = None + self.pending = "" + self.decoder = codecs.getincrementaldecoder("utf-8")("replace") + + def feed(self, data): + text = self.pending + self.decoder.decode(data) + self.pending = "" + i = 0 + while i < len(text): + if text[i] == "\x1b": + if i + 1 >= len(text): + break + if text[i + 1] == "[": + match = re.match(r"\x1b\[([0-?]*)([ -/]*)([@-~])", text[i:]) + if not match: + break + raw, _, code = match.groups() + args = [int(x) if x else 0 for x in raw.lstrip("?=>").split(";")] + a = args[0] or 1 + if code in "Hf": + self.row = a - 1 + self.col = (args[1] if len(args) > 1 and args[1] else 1) - 1 + elif code == "G": + self.col = a - 1 + elif code == "d": + self.row = a - 1 + elif code == "A": + self.row -= a + elif code == "B": + self.row += a + elif code == "C": + self.col += a + elif code == "D": + self.col -= a + elif code == "J" and args[0] in (2, 3): + self.cells = [[(" ", None) for _ in range(self.cols)] for _ in range(self.rows)] + elif code == "K" and 0 <= self.row < self.rows: + for col in range(max(0, self.col), self.cols): + self.cells[self.row][col] = (" ", self.bg) + elif code == "m": + j = 0 + while j < len(args): + value = args[j] + if value in (0, 49): + self.bg = None + elif 40 <= value <= 47: + self.bg = value - 40 + elif 100 <= value <= 107: + self.bg = value - 100 + 8 + elif value in (38, 48) and j + 2 < len(args): + if args[j + 1] == 5: + if value == 48: + self.bg = args[j + 2] + j += 2 + elif args[j + 1] == 2 and j + 4 < len(args): + if value == 48: + self.bg = tuple(args[j + 2:j + 5]) + j += 4 + j += 1 + i += match.end() + continue + if text[i + 1] == "]": + end = re.search(r"\x07|\x1b\\", text[i + 2:]) + if not end: + break + i += 2 + end.end() + continue + i += 2 + continue + character = text[i] + if character == "\r": + self.col = 0 + elif character == "\n": + self.row += 1 + elif ord(character) >= 32: + if 0 <= self.row < self.rows and 0 <= self.col < self.cols: + self.cells[self.row][self.col] = (character, self.bg) + self.col += 0 if unicodedata.combining(character) else (2 if unicodedata.east_asian_width(character) in "WF" else 1) + i += 1 + self.pending = text[i:] + + def text(self): + return "\n".join("".join(c for c, _ in row) for row in self.cells) + + def selected(self): + for row in self.cells[1:-1]: + label = "".join(c for c, bg in row[:45] if bg == 8).strip() + if label: + return label + return "" + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--ane", type=Path, default=Path("target/release/ane")) + parser.add_argument("--mock-server", type=Path, default=Path("target/release/mock_lsp_server")) + parser.add_argument("--groups", type=int, default=100) + args = parser.parse_args() + ane, mock = args.ane.resolve(), args.mock_server.resolve() + assert ane.is_file() and mock.is_file(), "Build both binaries first" + with tempfile.TemporaryDirectory(prefix="ane-pty-") as root_name, tempfile.TemporaryDirectory(prefix="ane-lsp-") as bin_name: + root = Path(root_name) + source = "let value = 123;\n" * 10_000 + (root / "main.rs").write_text(source) + for group in range(args.groups): + for directory in range(100): + leaf = root / "target" / f"group{group:03}" / f"dir{directory:03}" + leaf.mkdir(parents=True) + for file in range(4): + (leaf / f"file{file}").touch() + hook = Path(bin_name) / "rust-analyzer" + hook.write_text(f"#!/bin/sh\nexec {shlex.quote(str(mock))} --ignore-semantic --ignore-symbols --client-request \"$@\"\n") + hook.chmod(0o755) + master, slave = pty.openpty() + fcntl.ioctl(slave, termios.TIOCSWINSZ, struct.pack("HHHH", 24, 100, 0, 0)) + original = termios.tcgetattr(slave) + env = dict(os.environ, PATH=bin_name + os.pathsep + os.environ.get("PATH", ""), TERM="xterm-256color", ANE_TIMINGS="1") + env.pop("NO_COLOR", None) + process = subprocess.Popen([str(ane), "main.rs"], cwd=root, env=env, stdin=slave, stdout=slave, stderr=slave) + screen = Screen() + raw = bytearray() + stop = threading.Event() + storm = None + + def wait(predicate, timeout=5): + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + if predicate(): + return + ready, _, _ = select.select([master], [], [], 0.02) + if ready: + try: + data = os.read(master, 65536) + except OSError as error: + if error.errno == errno.EIO: + break + raise + raw.extend(data) + screen.feed(data) + raise AssertionError("Timed out waiting for a visible update; selected=" + repr(screen.selected()) + "\n" + screen.text()) + + def key(data, predicate): + start = time.perf_counter() + os.write(master, data) + wait(predicate) + return (time.perf_counter() - start) * 1000 + + try: + wait(lambda: "main.rs" in screen.text() and "rs:●" in screen.text(), 10) + start = time.perf_counter() + os.write(master, b"\x14") + wait(lambda: "main.rs" in screen.selected()) + open_ms = (time.perf_counter() - start) * 1000 + + def generate_storm(): + leaf = root / "target" / "group000" / "dir000" + i = 0 + while not stop.is_set(): + path = leaf / f"event-{i % 3000}" + path.write_text("changed") + path.unlink(missing_ok=True) + i += 1 + + storm = threading.Thread(target=generate_storm) + storm.start() + navigation = [] + for _ in range(20): + navigation.append(key(b"\x1b[B", lambda: "target" in screen.selected())) + navigation.append(key(b"\x1b[A", lambda: "main.rs" in screen.selected())) + key(b"\x05", lambda: "EDIT" in screen.text()) + typing = [] + for column in range(1, 21): + typing.append(key(b"x", lambda column=column: f"0:{column} |" in screen.text())) + stop.set() + storm.join() + key(b"\x05", lambda: "CHORD" in screen.text()) + key(b"cifn", lambda: "chord running" in screen.text()) + cancel_ms = key(b"\x1b", lambda: "chord cancelled" in screen.text()) + key(b"cifn", lambda: "chord running" in screen.text()) + key(b"\x03", lambda: "Exit ane?" in screen.text()) + start = time.perf_counter() + os.write(master, b"\x03") + wait(lambda: b"\x1b[?1049l" in raw, 3) + restore_ms = (time.perf_counter() - start) * 1000 + process.wait(timeout=3) + after = termios.tcgetattr(slave) + assert after[3] & (termios.ICANON | termios.ECHO) == original[3] & (termios.ICANON | termios.ECHO) + assert process.returncode == 0 + p95 = lambda values: sorted(values)[int(0.95 * (len(values) - 1))] + result = dict(os=platform.system(), arch=platform.machine(), source_lines=10_000, + files=args.groups * 400 + 1, directories=args.groups * 101 + 1, + first_tree_visible_ms=round(open_ms, 2), + tree_navigation_p95_ms=round(p95(navigation), 2), + typing_p95_ms=round(p95(typing), 2), + max_input_ms=round(max(navigation + typing), 2), + cancel_ms=round(cancel_ms, 2), terminal_restore_ms=round(restore_ms, 2), + timings_file=f"{tempfile.gettempdir()}/ane-timings-{process.pid}.jsonl") + print(json.dumps(result, indent=2)) + assert max(navigation + typing) < 250, result + assert p95(navigation + typing) < 100, result + finally: + stop.set() + if storm: + storm.join(timeout=3) + if process.poll() is None: + process.kill() + process.wait() + os.close(master) + os.close(slave) + + +if __name__ == "__main__": + main() diff --git a/src/bin/mock_lsp_server.rs b/src/bin/mock_lsp_server.rs index 07303fe..f5fead6 100644 --- a/src/bin/mock_lsp_server.rs +++ b/src/bin/mock_lsp_server.rs @@ -41,6 +41,9 @@ fn main() { } } + let has = |flag: &str| args.iter().any(|arg| arg == flag); + let mut first_symbol = true; + let mut document_text = String::new(); let stdin = std::io::stdin(); let stdout = std::io::stdout(); let mut reader = BufReader::new(stdin.lock()); @@ -59,9 +62,46 @@ fn main() { .to_string(); let id = value.get("id").cloned(); + if method == "textDocument/semanticTokens/full" && has("--ignore-semantic") { + continue; + } + if method == "textDocument/documentSymbol" && has("--ignore-symbols") { + continue; + } + if method == "textDocument/semanticTokens/full" && has("--partial-semantic") { + write!(writer, "Content-Length: 100\r\n\r\n{{").unwrap(); + writer.flush().unwrap(); + continue; + } match method.as_str() { + "textDocument/didOpen" => { + document_text = value + .pointer("/params/textDocument/text") + .and_then(|v| v.as_str()) + .unwrap_or("") + .into(); + } + "textDocument/didChange" => { + document_text = value + .pointer("/params/contentChanges/0/text") + .and_then(|v| v.as_str()) + .unwrap_or("") + .into(); + } "initialize" => { if let Some(id) = id { + if has("--client-request") { + // Deliberately collide with the client's numeric ID. + write_msg( + &mut writer, + &serde_json::json!({"jsonrpc":"2.0", "id":id, + "method":"workspace/configuration", "params":{"items":[{}]}}) + .to_string(), + ); + let answer = read_msg(&mut reader).expect("client configuration response"); + let answer: serde_json::Value = serde_json::from_str(&answer).unwrap(); + assert_eq!(answer["result"], serde_json::json!([null])); + } let resp = serde_json::json!({ "jsonrpc": "2.0", "id": id, @@ -77,12 +117,24 @@ fn main() { } "textDocument/documentSymbol" => { if let Some(id) = id { + let delayed = first_symbol && has("--delay-first-symbol"); + first_symbol = false; + if delayed { + std::thread::sleep(std::time::Duration::from_millis(400)); + } + let name = if has("--echo-document") { + document_text.as_str() + } else if delayed { + "late" + } else { + "main" + }; let resp = serde_json::json!({ "jsonrpc": "2.0", "id": id, "result": [ { - "name": "main", + "name": name, "kind": 12, "range": { "start": {"line": 0, "character": 0}, diff --git a/src/commands/chord_engine/mod.rs b/src/commands/chord_engine/mod.rs index 22db421..3e35cef 100644 --- a/src/commands/chord_engine/mod.rs +++ b/src/commands/chord_engine/mod.rs @@ -9,7 +9,7 @@ use std::collections::HashMap; use anyhow::Result; -use crate::commands::lsp_engine::LspEngine; +use crate::commands::lsp_engine::LspProvider; use crate::data::buffer::Buffer; use types::{ChordAction, ChordQuery, ResolvedChord}; @@ -20,7 +20,7 @@ impl ChordEngine { pub fn execute( chord_input: &str, buffers: &HashMap, - lsp: &mut LspEngine, + lsp: &mut dyn LspProvider, ) -> Result> { let query = Self::parse(chord_input)?; let resolved = Self::resolve(&query, buffers, lsp)?; @@ -34,7 +34,7 @@ impl ChordEngine { pub fn resolve( query: &ChordQuery, buffers: &HashMap, - lsp: &mut LspEngine, + lsp: &mut dyn LspProvider, ) -> Result { resolver::resolve(query, buffers, lsp) } diff --git a/src/commands/chord_engine/resolver.rs b/src/commands/chord_engine/resolver.rs index 7d32462..a2b873d 100644 --- a/src/commands/chord_engine/resolver.rs +++ b/src/commands/chord_engine/resolver.rs @@ -3,7 +3,7 @@ use std::path::Path; use anyhow::Result; -use crate::commands::lsp_engine::LspEngine; +use crate::commands::lsp_engine::LspProvider; use crate::data::buffer::Buffer; use crate::data::chord_types::{Action, Component, Positional, Scope}; use crate::data::lsp::types::{DocumentSymbol, SymbolKind}; @@ -17,7 +17,7 @@ use super::types::{ pub fn resolve( query: &ChordQuery, buffers: &HashMap, - lsp: &mut LspEngine, + lsp: &mut dyn LspProvider, ) -> Result { let mut resolutions = HashMap::new(); @@ -36,7 +36,7 @@ fn resolve_buffer( query: &ChordQuery, buffer_name: &str, buffer: &Buffer, - lsp: &mut LspEngine, + lsp: &mut dyn LspProvider, ) -> Result { if query.action == Action::List { let (listed_items, warnings) = resolve_list(query, buffer, lsp, buffer_name)?; @@ -151,7 +151,7 @@ fn resolve_count_positional( query: &ChordQuery, buffer_name: &str, buffer: &Buffer, - lsp: &mut LspEngine, + lsp: &mut dyn LspProvider, n: u8, ) -> Result { let (cursor_line, cursor_col) = query.args.cursor_pos.ok_or_else(|| { @@ -215,7 +215,7 @@ fn resolve_count_line( query: &ChordQuery, buffer: &Buffer, buffer_name: &str, - lsp: &mut LspEngine, + lsp: &mut dyn LspProvider, n: u8, cursor_line: usize, cursor_col: usize, @@ -319,7 +319,7 @@ fn resolve_line_scope_at(buffer: &Buffer, line: usize, buffer_name: &str) -> Res fn resolve_count_lsp( query: &ChordQuery, buffer_name: &str, - lsp: &mut LspEngine, + lsp: &mut dyn LspProvider, n: u8, cursor_line: usize, cursor_col: usize, @@ -393,7 +393,7 @@ fn resolve_scope( query: &ChordQuery, buffer_name: &str, buffer: &Buffer, - lsp: &mut LspEngine, + lsp: &mut dyn LspProvider, ) -> Result { match query.scope { Scope::Line => resolve_line_scope(query, buffer, buffer_name), @@ -507,7 +507,7 @@ fn resolve_lsp_scope( query: &ChordQuery, buffer_name: &str, _buffer: &Buffer, - lsp: &mut LspEngine, + lsp: &mut dyn LspProvider, target_kinds: &[SymbolKind], ) -> Result { let path = Path::new(buffer_name); @@ -605,7 +605,7 @@ fn resolve_member_scope( query: &ChordQuery, buffer_name: &str, _buffer: &Buffer, - lsp: &mut LspEngine, + lsp: &mut dyn LspProvider, ) -> Result { let path = Path::new(buffer_name); let symbols = lsp.document_symbols(path).map_err(|e| { @@ -717,7 +717,7 @@ fn resolve_variable_scope( query: &ChordQuery, buffer_name: &str, buffer: &Buffer, - lsp: &mut LspEngine, + lsp: &mut dyn LspProvider, ) -> Result { let target_kinds = &[SymbolKind::Variable, SymbolKind::Const]; let path = Path::new(buffer_name); @@ -800,7 +800,7 @@ fn resolve_variable_scope_via_selection_range( query: &ChordQuery, buffer_name: &str, buffer: &Buffer, - lsp: &mut LspEngine, + lsp: &mut dyn LspProvider, ) -> Result { let (line, col) = query.args.cursor_pos.ok_or_else(|| { ChordError::resolve( @@ -860,7 +860,7 @@ fn resolve_component( buffer: &Buffer, scope_range: &TextRange, buffer_name: &str, - lsp: &mut LspEngine, + lsp: &mut dyn LspProvider, ) -> Result { match query.component { Component::Beginning => Ok(TextRange::point( @@ -887,7 +887,7 @@ fn resolve_name_component( query: &ChordQuery, buffer: &Buffer, buffer_name: &str, - lsp: &mut LspEngine, + lsp: &mut dyn LspProvider, scope_range: &TextRange, ) -> Result { if query.scope == Scope::Line || query.scope == Scope::Buffer { @@ -2522,7 +2522,7 @@ fn char_offset_to_range( fn resolve_list( query: &ChordQuery, buffer: &Buffer, - lsp: &mut LspEngine, + lsp: &mut dyn LspProvider, buffer_name: &str, ) -> Result<(Vec, Vec)> { let mut items = collect_list_candidates(query, buffer, lsp, buffer_name)?; @@ -2581,7 +2581,7 @@ fn resolve_list( fn collect_list_candidates( query: &ChordQuery, buffer: &Buffer, - lsp: &mut LspEngine, + lsp: &mut dyn LspProvider, buffer_name: &str, ) -> Result> { match query.component { @@ -2651,7 +2651,7 @@ fn collect_word_items( fn collect_lsp_items( query: &ChordQuery, buffer: &Buffer, - lsp: &mut LspEngine, + lsp: &mut dyn LspProvider, buffer_name: &str, ) -> Result> { let path = Path::new(buffer_name); diff --git a/src/commands/diagnostics.rs b/src/commands/diagnostics.rs new file mode 100644 index 0000000..f4b34b8 --- /dev/null +++ b/src/commands/diagnostics.rs @@ -0,0 +1,69 @@ +//! Opt-in numeric timings. No paths or file contents are recorded. +use std::io::Write; +use std::sync::{OnceLock, mpsc}; +use std::time::Instant; + +type Sample = (&'static str, u128, usize); +static SINK: OnceLock>> = OnceLock::new(); +static TIMING_NAME: OnceLock = OnceLock::new(); + +pub struct Timing { + operation: &'static str, + start: Option, + count: usize, +} + +impl Timing { + pub fn new(operation: &'static str) -> Self { + let sink = SINK.get_or_init(|| { + std::env::var_os("ANE_TIMINGS")?; + // Always outside the working tree; writes are handled off-thread. + let path = + std::env::temp_dir().join(format!("ane-timings-{}.jsonl", std::process::id())); + let (tx, rx) = mpsc::sync_channel::(1024); + std::thread::spawn(move || { + let Ok(file) = std::fs::OpenOptions::new().create(true).append(true).open(path) else { return; }; + let mut file = std::io::BufWriter::new(file); + let _ = writeln!(file, "{}", serde_json::json!({"profile": if cfg!(debug_assertions) { "debug" } else { "release" }, + "os": std::env::consts::OS, "arch": std::env::consts::ARCH})); + while let Ok((op, micros, count)) = rx.recv() { + let _ = writeln!(file, "{{\"operation\":\"{op}\",\"micros\":{micros},\"count\":{count}}}"); + let _ = file.flush(); + } + }); + Some(tx) + }); + Self { + operation, + start: sink.as_ref().map(|_| Instant::now()), + count: 0, + } + } + pub fn new_count(operation: &'static str, count: usize) -> Self { + let mut timing = Self::new(operation); + timing.count = count; + timing + } + pub fn set_count(&mut self, count: usize) { + self.count = count; + } +} + +/// Prevent self-generated diagnostics events if the watched root includes TMPDIR. +pub fn is_timing_path(path: &std::path::Path) -> bool { + path.file_name().is_some_and(|name| { + name == TIMING_NAME + .get_or_init(|| format!("ane-timings-{}.jsonl", std::process::id())) + .as_str() + }) +} + +impl Drop for Timing { + fn drop(&mut self) { + if let Some(start) = self.start + && let Some(Some(sink)) = SINK.get() + { + let _ = sink.try_send((self.operation, start.elapsed().as_micros(), self.count)); + } + } +} diff --git a/src/commands/lsp_engine/engine.rs b/src/commands/lsp_engine/engine.rs index 5fa4fb7..47fa7df 100644 --- a/src/commands/lsp_engine/engine.rs +++ b/src/commands/lsp_engine/engine.rs @@ -27,6 +27,7 @@ const SHUTDOWN_GRACE: Duration = Duration::from_millis(1_500); pub struct LspEngineConfig { pub auto_install: bool, pub startup_timeout: Duration, + pub request_timeout: Duration, binary_name_override: Option, binary_args_override: Vec, check_command_override: Option, @@ -37,6 +38,7 @@ impl Default for LspEngineConfig { Self { auto_install: true, startup_timeout: Duration::from_secs(30), + request_timeout: Duration::from_secs(5), binary_name_override: None, binary_args_override: Vec::new(), check_command_override: None, @@ -50,6 +52,11 @@ impl LspEngineConfig { self } + pub fn with_request_timeout(mut self, timeout: Duration) -> Self { + self.request_timeout = timeout; + self + } + pub fn with_auto_install(mut self, auto: bool) -> Self { self.auto_install = auto; self @@ -82,6 +89,7 @@ struct StartupContext { auto_install: bool, overrides: ServerOverrides, startup_timeout: Duration, + request_timeout: Duration, install_progress: Option>, install_lock: Arc>, } @@ -95,6 +103,7 @@ struct ServerInstance { startup_rx: Option>, opened_files: HashSet, root_path: PathBuf, + documents: Arc>>, } impl ServerInstance { @@ -132,6 +141,7 @@ fn try_transition( pub struct LspEngine { servers: HashMap, config: LspEngineConfig, + interactive: Arc, event_tx: mpsc::Sender, event_rx: mpsc::Receiver, install_progress: Option>, @@ -154,6 +164,7 @@ impl LspEngine { Self { servers: HashMap::new(), config, + interactive: Arc::new(std::sync::atomic::AtomicUsize::new(0)), event_tx, event_rx, install_progress: None, @@ -249,6 +260,7 @@ impl LspEngine { check_command: self.config.check_command_override.clone(), }, startup_timeout: self.config.startup_timeout, + request_timeout: self.config.request_timeout, install_progress: self.install_progress.clone(), install_lock: Arc::clone(&self.install_lock), }; @@ -266,6 +278,7 @@ impl LspEngine { transport: None, startup_rx: Some(transport_rx), opened_files: HashSet::new(), + documents: Arc::new(Mutex::new(HashMap::new())), root_path, }, ); @@ -399,10 +412,65 @@ impl LspEngine { pub fn status_summary(&self) -> Vec<(Language, ServerState)> { self.servers .iter() - .map(|(lang, server)| (*lang, server.get_state())) + .map(|(lang, server)| { + if server + .transport + .as_ref() + .is_some_and(LspTransport::is_closed) + { + try_transition(&server.state, *lang, ServerState::Failed, &self.event_tx); + } + (*lang, server.get_state()) + }) .collect() } + /// Snapshot ready request endpoints while holding only a short engine lock. + /// Pipe I/O and response waits happen after the caller releases that lock. + pub fn request_client(&mut self) -> LspClient { + let languages: Vec<_> = self.servers.keys().copied().collect(); + for lang in languages { + self.try_recv_startup(lang); + } + LspClient { + interactive: Arc::clone(&self.interactive), + lease: None, + contents: HashMap::new(), + servers: self + .servers + .iter() + .filter_map(|(&lang, server)| { + (server.get_state() == ServerState::Running) + .then(|| { + server.transport.as_ref().map(|transport| { + (lang, (transport.clone(), Arc::clone(&server.documents))) + }) + }) + .flatten() + }) + .collect(), + #[cfg(any(test, feature = "test-support"))] + test_symbols: self.test_symbols.clone(), + #[cfg(any(test, feature = "test-support"))] + test_selection_ranges: self.test_selection_ranges.clone(), + #[cfg(any(test, feature = "test-support"))] + test_semantic_tokens: self.test_semantic_tokens.clone(), + #[cfg(any(test, feature = "test-support"))] + test_semantic_tokens_delay: self.test_semantic_tokens_delay, + } + } + + /// Child handles are independent of the engine mutex, including during startup. + pub fn shutdown_control(&self) -> ShutdownControl { + ShutdownControl { + children: self + .servers + .values() + .map(|s| Arc::clone(&s.child)) + .collect(), + } + } + // --- LSP Query Methods --- pub fn document_symbols(&mut self, file_path: &Path) -> Result> { @@ -685,6 +753,7 @@ impl LspEngine { server.opened_files.clear(); // Reap the child so we don't leave a zombie. if let Some(mut c) = server.child.lock().unwrap().take() { + let _ = c.kill(); let _ = c.wait(); } if !current.is_terminal() || current == ServerState::Running { @@ -714,6 +783,7 @@ impl LspEngine { // SHUTDOWN_GRACE. Take ownership of the transport so the worker thread // can drive it without an aliased borrow. if let Some(mut transport) = server.transport.take() { + transport.cancel_pending(); let (done_tx, done_rx) = mpsc::channel::<()>(); let handle = thread::spawn(move || { let _ = transport.send_request("shutdown", Value::Null); @@ -777,6 +847,186 @@ impl LspEngine { } } +/// The resolver depends on these queries, not ownership of the global engine. +pub trait LspProvider { + fn document_symbols(&mut self, path: &Path) -> Result>; + fn selection_range(&mut self, path: &Path, line: usize, col: usize) -> Result; +} + +impl LspProvider for LspEngine { + fn document_symbols(&mut self, path: &Path) -> Result> { + self.document_symbols(path) + } + fn selection_range(&mut self, path: &Path, line: usize, col: usize) -> Result { + self.selection_range(path, line, col) + } +} + +pub struct NoLsp; +impl LspProvider for NoLsp { + fn document_symbols(&mut self, _: &Path) -> Result> { + bail!("LSP unavailable") + } + fn selection_range(&mut self, _: &Path, _: usize, _: usize) -> Result { + bail!("LSP unavailable") + } +} + +type ClientServer = (LspTransport, Arc>>); +#[derive(Clone)] +pub struct LspClient { + interactive: Arc, + lease: Option>, + contents: HashMap, + servers: HashMap, + #[cfg(any(test, feature = "test-support"))] + test_symbols: HashMap>, + #[cfg(any(test, feature = "test-support"))] + test_selection_ranges: HashMap<(PathBuf, usize, usize), SelectionRange>, + #[cfg(any(test, feature = "test-support"))] + test_semantic_tokens: HashMap>, + #[cfg(any(test, feature = "test-support"))] + test_semantic_tokens_delay: Option, +} + +struct InteractiveLease(Arc); +impl Drop for InteractiveLease { + fn drop(&mut self) { + self.0.fetch_sub(1, std::sync::atomic::Ordering::AcqRel); + } +} +impl LspClient { + pub fn interactive_pending(&self) -> bool { + self.interactive.load(std::sync::atomic::Ordering::Acquire) > 0 + } + pub fn with_contents(mut self, contents: HashMap) -> Self { + self.contents = contents; + self + } + pub fn with_cancellation(mut self, cancel: Arc) -> Self { + if self.lease.is_none() { + self.interactive + .fetch_add(1, std::sync::atomic::Ordering::AcqRel); + self.lease = Some(Arc::new(InteractiveLease(Arc::clone(&self.interactive)))); + } + for (transport, _) in self.servers.values_mut() { + *transport = transport.clone().with_cancellation(Arc::clone(&cancel)); + } + self + } + + fn request( + &self, + path: &Path, + content: Option<&str>, + method: &str, + params: Value, + ) -> Result { + use std::hash::{Hash, Hasher}; + let lang = Language::from_path(path).ok_or_else(|| anyhow::anyhow!("unknown language"))?; + let (endpoint, documents) = self + .servers + .get(&lang) + .ok_or_else(|| anyhow::anyhow!("LSP not ready"))?; + let mut transport = endpoint.clone(); + let text = content + .or_else(|| self.contents.get(path).map(String::as_str)) + .map(str::to_owned) + .unwrap_or_else(|| std::fs::read_to_string(path).unwrap_or_default()); + let mut hash = std::collections::hash_map::DefaultHasher::new(); + text.hash(&mut hash); + let hash = hash.finish(); + // The lock covers only nonblocking notification enqueueing. It orders + // didOpen/didChange across syntax and interactive request workers. + let mut documents = documents.lock().unwrap(); + match documents.get(path).copied() { + None => { + transport.send_notification("textDocument/didOpen", json!({"textDocument": { + "uri": path_to_uri(path), "languageId": Language::language_id_for_path(path).unwrap_or(lang.name()), + "version": 1, "text": text + }}))?; + documents.insert(path.to_path_buf(), (hash, 1)); + } + Some((old, version)) if old != hash => { + let version = version.saturating_add(1); + transport.send_notification( + "textDocument/didChange", + json!({ + "textDocument": {"uri":path_to_uri(path), "version":version}, + "contentChanges":[{"text":text}] + }), + )?; + documents.insert(path.to_path_buf(), (hash, version)); + } + _ => {} + } + let request = transport.start_request(method, params)?; + drop(documents); + request.wait() + } + + pub fn semantic_tokens(&mut self, path: &Path, content: &str) -> Result> { + #[cfg(any(test, feature = "test-support"))] + if let Some(tokens) = self.test_semantic_tokens.get(path) { + if let Some(delay) = self.test_semantic_tokens_delay { + thread::sleep(delay); + } + return Ok(tokens.clone()); + } + let result = self.request( + path, + Some(content), + "textDocument/semanticTokens/full", + json!({"textDocument":{"uri":path_to_uri(path)}}), + )?; + parse_semantic_tokens(&result) + } +} + +impl LspProvider for LspClient { + fn document_symbols(&mut self, path: &Path) -> Result> { + #[cfg(any(test, feature = "test-support"))] + if let Some(symbols) = self.test_symbols.get(path) { + return Ok(symbols.clone()); + } + let result = self.request( + path, + None, + "textDocument/documentSymbol", + json!({"textDocument":{"uri":path_to_uri(path)}}), + )?; + parse_document_symbols(&result) + } + fn selection_range(&mut self, path: &Path, line: usize, col: usize) -> Result { + #[cfg(any(test, feature = "test-support"))] + if let Some(range) = self + .test_selection_ranges + .get(&(path.to_path_buf(), line, col)) + { + return Ok(range.clone()); + } + let result = self.request(path, None,"textDocument/selectionRange", + json!({"textDocument":{"uri":path_to_uri(path)},"positions":[{"line":line,"character":col}]}))?; + parse_selection_range(&result) + } +} + +pub struct ShutdownControl { + children: Vec>>>, +} +impl ShutdownControl { + /// This bypasses engine ownership so a stuck request cannot block cleanup. + pub fn terminate(&self) { + for child in &self.children { + if let Ok(mut slot) = child.try_lock() + && let Some(child) = slot.as_mut() + { + let _ = child.kill(); + } + } + } +} + impl Drop for LspEngine { fn drop(&mut self) { self.shutdown_all(); @@ -944,6 +1194,7 @@ fn startup_thread( }); let mut transport = LspTransport::new(stdin, stdout); + transport.set_timeout(ctx.startup_timeout); let root_uri = path_to_uri(&ctx.root_path); let init_params = json!({ @@ -970,6 +1221,7 @@ fn startup_thread( return; } + transport.set_timeout(ctx.request_timeout); if let Err(e) = transport.send_notification("initialized", json!({})) { try_transition(&state, lang, ServerState::Failed, &event_tx); if let Some(mut c) = child_slot.lock().unwrap().take() { diff --git a/src/commands/lsp_engine/mod.rs b/src/commands/lsp_engine/mod.rs index a6368ef..c8e0a3f 100644 --- a/src/commands/lsp_engine/mod.rs +++ b/src/commands/lsp_engine/mod.rs @@ -4,5 +4,5 @@ mod health; mod installer; mod transport; -pub use engine::{LspEngine, LspEngineConfig}; +pub use engine::{LspClient, LspEngine, LspEngineConfig, LspProvider, NoLsp, ShutdownControl}; pub use installer::InstallProgress; diff --git a/src/commands/lsp_engine/transport.rs b/src/commands/lsp_engine/transport.rs index 16957aa..e38125b 100644 --- a/src/commands/lsp_engine/transport.rs +++ b/src/commands/lsp_engine/transport.rs @@ -1,5 +1,9 @@ +use std::collections::HashMap; use std::io::{BufRead, BufReader, BufWriter, Write}; use std::process::{ChildStdin, ChildStdout}; +use std::sync::atomic::{AtomicBool, AtomicI64, Ordering}; +use std::sync::{Arc, Mutex, mpsc}; +use std::time::{Duration, Instant}; use anyhow::{Result, bail}; use serde::Serialize; @@ -30,6 +34,9 @@ pub(crate) fn decode_lsp_message(reader: &mut R) -> Result { } let length = content_length .ok_or_else(|| anyhow::anyhow!("missing Content-Length header in LSP message"))?; + if length > 16 * 1024 * 1024 { + bail!("LSP message exceeds 16 MiB limit"); + } let mut body = vec![0u8; length]; reader.read_exact(&mut body)?; let value: Value = serde_json::from_slice(&body)?; @@ -51,72 +58,224 @@ struct JsonRpcNotification { params: Value, } +type Response = std::result::Result; +type Pending = Arc>>>; + +/// Cloneable request endpoint. Dedicated threads own both pipes, so neither a +/// silent stdout nor a full stdin pipe can trap a caller beyond its deadline. +#[derive(Clone)] pub struct LspTransport { - writer: BufWriter, - reader: BufReader, - next_id: i64, + outgoing: mpsc::SyncSender, + pending: Pending, + next_id: Arc, + closed: Arc, + timeout: Duration, + cancellation: Option>, +} + +fn fail_pending(pending: &Pending, error: String) { + let requests = std::mem::take(&mut *pending.lock().unwrap()); + for (_, sender) in requests { + let _ = sender.try_send(Err(error.clone())); + } } impl LspTransport { pub fn new(stdin: ChildStdin, stdout: ChildStdout) -> Self { + let (outgoing, rx) = mpsc::sync_channel::(128); + let pending: Pending = Arc::new(Mutex::new(HashMap::new())); + let closed = Arc::new(AtomicBool::new(false)); + let writer_pending = Arc::clone(&pending); + let writer_closed = Arc::clone(&closed); + std::thread::spawn(move || { + let mut writer = BufWriter::new(stdin); + while let Ok(value) = rx.recv() { + let result = serde_json::to_vec(&value) + .map_err(anyhow::Error::from) + .and_then(|body| { + writer.write_all(&encode_lsp_message(&body))?; + writer.flush()?; + Ok(()) + }); + if let Err(error) = result { + writer_closed.store(true, Ordering::Release); + fail_pending(&writer_pending, error.to_string()); + break; + } + } + }); + let reader_pending = Arc::clone(&pending); + let reader_closed = Arc::clone(&closed); + let replies = outgoing.clone(); + std::thread::spawn(move || { + let mut reader = BufReader::new(stdout); + loop { + let message = match decode_lsp_message(&mut reader) { + Ok(message) => message, + Err(error) => { + reader_closed.store(true, Ordering::Release); + fail_pending(&reader_pending, error.to_string()); + break; + } + }; + // Server requests have their own ID space. Never mistake one + // for a response, even if its numeric ID matches ours. + if let Some(method) = message.get("method").and_then(Value::as_str) { + if let Some(id) = message.get("id") { + let result = match method { + "workspace/configuration" => { + let count = message + .pointer("/params/items") + .and_then(Value::as_array) + .map_or(0, Vec::len); + serde_json::json!({"jsonrpc":"2.0", "id": id, "result": vec![Value::Null; count]}) + } + "client/registerCapability" + | "client/unregisterCapability" + | "window/workDoneProgress/create" + | "workspace/workspaceFolders" => { + serde_json::json!({"jsonrpc":"2.0", "id": id, "result": null}) + } + _ => serde_json::json!({"jsonrpc":"2.0", "id": id, + "error": {"code": -32601, "message": "Unsupported client request"}}), + }; + if replies.try_send(result).is_err() { + reader_closed.store(true, Ordering::Release); + fail_pending(&reader_pending, "LSP writer queue full".into()); + break; + } + } + continue; + } + if let Some(id) = message.get("id").and_then(Value::as_i64) + && let Some(sender) = reader_pending.lock().unwrap().remove(&id) + { + let response = if let Some(error) = message.get("error") { + Err(format!("LSP error: {error}")) + } else { + Ok(message.get("result").cloned().unwrap_or(Value::Null)) + }; + let _ = sender.try_send(response); + } + } + }); Self { - writer: BufWriter::new(stdin), - reader: BufReader::new(stdout), - next_id: 0, + outgoing, + pending, + next_id: Arc::new(AtomicI64::new(0)), + closed, + timeout: Duration::from_secs(5), + cancellation: None, } } - pub fn send_request(&mut self, method: &str, params: Value) -> Result { - let id = self.next_id(); - let request = JsonRpcRequest { + pub fn is_closed(&self) -> bool { + self.closed.load(Ordering::Acquire) + } + + pub fn set_timeout(&mut self, timeout: Duration) { + self.timeout = timeout; + } + + pub fn with_cancellation(mut self, cancellation: Arc) -> Self { + self.cancellation = Some(cancellation); + self + } + + pub fn cancel_pending(&self) { + fail_pending(&self.pending, "LSP session closed".into()); + } + + pub(crate) fn start_request(&mut self, method: &str, params: Value) -> Result { + if self.closed.load(Ordering::Acquire) { + bail!("LSP connection closed"); + } + let id = self.next_id.fetch_add(1, Ordering::Relaxed) + 1; + let (tx, rx) = mpsc::sync_channel(1); + self.pending.lock().unwrap().insert(id, tx); + let request = serde_json::to_value(JsonRpcRequest { jsonrpc: "2.0", id, - method: method.to_string(), + method: method.into(), params, - }; - let content = serde_json::to_string(&request)?; - self.write_message(content.as_bytes())?; - - loop { - let msg = self.read_message()?; - if msg.get("id").and_then(|v| v.as_i64()) == Some(id) { - if let Some(error) = msg.get("error") { - let code = error.get("code").and_then(|c| c.as_i64()).unwrap_or(-1); - let message = error - .get("message") - .and_then(|m| m.as_str()) - .unwrap_or("unknown error"); - bail!("LSP error {}: {}", code, message); - } - return Ok(msg.get("result").cloned().unwrap_or(Value::Null)); - } + })?; + if let Err(error) = self.outgoing.try_send(request) { + self.pending.lock().unwrap().remove(&id); + bail!("LSP request queue unavailable: {error}"); } + Ok(PendingRequest { + transport: self.clone(), + id, + receiver: rx, + deadline: Instant::now() + self.timeout, + completed: false, + _timing: crate::commands::diagnostics::Timing::new("lsp_request"), + }) + } + + pub fn send_request(&mut self, method: &str, params: Value) -> Result { + self.start_request(method, params)?.wait() } pub fn send_notification(&mut self, method: &str, params: Value) -> Result<()> { - let notification = JsonRpcNotification { + if self.closed.load(Ordering::Acquire) { + bail!("LSP connection closed"); + } + let notification = serde_json::to_value(JsonRpcNotification { jsonrpc: "2.0", - method: method.to_string(), + method: method.into(), params, - }; - let content = serde_json::to_string(¬ification)?; - self.write_message(content.as_bytes())?; - Ok(()) - } - - fn next_id(&mut self) -> i64 { - self.next_id += 1; - self.next_id + })?; + self.outgoing + .try_send(notification) + .map_err(|e| anyhow::anyhow!("LSP notification queue unavailable: {e}")) } +} - fn write_message(&mut self, content: &[u8]) -> Result<()> { - self.writer.write_all(&encode_lsp_message(content))?; - self.writer.flush()?; - Ok(()) +/// Enqueue under a short document-ordering lock, then wait after releasing it. +pub(crate) struct PendingRequest { + transport: LspTransport, + id: i64, + receiver: mpsc::Receiver, + deadline: Instant, + completed: bool, + _timing: crate::commands::diagnostics::Timing, +} +impl PendingRequest { + pub fn wait(mut self) -> Result { + loop { + if self + .transport + .cancellation + .as_ref() + .is_some_and(|c| c.load(Ordering::Acquire)) + { + bail!("LSP request cancelled"); + } + let remaining = self.deadline.saturating_duration_since(Instant::now()); + if remaining.is_zero() { + bail!("LSP request timed out after {:?}", self.transport.timeout); + } + match self + .receiver + .recv_timeout(remaining.min(Duration::from_millis(10))) + { + Ok(response) => { + self.completed = true; + return response.map_err(anyhow::Error::msg); + } + Err(mpsc::RecvTimeoutError::Timeout) => {} + Err(mpsc::RecvTimeoutError::Disconnected) => bail!("LSP connection closed"), + } + } } - - fn read_message(&mut self) -> Result { - decode_lsp_message(&mut self.reader) +} +impl Drop for PendingRequest { + fn drop(&mut self) { + self.transport.pending.lock().unwrap().remove(&self.id); + if !self.completed { + let _ = self.transport.outgoing.try_send(serde_json::json!({"jsonrpc":"2.0", "method":"$/cancelRequest", "params":{"id":self.id}})); + } } } diff --git a/src/commands/mod.rs b/src/commands/mod.rs index 79eb407..94d9d0b 100644 --- a/src/commands/mod.rs +++ b/src/commands/mod.rs @@ -3,3 +3,5 @@ pub mod chord_engine; pub mod diff; pub mod lsp_engine; pub mod syntax_engine; + +pub mod diagnostics; diff --git a/src/commands/syntax_engine/merge.rs b/src/commands/syntax_engine/merge.rs index 3fd3d30..8a0434b 100644 --- a/src/commands/syntax_engine/merge.rs +++ b/src/commands/syntax_engine/merge.rs @@ -3,16 +3,39 @@ use crate::data::lsp::types::SemanticToken; /// LSP tokens win over tree-sitter tokens on any overlapping character range. /// Both inputs must be sorted by (line, start_col). pub fn merge(ts: &[SemanticToken], lsp: &[SemanticToken]) -> Vec { + let _timing = + crate::commands::diagnostics::Timing::new_count("token_merge", ts.len() + lsp.len()); let mut result = Vec::with_capacity(ts.len() + lsp.len()); - for ts_tok in ts { - let overlaps_lsp = lsp.iter().any(|l| { - l.line == ts_tok.line - && l.start_col < ts_tok.start_col + ts_tok.length - && l.start_col + l.length > ts_tok.start_col + // Collapse LSP intervals into their union. This handles nested and + // overlapping intervals without assuming their ends are monotonic. + let mut intervals: Vec<(usize, usize, usize)> = Vec::new(); + for token in lsp { + let end = token.start_col.saturating_add(token.length); + if let Some(last) = intervals.last_mut() + && last.0 == token.line + && token.start_col < last.2 + { + last.2 = last.2.max(end); + } else { + intervals.push((token.line, token.start_col, end)); + } + } + let mut i = 0; + for token in ts { + while i < intervals.len() + && (intervals[i].0 < token.line + || (intervals[i].0 == token.line && intervals[i].2 <= token.start_col)) + { + i += 1; + } + let overlaps = intervals.get(i).is_some_and(|&(line, start, end)| { + line == token.line + && start < token.start_col.saturating_add(token.length) + && end > token.start_col }); - if !overlaps_lsp { - result.push(ts_tok.clone()); + if !overlaps { + result.push(token.clone()); } } @@ -75,4 +98,49 @@ mod tests { assert_eq!(result[0].token_type, "keyword"); assert_eq!(result[1].token_type, "variable"); } + #[test] + fn sweep_matches_overlap_oracle_for_nested_and_empty_intervals() { + // Exercise sorted inputs with nested tokens, equal starts, touching + // ends and empty ranges, rather than only disjoint happy paths. + let ts: Vec<_> = (0..3) + .flat_map(|line| { + (0..12) + .flat_map(move |start| (0..8).map(move |length| tok(line, start, length, "ts"))) + }) + .collect(); + for seed in 0..40 { + let mut lsp: Vec<_> = (0..30) + .map(|i| { + tok( + (i + seed) % 3, + (i * 7 + seed) % 12, + (i * 11 + seed) % 9, + "lsp", + ) + }) + .collect(); + lsp.sort_by_key(|t| (t.line, t.start_col)); + let mut expected: Vec<_> = ts + .iter() + .filter(|t| { + !lsp.iter().any(|l| { + l.line == t.line + && l.start_col < t.start_col + t.length + && l.start_col + l.length > t.start_col + }) + }) + .cloned() + .collect(); + expected.extend_from_slice(&lsp); + expected.sort_by_key(|t| (t.line, t.start_col)); + let actual = merge(&ts, &lsp); + let key = |tokens: &[SemanticToken]| { + tokens + .iter() + .map(|t| (t.line, t.start_col, t.length, t.token_type.clone())) + .collect::>() + }; + assert_eq!(key(&actual), key(&expected), "seed {seed}"); + } + } } diff --git a/src/commands/syntax_engine/mod.rs b/src/commands/syntax_engine/mod.rs index 5497118..3d26f8f 100644 --- a/src/commands/syntax_engine/mod.rs +++ b/src/commands/syntax_engine/mod.rs @@ -15,6 +15,11 @@ use crate::data::lsp::types::{Language, SemanticToken}; /// Layer 1 defines this; Layer 2 implements it. pub trait SyntaxFrontend: Send + Sync { fn set_semantic_tokens(&self, path: &Path, tokens: Vec); + fn begin_revision(&self, _path: &Path, _hash: u64) {} + fn set_buffer_metrics_versioned(&self, _path: &Path, _hash: u64, _loc: usize, _tokens: usize) {} + fn set_semantic_tokens_versioned(&self, path: &Path, _hash: u64, tokens: Vec) { + self.set_semantic_tokens(path, tokens); + } } struct LspRequest { @@ -56,6 +61,13 @@ impl LspRequestSlot { self.cv.notify_all(); } + fn defer(&self, request: LspRequest) { + let mut state = self.inner.lock().unwrap(); + if state.request.is_none() && !state.shutdown { + state.request = Some(request); + } + } + /// Block until a request is available, or return None on shutdown. fn take(&self) -> Option { let mut s = self.inner.lock().unwrap(); @@ -105,13 +117,15 @@ impl LspRequestSlot { pub struct SyntaxEngine { ts_cache: HashMap)>, + local_slot: Option>, + worker_mode: bool, lsp_cache: Arc>>>, content_hashes: Arc>>, frontend: Arc, request_slot: Arc, } -fn hash_content(content: &str) -> u64 { +pub fn hash_content(content: &str) -> u64 { let mut hasher = DefaultHasher::new(); content.hash(&mut hasher); hasher.finish() @@ -136,6 +150,8 @@ impl SyntaxEngine { Self { ts_cache: HashMap::new(), + local_slot: None, + worker_mode: false, lsp_cache, content_hashes, frontend, @@ -143,22 +159,101 @@ impl SyntaxEngine { } } - /// Returns immediately. Runs tree-sitter synchronously (<2ms), then + /// Local parsing, merging and counting use their own latest-wins worker, + /// independent of the LSP worker and its response latency. + pub fn new_background( + lsp_engine: Arc>, + frontend: Arc, + ) -> Self { + let mut worker = Self::new(lsp_engine, Arc::clone(&frontend)); + worker.worker_mode = true; + let slot = Arc::new(LspRequestSlot::new()); + let shell = Self { + ts_cache: HashMap::new(), + local_slot: Some(Arc::clone(&slot)), + worker_mode: false, + lsp_cache: Arc::clone(&worker.lsp_cache), + content_hashes: Arc::clone(&worker.content_hashes), + frontend, + request_slot: Arc::clone(&worker.request_slot), + }; + std::thread::spawn(move || { + while let Some(mut req) = slot.take() { + while let Some(newer) = slot.wait_for_newer(Duration::from_millis(10)) { + req = newer; + } + if slot.is_shutdown() { + break; + } + if worker.content_hashes.lock().unwrap().get(&req.path) != Some(&req.content_hash) { + continue; + } + worker.compute(&req.path, &req.content); + let loc = req + .content + .lines() + .filter(|line| !line.trim().is_empty()) + .count(); + #[cfg(feature = "frontends")] + let count = { + let _timing = crate::commands::diagnostics::Timing::new("token_count"); + tiktoken::get_encoding("o200k_base") + .expect("built-in encoding") + .count(&req.content) + }; + #[cfg(not(feature = "frontends"))] + let count = 0; + worker.frontend.set_buffer_metrics_versioned( + &req.path, + req.content_hash, + loc, + count, + ); + } + }); + shell + } + + pub fn is_background(&self) -> bool { + self.local_slot.is_some() + } + + /// Runs cached local highlighting synchronously, then /// queues a debounced LSP token request on the background worker. pub fn compute(&mut self, path: &Path, content: &str) { + let content_hash = hash_content(content); + if !self.worker_mode { + self.frontend.begin_revision(path, content_hash); + self.content_hashes + .lock() + .unwrap() + .insert(path.to_path_buf(), content_hash); + } + if let Some(slot) = &self.local_slot { + slot.submit(LspRequest { + path: path.to_path_buf(), + content: content.to_owned(), + content_hash, + ts_tokens: Vec::new(), + }); + return; + } + let _timing = crate::commands::diagnostics::Timing::new("syntax_compute"); let lang = match Language::from_path(path) { Some(l) => l, None => { // Unknown extension: clear any previous tokens so the // frontend renders plain text. Matches spec edge case // "Language with no tree-sitter and no LSP". - self.frontend.set_semantic_tokens(path, Vec::new()); + self.frontend.set_semantic_tokens_versioned( + path, + hash_content(content), + Vec::new(), + ); return; } }; let caps = lang.capabilities(); - let content_hash = hash_content(content); - // Phase 1: tree-sitter (synchronous, cached by content hash) let ts_tokens = if caps.has_tree_sitter { if self.ts_cache.get(path).map(|(h, _)| *h) != Some(content_hash) { @@ -186,13 +281,8 @@ impl SyntaxEngine { }; // Deliver best-effort tokens to frontend immediately - self.frontend.set_semantic_tokens(path, merged); - - // Update content hash for staleness detection by the worker - self.content_hashes - .lock() - .unwrap() - .insert(path.to_path_buf(), content_hash); + self.frontend + .set_semantic_tokens_versioned(path, content_hash, merged); // Phase 2: submit LSP request to the latest-wins slot. Any prior // unprocessed request is silently overwritten — the worker reads @@ -227,9 +317,15 @@ impl SyntaxEngine { } // Fetch LSP semantic tokens - let lsp_tokens = engine - .lock() - .unwrap() + let mut client = engine.lock().unwrap().request_client(); + if client.interactive_pending() { + slot.defer(req); + continue; + } + if content_hashes.lock().unwrap().get(&req.path) != Some(&req.content_hash) { + continue; + } + let lsp_tokens = client .semantic_tokens(&req.path, &req.content) .unwrap_or_default(); @@ -251,7 +347,7 @@ impl SyntaxEngine { } else { req.ts_tokens }; - frontend.set_semantic_tokens(&req.path, merged); + frontend.set_semantic_tokens_versioned(&req.path, req.content_hash, merged); } } } @@ -261,6 +357,9 @@ impl Drop for SyntaxEngine { // Wake the worker thread so it can exit instead of blocking forever // on the slot's condvar. self.request_slot.signal_shutdown(); + if let Some(slot) = &self.local_slot { + slot.signal_shutdown(); + } } } @@ -607,4 +706,62 @@ mod tests { "no additional worker delivery for config languages (has_lsp: false)" ); } + #[test] + fn background_local_worker_keeps_latest_revision_without_blocking_submission() { + use std::sync::{ + atomic::{AtomicBool, AtomicU64, Ordering}, + mpsc, + }; + struct Gated { + current: AtomicU64, + first: AtomicBool, + entered: mpsc::SyncSender<()>, + release: Mutex>, + metrics: mpsc::SyncSender, + } + impl SyntaxFrontend for Gated { + fn set_semantic_tokens(&self, _: &Path, _: Vec) {} + fn begin_revision(&self, _: &Path, hash: u64) { + self.current.store(hash, Ordering::Release); + } + fn set_semantic_tokens_versioned(&self, _: &Path, _: u64, _: Vec) { + if self.first.swap(false, Ordering::AcqRel) { + self.entered.send(()).unwrap(); + self.release + .lock() + .unwrap() + .recv_timeout(Duration::from_secs(5)) + .unwrap(); + } + } + fn set_buffer_metrics_versioned(&self, _: &Path, hash: u64, _: usize, _: usize) { + if self.current.load(Ordering::Acquire) == hash { + self.metrics.send(hash).unwrap(); + } + } + } + let (entered, started) = mpsc::sync_channel(1); + let (release, gate) = mpsc::sync_channel(1); + let (metrics, delivered) = mpsc::sync_channel(1); + let frontend = Arc::new(Gated { + current: AtomicU64::new(0), + first: AtomicBool::new(true), + entered, + release: Mutex::new(gate), + metrics, + }); + let engine = Arc::new(Mutex::new(LspEngine::new(LspEngineConfig::default()))); + let mut syntax = SyntaxEngine::new_background(engine, frontend); + let path = Path::new("latest.json"); + syntax.compute(path, "{\"value\":1}"); + started.recv_timeout(Duration::from_secs(5)).unwrap(); + syntax.compute(path, "{\"value\":2}"); + syntax.compute(path, "{\"value\":3}"); + release.send(()).unwrap(); + assert_eq!( + delivered.recv_timeout(Duration::from_secs(5)).unwrap(), + super::hash_content("{\"value\":3}") + ); + assert!(delivered.try_recv().is_err()); + } } diff --git a/src/commands/syntax_engine/tree_sitter_parse.rs b/src/commands/syntax_engine/tree_sitter_parse.rs index 8eb37e9..d959faa 100644 --- a/src/commands/syntax_engine/tree_sitter_parse.rs +++ b/src/commands/syntax_engine/tree_sitter_parse.rs @@ -8,6 +8,8 @@ thread_local! { const MAX_PARSE_SIZE: usize = 512 * 1024; pub fn parse(lang: Language, content: &str) -> Vec { + let _timing = + crate::commands::diagnostics::Timing::new_count("tree_sitter_parse", content.len()); #[cfg(test)] PARSE_COUNT.with(|c| c.set(c.get() + 1)); if content.len() > MAX_PARSE_SIZE { @@ -56,14 +58,14 @@ fn parse_with( let mut tokens = Vec::new(); let mut cursor = root.walk(); - walk_tree(&mut cursor, content, map_fn, &mut tokens); + walk_tree(&mut cursor, &SourceLines::new(content), map_fn, &mut tokens); tokens.sort_by_key(|t| (t.line, t.start_col)); Some(tokens) } fn walk_tree( cursor: &mut tree_sitter::TreeCursor, - content: &str, + content: &SourceLines, map_fn: fn(&str) -> Option<&'static str>, tokens: &mut Vec, ) { @@ -128,7 +130,7 @@ fn is_leaf_like(kind: &str) -> bool { fn emit_tokens_for_node( node: &tree_sitter::Node, - content: &str, + content: &SourceLines, token_type: &'static str, tokens: &mut Vec, ) { @@ -147,9 +149,10 @@ fn emit_tokens_for_node( }); } } else { - let lines: Vec<&str> = content.lines().collect(); + let lines = &content.lines; for line_num in start_line..=end_line { - if let Some(line_text) = lines.get(line_num) { + if let Some(line) = lines.get(line_num) { + let line_text = line.text; let char_count = line_text.chars().count(); let (start_col, end_col) = if line_num == start_line { let sc = byte_to_char_col(content, line_num, node.start_position().column); @@ -173,15 +176,43 @@ fn emit_tokens_for_node( } } -fn byte_to_char_col(content: &str, line_num: usize, byte_col: usize) -> usize { - content - .lines() - .nth(line_num) - .map(|line| { - let safe_byte = byte_col.min(line.len()); - line[..safe_byte].chars().count() +/// Index each line once, including UTF-8 character boundaries for non-ASCII lines. +/// Token conversion is O(1) for ASCII and O(log line length) for Unicode. +struct SourceLines<'a> { + lines: Vec>, +} + +struct SourceLine<'a> { + text: &'a str, + boundaries: Option>, +} + +impl<'a> SourceLines<'a> { + fn new(content: &'a str) -> Self { + Self { + lines: content + .lines() + .map(|text| SourceLine { + text, + boundaries: (!text.is_ascii()).then(|| { + text.char_indices() + .map(|(i, _)| i) + .chain(std::iter::once(text.len())) + .collect() + }), + }) + .collect(), + } + } +} + +fn byte_to_char_col(content: &SourceLines, line_num: usize, byte_col: usize) -> usize { + content.lines.get(line_num).map_or(0, |line| { + let byte_col = byte_col.min(line.text.len()); + line.boundaries.as_ref().map_or(byte_col, |boundaries| { + boundaries.partition_point(|&offset| offset < byte_col) }) - .unwrap_or(0) + }) } fn parse_markdown(content: &str) -> Vec { @@ -333,12 +364,16 @@ fn parse_json(content: &str) -> Vec { }; let mut tokens = Vec::new(); let mut cursor = tree.root_node().walk(); - walk_json(&mut cursor, content, &mut tokens); + walk_json(&mut cursor, &SourceLines::new(content), &mut tokens); tokens.sort_by_key(|t| (t.line, t.start_col)); tokens } -fn walk_json(cursor: &mut tree_sitter::TreeCursor, content: &str, tokens: &mut Vec) { +fn walk_json( + cursor: &mut tree_sitter::TreeCursor, + content: &SourceLines, + tokens: &mut Vec, +) { loop { let node = cursor.node(); let kind = node.kind(); @@ -402,14 +437,14 @@ fn parse_yaml(content: &str) -> Vec { }; let mut tokens = Vec::new(); let mut cursor = tree.root_node().walk(); - walk_yaml(&mut cursor, content, &mut tokens, false); + walk_yaml(&mut cursor, &SourceLines::new(content), &mut tokens, false); tokens.sort_by_key(|t| (t.line, t.start_col)); tokens } fn walk_yaml( cursor: &mut tree_sitter::TreeCursor, - content: &str, + content: &SourceLines, tokens: &mut Vec, is_key: bool, ) { @@ -527,12 +562,16 @@ fn parse_xml(content: &str) -> Vec { }; let mut tokens = Vec::new(); let mut cursor = tree.root_node().walk(); - walk_xml(&mut cursor, content, &mut tokens); + walk_xml(&mut cursor, &SourceLines::new(content), &mut tokens); tokens.sort_by_key(|t| (t.line, t.start_col)); tokens } -fn walk_xml(cursor: &mut tree_sitter::TreeCursor, content: &str, tokens: &mut Vec) { +fn walk_xml( + cursor: &mut tree_sitter::TreeCursor, + content: &SourceLines, + tokens: &mut Vec, +) { loop { let node = cursor.node(); let kind = node.kind(); @@ -579,7 +618,7 @@ fn walk_xml(cursor: &mut tree_sitter::TreeCursor, content: &str, tokens: &mut Ve fn walk_xml_attribute( cursor: &mut tree_sitter::TreeCursor, - content: &str, + content: &SourceLines, tokens: &mut Vec, ) { loop { @@ -775,4 +814,22 @@ mod tests { "content over 512 KB should return empty tokens" ); } + #[test] + fn indexed_columns_preserve_unicode_crlf_and_final_lines() { + let content = "éλx\r\nascii\n😺z\n"; + let source = super::SourceLines::new(content); + for (row, line) in content.lines().enumerate() { + for (byte, _) in line + .char_indices() + .chain(std::iter::once((line.len(), ' '))) + { + assert_eq!( + super::byte_to_char_col(&source, row, byte), + line[..byte].chars().count() + ); + } + } + assert_eq!(super::byte_to_char_col(&source, 3, 0), 0); + assert!(parse(Language::Rust, &"let value = 123;\n".repeat(10_000)).len() >= 20_000); + } } diff --git a/src/data/state.rs b/src/data/state.rs index a3dc87f..df4cbad 100644 --- a/src/data/state.rs +++ b/src/data/state.rs @@ -72,6 +72,7 @@ pub struct EditorState { pub buffers: Vec, pub active_buffer: usize, pub file_tree: Option, + pub tree_loading: bool, pub cursor_line: usize, pub cursor_col: usize, pub scroll_offset: usize, @@ -95,6 +96,8 @@ pub struct EditorState { pub selection: Option, pub list_dialog: Option, pub cached_token_count: usize, + pub cached_loc: usize, + pub buffer_generation: u64, pub disk_changed_path: Option, pub pending_rewatch_path: Option, pub tree_rename_state: Option, @@ -113,6 +116,7 @@ impl EditorState { buffers: vec![buf], active_buffer: 0, file_tree: None, + tree_loading: false, cursor_line: 0, cursor_col: 0, scroll_offset: 0, @@ -136,6 +140,8 @@ impl EditorState { selection: None, list_dialog: None, cached_token_count: 0, + cached_loc: 0, + buffer_generation: 0, disk_changed_path: None, pending_rewatch_path: None, tree_rename_state: None, @@ -145,7 +151,21 @@ impl EditorState { } pub fn for_directory(path: &Path) -> Result { - let tree = FileTree::from_dir(path)?; + Self::with_tree(path, FileTree::from_dir(path)?, false) + } + + pub fn for_directory_loading(path: &Path) -> Result { + Self::with_tree( + path, + FileTree { + root: path.to_path_buf(), + entries: Vec::new(), + }, + true, + ) + } + + fn with_tree(path: &Path, tree: FileTree, loading: bool) -> Result { let tree_view: Vec = tree .entries .iter() @@ -156,6 +176,7 @@ impl EditorState { buffers: Vec::new(), active_buffer: 0, file_tree: Some(tree), + tree_loading: loading, cursor_line: 0, cursor_col: 0, scroll_offset: 0, @@ -179,6 +200,8 @@ impl EditorState { selection: None, list_dialog: None, cached_token_count: 0, + cached_loc: 0, + buffer_generation: 0, disk_changed_path: None, pending_rewatch_path: None, tree_rename_state: None, diff --git a/src/frontend/tui/app.rs b/src/frontend/tui/app.rs index a4b0c2d..4e8ca26 100644 --- a/src/frontend/tui/app.rs +++ b/src/frontend/tui/app.rs @@ -43,8 +43,17 @@ use super::tree_pane; fn refresh_buffer_caches(state: &mut EditorState, syntax_engine: &mut SyntaxEngine) { if let Some(buf) = state.current_buffer() { let content = buf.content(); + let loc = if syntax_engine.is_background() { + 0 + } else { + title_bar::compute_loc(&buf.lines) + }; syntax_engine.compute(&buf.path, &content); - state.cached_token_count = title_bar::compute_token_count(&content); + state.buffer_generation = state.buffer_generation.wrapping_add(1); + if !syntax_engine.is_background() { + state.cached_token_count = title_bar::compute_token_count(&content); + state.cached_loc = loc; + } } } @@ -143,12 +152,8 @@ fn delete_selection(state: &mut EditorState) -> bool { fn compute_text_width(state: &EditorState, term_width: u16) -> usize { let total = state.current_buffer().map_or(1, |b| b.line_count()); let line_num_width = format!("{}", total.saturating_sub(1)).len(); - let has_tree = state.file_tree.is_some() && state.focus_tree; - let editor_width = if has_tree { - (term_width as usize) / 2 - } else { - term_width as usize - }; + let editor_width = + compute_editor_render_area(state, Rect::new(0, 0, term_width, 24)).width as usize; editor_width.saturating_sub(line_num_width + 1) } @@ -272,8 +277,6 @@ fn move_cursor_down(state: &mut EditorState, text_width: usize) { use super::tui_frontend::TuiFrontend; -use crate::frontend::traits::ApplyChordAction; - struct TuiInstallProgress { shared: Arc>, } @@ -293,40 +296,112 @@ impl InstallProgress for TuiInstallProgress { } } -/// TUI implementation of SyntaxFrontend — stores tokens per-path for the render loop. +/// Immutable tokens shared with rendering; revisions gate both local and LSP results. +#[derive(Default)] +struct SyntaxSnapshot { + hash: u64, + tokens: Arc>, + metrics: Option<(usize, usize)>, +} struct TuiSyntaxReceiver { - tokens: Arc>>>, + snapshots: Mutex>, } - impl TuiSyntaxReceiver { fn new() -> Self { Self { - tokens: Arc::new(Mutex::new(HashMap::new())), + snapshots: Mutex::new(HashMap::new()), } } - - fn tokens_for(&self, path: &Path) -> Vec { - self.tokens + fn tokens_for(&self, path: &Path) -> Arc> { + self.snapshots .lock() .unwrap() .get(path) - .cloned() + .map(|s| Arc::clone(&s.tokens)) .unwrap_or_default() } + fn metrics_for(&self, path: &Path) -> Option<(usize, usize)> { + self.snapshots + .lock() + .unwrap() + .get(path) + .and_then(|s| s.metrics) + } } - impl SyntaxFrontend for TuiSyntaxReceiver { + fn begin_revision(&self, path: &Path, hash: u64) { + let mut snapshots = self.snapshots.lock().unwrap(); + let snapshot = snapshots.entry(path.to_path_buf()).or_default(); + if snapshot.hash != hash { + snapshot.hash = hash; + snapshot.metrics = None; + } + } fn set_semantic_tokens(&self, path: &Path, tokens: Vec) { - self.tokens + self.snapshots .lock() .unwrap() - .insert(path.to_path_buf(), tokens); + .entry(path.to_path_buf()) + .or_default() + .tokens = Arc::new(tokens); + } + fn set_semantic_tokens_versioned(&self, path: &Path, hash: u64, tokens: Vec) { + let mut snapshots = self.snapshots.lock().unwrap(); + if let Some(snapshot) = snapshots.get_mut(path) + && snapshot.hash == hash + { + snapshot.tokens = Arc::new(tokens); + } + } + fn set_buffer_metrics_versioned(&self, path: &Path, hash: u64, loc: usize, tokens: usize) { + let mut snapshots = self.snapshots.lock().unwrap(); + if let Some(snapshot) = snapshots.get_mut(path) + && snapshot.hash == hash + { + snapshot.metrics = Some((loc, tokens)); + } + } +} + +/// Restore each successfully enabled terminal feature, even during setup errors. +struct TerminalGuard { + raw: bool, + alternate: bool, + mouse: bool, +} +impl TerminalGuard { + fn enter() -> Result { + let mut guard = Self { + raw: false, + alternate: false, + mouse: false, + }; + enable_raw_mode()?; + guard.raw = true; + io::stdout().execute(EnterAlternateScreen)?; + guard.alternate = true; + io::stdout().execute(EnableMouseCapture)?; + guard.mouse = true; + Ok(guard) + } +} +impl Drop for TerminalGuard { + fn drop(&mut self) { + if self.mouse { + let _ = io::stdout().execute(DisableMouseCapture); + } + if self.raw { + let _ = disable_raw_mode(); + } + if self.alternate { + let _ = io::stdout().execute(LeaveAlternateScreen); + } } } pub fn run(path: &Path) -> Result<()> { let mut state = if path.is_dir() { - EditorState::for_directory(path)? + EditorState::for_directory_loading(path)? } else { EditorState::for_file(path)? }; @@ -352,7 +427,7 @@ pub fn run(path: &Path) -> Result<()> { // Create SyntaxEngine (Layer 1) — it owns the LspEngine reference // and spawns its own background worker for debounced LSP requests - let mut syntax_engine = SyntaxEngine::new( + let mut syntax_engine = SyntaxEngine::new_background( Arc::clone(&engine), Arc::clone(&syntax_receiver) as Arc, ); @@ -380,12 +455,11 @@ pub fn run(path: &Path) -> Result<()> { }); } - enable_raw_mode()?; - io::stdout().execute(EnterAlternateScreen)?; - io::stdout().execute(EnableMouseCapture)?; + let terminal_guard = TerminalGuard::enter()?; let backend = CrosstermBackend::new(io::stdout()); let mut terminal = Terminal::new(backend)?; + let shutdown = engine.lock().unwrap().shutdown_control(); let mut frontend = TuiFrontend::new(); let result = event_loop( &mut terminal, @@ -397,13 +471,18 @@ pub fn run(path: &Path) -> Result<()> { &mut fs_watcher, ); - { - let mut eng = engine.lock().unwrap(); - eng.shutdown_all(); + frontend.cancel_chord(&mut state); + drop(terminal); + drop(terminal_guard); + drop(syntax_engine); + let (done, finished) = std::sync::mpsc::sync_channel(1); + std::thread::spawn(move || { + engine.lock().unwrap().shutdown_all(); + let _ = done.send(()); + }); + if finished.recv_timeout(Duration::from_secs(2)).is_err() { + shutdown.terminate(); } - io::stdout().execute(DisableMouseCapture)?; - disable_raw_mode()?; - io::stdout().execute(LeaveAlternateScreen)?; result } @@ -454,12 +533,61 @@ fn event_loop( let mut prev_lsp_statuses: Vec<(Language, ServerState)> = Vec::new(); loop { + let _iteration = crate::commands::diagnostics::Timing::new("event_loop"); + if frontend.poll_chord(state) { + refresh_buffer_caches(state, syntax_engine); + } if let Some(watcher) = fs_watcher.as_mut() { - while let Ok(event) = watcher.rx.try_recv() { + if let Some(snapshot) = watcher.take_tree() { + match snapshot { + Ok(tree) => apply_tree_snapshot(state, tree), + Err(error) => { + state.tree_loading = false; + state.file_tree = None; + state.tree_view.clear(); + state.focus_tree = false; + state.mode = state.pre_tree_mode; + watcher.unwatch_tree(); + state.status_msg = format!("tree error: {error}"); + } + } + } + let mut batch = crate::commands::diagnostics::Timing::new("filesystem_batch"); + let mut event_count = 0; + let batch_start = std::time::Instant::now(); + let mut scan_changed = false; + for _ in 0..64 { + if batch_start.elapsed() >= Duration::from_millis(5) { + break; + } + let Some(event) = watcher.next_event() else { + break; + }; + event_count += 1; if let Ok(ev) = event { + scan_changed |= state.tree_loading + && matches!( + ev.kind, + EventKind::Create(_) + | EventKind::Remove(_) + | EventKind::Modify(ModifyKind::Name(_)) + ); handle_fs_event(&ev, state, watcher); } } + batch.set_count(event_count); + if watcher.take_overflow() || scan_changed { + let _ = watcher.reconcile_tree(); + // Lost events may also include the active file. Reconcile its + // disk state rather than silently ignoring an overflow. + if let Some(path) = watcher.watched_file().map(Path::to_path_buf) { + if path.exists() { + mark_disk_changed(state); + } else { + handle_active_file_removed(state); + } + } + } } let lsp_statuses: Vec<(Language, ServerState)> = { @@ -479,19 +607,28 @@ fn event_loop( } prev_lsp_statuses = lsp_statuses.clone(); + if let Some(buf) = state.current_buffer() + && let Some((loc, count)) = syntax_receiver.metrics_for(&buf.path) + { + state.cached_loc = loc; + state.cached_token_count = count; + } // Read tokens from the syntax receiver let tokens = if let Some(buf) = state.current_buffer() { syntax_receiver.tokens_for(&buf.path) } else { - vec![] + Arc::default() }; let term_size = terminal.size()?; adjust_scroll_offset(state, term_size.height, term_size.width); - terminal.draw(|frame| { - draw(frame, state, &tokens, &lsp_statuses); - })?; + { + let _timing = crate::commands::diagnostics::Timing::new("render"); + terminal.draw(|frame| { + draw(frame, state, &tokens, &lsp_statuses); + })?; + } if state.should_quit { return Ok(()); @@ -502,7 +639,6 @@ fn event_loop( let prev_active = state.active_buffer; let prev_buf_path = state.current_buffer().map(|b| b.path.clone()); - let had_tree = state.file_tree.is_some(); if event::poll(Duration::from_millis(50))? { match event::read()? { @@ -518,6 +654,7 @@ fn event_loop( term_size.width, ); if buffer_modified { + frontend.cancel_chord(state); state.selection = None; refresh_buffer_caches(state, syntax_engine); } @@ -556,17 +693,61 @@ fn event_loop( } let has_tree = state.file_tree.is_some(); - if has_tree && !had_tree { + if has_tree && watcher.watched_tree().is_none() { if let Some(tree) = &state.file_tree { let _ = watcher.watch_tree(&tree.root); } - } else if !has_tree && had_tree { + } else if !has_tree && watcher.watched_tree().is_some() { watcher.unwatch_tree(); } } } } +fn apply_tree_snapshot(state: &mut EditorState, tree: crate::data::file_tree::FileTree) { + use std::collections::HashSet; + let selected = state + .tree_view + .get(state.tree_selected) + .map(|e| e.path.clone()) + .or_else(|| { + (state.tree_loading && state.tree_view.is_empty()) + .then(|| state.current_buffer().map(|b| b.path.clone())) + .flatten() + }); + let expanded: HashSet<_> = state + .tree_view + .windows(2) + .filter(|pair| pair[1].depth > pair[0].depth) + .map(|pair| pair[0].path.clone()) + .collect(); + let mut hidden_depth = None; + let mut view = Vec::new(); + for entry in &tree.entries { + if hidden_depth.is_some_and(|depth| entry.depth > depth) { + continue; + } + hidden_depth = None; + view.push(entry.clone()); + if entry.is_dir && !expanded.contains(&entry.path) { + hidden_depth = Some(entry.depth); + } + } + state.tree_view = view; + state.tree_selected = selected + .and_then(|path| state.tree_view.iter().position(|e| e.path == path)) + .unwrap_or( + state + .tree_selected + .min(state.tree_view.len().saturating_sub(1)), + ); + state.file_tree = Some(tree); + state.tree_loading = false; + if state.status_msg == "loading file tree" { + state.status_msg = "file tree opened".into(); + } +} + fn adjust_scroll_offset(state: &mut EditorState, term_height: u16, term_width: u16) { let total = match state.current_buffer() { Some(buf) => buf.line_count(), @@ -584,49 +765,43 @@ fn adjust_scroll_offset(state: &mut EditorState, term_height: u16, term_width: u } let line_num_width = format!("{}", total.saturating_sub(1)).len(); - let has_tree = state.file_tree.is_some() && state.focus_tree; - let editor_width = if has_tree { - (term_width as usize) / 2 - } else { - term_width as usize - }; + let editor_width = + compute_editor_render_area(state, Rect::new(0, 0, term_width, term_height)).width as usize; let text_width = editor_width.saturating_sub(line_num_width + 1); if state.cursor_line < state.scroll_offset { state.scroll_offset = state.cursor_line; } - let active = state.active_buffer; - loop { - let mut visual_rows = 0; - let cursor = state.cursor_line.min(total.saturating_sub(1)); - for i in state.scroll_offset..=cursor { - let line = state.buffers[active] - .lines - .get(i) - .map(|s| s.as_str()) - .unwrap_or(""); - if i == cursor { - let cursor_display_col = editor_pane::display_col(line, state.cursor_col); - let offsets = editor_pane::wrap_offsets(line, text_width); - let (cursor_row_in_line, _) = - editor_pane::display_col_to_wrap_pos(&offsets, cursor_display_col); - visual_rows += cursor_row_in_line + 1; - } else { - visual_rows += editor_pane::visual_row_count(line, text_width); - } - } - - if visual_rows <= visible || state.scroll_offset >= state.cursor_line { + // Walk backwards only as far as the viewport. Repeatedly scanning the + // entire distance from scroll_offset to a distant jump was quadratic. + let buffer = &state.buffers[state.active_buffer]; + let cursor = state.cursor_line.min(total.saturating_sub(1)); + let line = buffer.lines.get(cursor).map(String::as_str).unwrap_or(""); + let cursor_display = editor_pane::display_col(line, state.cursor_col); + let mut prefix_end = state.cursor_col.min(line.len()); + while !line.is_char_boundary(prefix_end) { + prefix_end -= 1; + } + let prefix_end = line[prefix_end..] + .char_indices() + .nth(text_width.max(1)) + .map_or(line.len(), |(offset, _)| prefix_end + offset); + let offsets = editor_pane::wrap_offsets(&line[..prefix_end], text_width); + let cursor_rows = editor_pane::display_col_to_wrap_pos(&offsets, cursor_display).0 + 1; + let mut remaining = visible.saturating_sub(cursor_rows); + let mut first = cursor; + let old = state.scroll_offset.min(cursor); + while first > old && remaining > 0 { + let line = editor_pane::visible_prefix(&buffer.lines[first - 1], text_width, remaining); + let rows = editor_pane::visual_row_count(line, text_width); + if rows > remaining { break; } - - state.scroll_offset += 1; - } - - if state.scroll_offset >= total { - state.scroll_offset = total.saturating_sub(1); + remaining -= rows; + first -= 1; } + state.scroll_offset = first.min(total.saturating_sub(1)); } struct PaneLayout { @@ -643,10 +818,8 @@ fn compute_pane_layout(state: &EditorState, total: Rect) -> PaneLayout { let has_tree = state.file_tree.is_some() && state.focus_tree; let h_constraints = if has_tree { - let content_w = tree_pane::content_width(&state.tree_view) as u16; - let desired = content_w + 3; - let max_w = total.width / 2; - let tree_w = desired.min(max_w); + let content_w = tree_pane::content_width(&state.tree_view); + let tree_w = content_w.saturating_add(3).min((total.width / 2) as usize) as u16; vec![Constraint::Length(tree_w), Constraint::Min(0)] } else { vec![Constraint::Length(0), Constraint::Percentage(100)] @@ -791,6 +964,10 @@ fn handle_key( return false; } + if code == KeyCode::Esc && frontend.has_pending() { + frontend.cancel_chord(state); + return false; + } // Priority 4: Ctrl-O reloads file from disk when disk_changed is set if code == KeyCode::Char('o') && modifiers.contains(KeyModifiers::CONTROL) @@ -1093,14 +1270,6 @@ fn handle_chord_mode( Ok(_) => { clear_chord(state); execute_chord_input(state, frontend, engine, &input, lsp_statuses); - if !state.status_msg.starts_with("error:") - && !state.status_msg.starts_with("resolve error:") - && !state.status_msg.starts_with("patch error:") - && !state.status_msg.starts_with("parse error:") - { - state.chord_history.push(input.clone()); - } - refresh_buffer_caches(state, syntax_engine); } Err(_) => { state.chord_error = true; @@ -1138,7 +1307,7 @@ fn try_auto_submit( state: &mut EditorState, frontend: &mut TuiFrontend, engine: &Arc>, - syntax_engine: &mut SyntaxEngine, + _syntax_engine: &mut SyntaxEngine, lsp_statuses: &[(Language, ServerState)], ) { let input = &state.chord_input; @@ -1148,18 +1317,8 @@ fn try_auto_submit( ChordEngine::try_auto_submit_short(input, state.cursor_line, state.cursor_col) { let input_clone = state.chord_input.clone(); - state.chord_running = true; clear_chord(state); execute_chord_input(state, frontend, engine, &input_clone, lsp_statuses); - if !state.status_msg.starts_with("error:") - && !state.status_msg.starts_with("resolve error:") - && !state.status_msg.starts_with("patch error:") - && !state.status_msg.starts_with("parse error:") - { - state.chord_history.push(input_clone); - } - state.chord_running = false; - refresh_buffer_caches(state, syntax_engine); } } else if input.ends_with(')') && input.chars().next().is_some_and(|c| c.is_uppercase()) @@ -1167,18 +1326,8 @@ fn try_auto_submit( && ChordEngine::parse(input).is_ok() { let input_clone = state.chord_input.clone(); - state.chord_running = true; clear_chord(state); execute_chord_input(state, frontend, engine, &input_clone, lsp_statuses); - if !state.status_msg.starts_with("error:") - && !state.status_msg.starts_with("resolve error:") - && !state.status_msg.starts_with("patch error:") - && !state.status_msg.starts_with("parse error:") - { - state.chord_history.push(input_clone); - } - state.chord_running = false; - refresh_buffer_caches(state, syntax_engine); } } @@ -1325,10 +1474,15 @@ fn mark_disk_changed(state: &mut EditorState) { } fn handle_fs_event(event: ¬ify::Event, state: &mut EditorState, watcher: &mut FsWatcher) { + if matches!(event.kind, EventKind::Access(_)) { + return; + } let watched_file = watcher.watched_file().map(|p| p.to_path_buf()); for path in &event.paths { - let canonical = path.canonicalize().unwrap_or_else(|_| path.clone()); + // Worker-installed watches use canonical roots; notification paths + // already share that namespace, including paths that no longer exist. + let canonical = path.clone(); // Fast path: a file we're waiting to re-watch just reappeared (atomic save completed). if let Some(rewatch) = &state.pending_rewatch_path @@ -1407,6 +1561,10 @@ fn handle_fs_event(event: ¬ify::Event, state: &mut EditorState, watcher: &mut } } + let selected = state + .tree_view + .get(state.tree_selected) + .map(|e| e.path.clone()); if state.file_tree.is_some() { match event.kind { EventKind::Create(_) | EventKind::Remove(_) => { @@ -1418,6 +1576,11 @@ fn handle_fs_event(event: ¬ify::Event, state: &mut EditorState, watcher: &mut _ => {} } } + if let Some(path) = selected + && let Some(index) = state.tree_view.iter().position(|e| e.path == path) + { + state.tree_selected = index; + } } fn remove_path_from_tree(state: &mut EditorState, path: &Path) { @@ -1492,6 +1655,7 @@ fn rename_subtree(tree_opt: &mut Option, from: entry.path = new_path; } } + tree.entries.sort_by(|a, b| a.path.cmp(&b.path)); } fn rename_in_tree_view(tree_view: &mut [FileEntry], from: &Path, to: &Path) { @@ -1581,9 +1745,10 @@ fn insert_entry_sorted( if !canonical.starts_with(&tree.root) { return None; } - if tree.entries.iter().any(|e| e.path == canonical) { - return None; - } + let insert = match tree.entries.binary_search_by(|e| e.path.cmp(&canonical)) { + Ok(_) => return None, + Err(index) => index, + }; let depth = canonical .strip_prefix(&tree.root) .map(|rel| rel.components().count().saturating_sub(1)) @@ -1594,8 +1759,7 @@ fn insert_entry_sorted( depth, is_dir, }; - tree.entries.push(entry.clone()); - tree.entries.sort_by(|a, b| a.path.cmp(&b.path)); + tree.entries.insert(insert, entry.clone()); Some(entry) } @@ -2037,30 +2201,23 @@ fn toggle_tree(state: &mut EditorState) { _ => PathBuf::from("."), } }; - match crate::data::file_tree::FileTree::from_dir(&dir) { - Ok(tree) => { - let tree_view: Vec<_> = tree - .entries - .iter() - .filter(|e| e.depth == 0) - .cloned() - .collect(); - state.file_tree = Some(tree); - state.tree_view = tree_view; - state.pre_tree_mode = state.mode; - state.focus_tree = true; - if let Some(buf) = state.current_buffer() { - let buf_path = buf.path.clone(); - if let Some(idx) = state.tree_view.iter().position(|e| e.path == buf_path) { - state.tree_selected = idx; - } - } - state.status_msg = "file tree opened".into(); - } - Err(e) => { - state.status_msg = format!("tree error: {e}"); - } - } + let dir = if dir.is_absolute() { + dir + } else { + std::env::current_dir() + .unwrap_or_else(|_| PathBuf::from(".")) + .join(dir) + }; + state.file_tree = Some(crate::data::file_tree::FileTree { + root: dir, + entries: Vec::new(), + }); + state.tree_view.clear(); + state.tree_selected = 0; + state.tree_loading = true; + state.pre_tree_mode = state.mode; + state.focus_tree = true; + state.status_msg = "loading file tree".into(); } } @@ -2095,41 +2252,32 @@ fn execute_chord_input( query.args.cursor_pos = Some((state.cursor_line, state.cursor_col)); - let mut buffers = HashMap::new(); - if let Some(buf) = state.current_buffer() { - let path_str = buf.path.to_string_lossy().to_string(); - buffers.insert(path_str, buf.clone()); - } - - let resolve_result = { - let mut eng = engine.lock().unwrap(); - ChordEngine::resolve(&query, &buffers, &mut eng) - }; - - match resolve_result { - Ok(resolved) => match ChordEngine::patch(&resolved, &buffers) { - Ok(actions) => { - for action in actions.values() { - match frontend.apply(state, action) { - Ok(msg) => { - if !msg.is_empty() && state.status_msg.is_empty() { - state.status_msg = msg; - } - } - Err(e) => { - state.status_msg = format!("error: {e}"); - } - } + let cancel = Arc::new(std::sync::atomic::AtomicBool::new(false)); + let provider: Box = + if query.requires_lsp { + match engine.try_lock() { + Ok(mut engine) => { + let contents = state + .current_buffer() + .map(|buf| (buf.path.clone(), buf.content())) + .into_iter() + .collect(); + Box::new( + engine + .request_client() + .with_contents(contents) + .with_cancellation(Arc::clone(&cancel)), + ) + } + Err(_) => { + state.status_msg = "LSP busy — retry chord".into(); + return; } } - Err(e) => { - state.status_msg = format!("patch error: {e}"); - } - }, - Err(e) => { - state.status_msg = format!("resolve error: {e}"); - } - } + } else { + Box::new(crate::commands::lsp_engine::NoLsp) + }; + frontend.submit_chord(state, input, query, provider, cancel); } Err(e) => { state.status_msg = format!("parse error: {e}"); @@ -3735,4 +3883,48 @@ mod tests { "focus_tree must not toggle when Ctrl-T is pressed mid-rename" ); } + #[test] + fn old_syntax_and_metrics_cannot_overwrite_new_revision() { + let receiver = TuiSyntaxReceiver::new(); + let path = Path::new("test.rs"); + receiver.begin_revision(path, 1); + receiver.begin_revision(path, 2); + receiver.set_semantic_tokens_versioned( + path, + 1, + vec![SemanticToken { + line: 0, + start_col: 0, + length: 1, + token_type: "keyword".into(), + }], + ); + receiver.set_buffer_metrics_versioned(path, 1, 100, 200); + assert!(receiver.tokens_for(path).is_empty()); + assert!(receiver.metrics_for(path).is_none()); + receiver.set_buffer_metrics_versioned(path, 2, 3, 4); + assert_eq!(receiver.metrics_for(path), Some((3, 4))); + } + + #[test] + fn distant_jump_scrolls_to_viewport_without_scanning_the_prefix() { + let (_f, mut state) = make_state_with_lines(&["x"]); + state.buffers[0].lines = vec!["x".into(); 100_000]; + state.mode = Mode::Edit; + state.cursor_line = 99_999; + adjust_scroll_offset(&mut state, 24, 80); + assert_eq!(state.scroll_offset, 99_978); + } + + #[test] + fn non_lsp_chord_submission_does_not_acquire_engine_lock() { + let (_f, mut state) = make_state_with_lines(&["hello"]); + let engine = Arc::new(Mutex::new(LspEngine::new(LspEngineConfig::default()))); + let guard = engine.lock().unwrap(); + let mut frontend = TuiFrontend::new(); + execute_chord_input(&mut state, &mut frontend, &engine, "yebs", &[]); + assert!(frontend.has_pending()); + drop(guard); + frontend.cancel_chord(&mut state); + } } diff --git a/src/frontend/tui/background_chord.rs b/src/frontend/tui/background_chord.rs new file mode 100644 index 0000000..37a3997 --- /dev/null +++ b/src/frontend/tui/background_chord.rs @@ -0,0 +1,80 @@ +//! A bounded, cancellable chord worker. UI state is only mutated by `poll`. +use std::collections::HashMap; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Condvar, Mutex, mpsc}; + +use crate::commands::chord_engine::ChordEngine; +use crate::commands::chord_engine::types::{ChordAction, ChordQuery}; +use crate::commands::lsp_engine::LspProvider; +use crate::data::buffer::Buffer; + +pub(super) type ChordResult = Result, String>; +pub(super) struct Job { + pub query: ChordQuery, + pub buffers: HashMap, + pub provider: Box, + pub cancel: Arc, + pub reply: mpsc::SyncSender, +} +#[derive(Default)] +struct Queue { + job: Option, + shutdown: bool, +} + +pub(super) struct ChordWorker { + queue: Arc<(Mutex, Condvar)>, +} +impl ChordWorker { + pub fn new() -> Self { + let queue = Arc::new((Mutex::new(Queue::default()), Condvar::new())); + let worker = Arc::clone(&queue); + std::thread::spawn(move || { + loop { + let (lock, cv) = &*worker; + let mut state = lock.lock().unwrap(); + while state.job.is_none() && !state.shutdown { + state = cv.wait(state).unwrap(); + } + if state.shutdown { + break; + } + let mut job = state.job.take().unwrap(); + drop(state); + if job.cancel.load(Ordering::Acquire) { + continue; + } + let _timing = crate::commands::diagnostics::Timing::new("chord_resolution"); + let result = ChordEngine::resolve(&job.query, &job.buffers, &mut *job.provider) + .map_err(|error| format!("resolve error: {error}")) + .and_then(|resolved| { + ChordEngine::patch(&resolved, &job.buffers) + .map_err(|error| format!("patch error: {error}")) + }); + if !job.cancel.load(Ordering::Acquire) { + let _ = job.reply.try_send(result); + } + } + }); + Self { queue } + } + pub fn submit(&self, job: Job) { + let (lock, cv) = &*self.queue; + let mut queue = lock.lock().unwrap(); + if let Some(old) = queue.job.replace(job) { + old.cancel.store(true, Ordering::Release); + } + cv.notify_one(); + } +} +impl Drop for ChordWorker { + fn drop(&mut self) { + let (lock, cv) = &*self.queue; + let mut queue = lock.lock().unwrap(); + queue.shutdown = true; + if let Some(job) = queue.job.take() { + job.cancel.store(true, Ordering::Release); + } + cv.notify_one(); + } +} diff --git a/src/frontend/tui/editor_pane.rs b/src/frontend/tui/editor_pane.rs index a9cbbf6..b62c516 100644 --- a/src/frontend/tui/editor_pane.rs +++ b/src/frontend/tui/editor_pane.rs @@ -161,7 +161,7 @@ pub(crate) fn screen_to_buffer( let mut logical_line = state.scroll_offset; while logical_line < buf.line_count() { - let line_text = &buf.lines[logical_line]; + let line_text = visible_prefix(&buf.lines[logical_line], text_width, target_visual_row + 1); let row_count = visual_row_count(line_text, text_width); if accumulated_rows + row_count > target_visual_row { @@ -185,6 +185,19 @@ pub(crate) fn screen_to_buffer( } } +pub(crate) fn visible_prefix(line: &str, width: usize, rows: usize) -> &str { + // For zero-width panes still bound work to a single cell per row. + let limit = width.max(1).saturating_mul(rows.max(1)).saturating_add(1); + let mut columns = 0; + for (byte, character) in line.char_indices() { + if columns >= limit { + return &line[..byte]; + } + columns += if character == '\t' { 4 } else { 1 }; + } + line +} + fn styled_line_with_tokens<'a>( line_text: &'a str, line_num: usize, @@ -196,7 +209,9 @@ fn styled_line_with_tokens<'a>( return vec![Span::styled(expand_tabs(line_text), base_style)]; } - let line_tokens: Vec<&SemanticToken> = tokens.iter().filter(|t| t.line == line_num).collect(); + let first = tokens.partition_point(|token| token.line < line_num); + let end = tokens.partition_point(|token| token.line <= line_num); + let line_tokens: Vec<&SemanticToken> = tokens[first..end].iter().collect(); if line_tokens.is_empty() { return vec![Span::styled(expand_tabs(line_text), base_style)]; } @@ -252,7 +267,7 @@ fn wrap_spans(spans: Vec>, text_width: usize) -> Vec> let mut span_chars: Vec<(char, Style)> = Vec::new(); for span in &spans { let style = span.style; - for ch in span.content.chars() { + for ch in expand_tabs(&span.content).chars() { span_chars.push((ch, style)); } } @@ -314,7 +329,11 @@ fn render_selection_highlight( let mut line_idx = scroll_offset; while accumulated_rows < visible_height && line_idx < buf.line_count() { - let line_text = &buf.lines[line_idx]; + let line_text = visible_prefix( + &buf.lines[line_idx], + text_width, + visible_height - accumulated_rows, + ); let offsets = wrap_offsets(line_text, text_width); let row_count = offsets.len(); @@ -390,6 +409,8 @@ pub fn render( while visual_lines.len() < visible_height && logical_line < buf.line_count() { let line_text = &buf.lines[logical_line]; + let remaining = visible_height - visual_lines.len(); + let line_text = visible_prefix(line_text, text_width, remaining); let is_current = logical_line == state.cursor_line; let gutter_bg = Style::default().bg(Color::Rgb(40, 40, 50)); @@ -417,7 +438,7 @@ pub fn render( let wrapped_rows = wrap_spans(content_spans, text_width); - if is_current { + if is_current && state.cursor_col <= line_text.len() { let cursor_display_col = display_col(line_text, state.cursor_col); let offsets = wrap_offsets(line_text, text_width); let (c_wrap_row, c_col_in_row) = @@ -740,4 +761,26 @@ mod tests { fn token_style_key_differs_from_string() { assert_ne!(token_style("key"), token_style("string")); } + #[test] + fn viewport_prefix_preserves_visible_word_wrap_rows() { + for line in [ + "aaa bbb ccc ddd eee fff", + "αβγ δ εζη θ", + "a\tb c\td ef", + "abcdefghijklmnop", + ] { + for width in 1..12 { + for rows in 1..5 { + let full = super::wrap_spans(vec![Span::raw(line)], width); + let prefix = super::visible_prefix(line, width, rows); + let clipped = super::wrap_spans(vec![Span::raw(prefix)], width); + assert_eq!( + clipped.iter().take(rows).collect::>(), + full.iter().take(rows).collect::>() + ); + } + } + } + assert!(super::visible_prefix(&"a".repeat(1_000_000), 80, 24).len() <= 1921); + } } diff --git a/src/frontend/tui/fs_watcher.rs b/src/frontend/tui/fs_watcher.rs index d8766df..17cee4f 100644 --- a/src/frontend/tui/fs_watcher.rs +++ b/src/frontend/tui/fs_watcher.rs @@ -1,60 +1,330 @@ +use std::collections::HashSet; use std::path::{Path, PathBuf}; -use std::sync::mpsc; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; +use std::sync::{Arc, Condvar, Mutex, mpsc}; +use crate::commands::diagnostics::Timing; +use crate::data::file_tree::FileTree; use anyhow::Result; -use notify::{RecommendedWatcher, RecursiveMode, Watcher}; +use notify::{EventKind, RecommendedWatcher, RecursiveMode, Watcher}; + +const EVENT_CAPACITY: usize = 2048; +enum Command { + File(Option), + Tree(Option, u64), + Scan(PathBuf, u64), +} +#[derive(Default)] +struct Desired { + file: Option>, + tree: Option<(Option, u64)>, + scan: Option<(PathBuf, u64)>, + shutdown: bool, +} +struct Commands { + state: Mutex, + changed: Condvar, +} +impl Commands { + fn submit(&self, command: Command) { + let mut state = self.state.lock().unwrap(); + match command { + Command::File(path) => state.file = Some(path), + Command::Tree(path, generation) => { + state.tree = Some((path, generation)); + state.scan = None; + } + Command::Scan(root, generation) => state.scan = Some((root, generation)), + } + self.changed.notify_one(); + } + fn take(&self) -> Option { + let mut state = self.state.lock().unwrap(); + loop { + if state.shutdown { + return None; + } + if let Some(file) = state.file.take() { + return Some(Command::File(file)); + } + if let Some((tree, generation)) = state.tree.take() { + return Some(Command::Tree(tree, generation)); + } + if let Some((root, generation)) = state.scan.take() { + return Some(Command::Scan(root, generation)); + } + state = self.changed.wait(state).unwrap(); + } + } +} +fn update_path(event: ¬ify::Result) -> Option<&PathBuf> { + let event = event.as_ref().ok()?; + if event.paths.len() == 1 + && matches!( + event.kind, + EventKind::Modify( + notify::event::ModifyKind::Data(_) + | notify::event::ModifyKind::Metadata(_) + | notify::event::ModifyKind::Any + ) + ) + { + event.paths.first() + } else { + None + } +} +type TreeResult = Option<(u64, Result)>; pub struct FsWatcher { - watcher: RecommendedWatcher, + commands: Arc, + coalesced: Arc>>, pub rx: mpsc::Receiver>, + overflow: Arc, + tree_result: Arc>, watched_file: Option, watched_tree: Option, + generation: u64, + scanning: bool, } impl FsWatcher { pub fn new() -> Result { - let (tx, rx) = mpsc::channel(); - let watcher = RecommendedWatcher::new(tx, notify::Config::default())?; + let (events, rx) = mpsc::sync_channel(EVENT_CAPACITY); + let commands = Arc::new(Commands { + state: Mutex::new(Desired::default()), + changed: Condvar::new(), + }); + let command_rx = Arc::clone(&commands); + let coalesced = Arc::new(Mutex::new(HashSet::new())); + let pending_updates = Arc::clone(&coalesced); + let overflow = Arc::new(AtomicBool::new(false)); + let tree_result = Arc::new(Mutex::new(None)); + let worker_overflow = Arc::clone(&overflow); + let worker_result = Arc::clone(&tree_result); + std::thread::spawn(move || { + let changes = Arc::new(AtomicU64::new(0)); + let observed = Arc::clone(&changes); + let event_overflow = Arc::clone(&worker_overflow); + let notices = events.clone(); + let mut watcher = RecommendedWatcher::new( + move |event: notify::Result| { + if let Ok(event) = &event { + if event + .paths + .iter() + .any(|path| crate::commands::diagnostics::is_timing_path(path)) + { + return; + } + if matches!(event.kind, EventKind::Access(_)) { + return; + } + if matches!( + event.kind, + EventKind::Create(_) + | EventKind::Remove(_) + | EventKind::Modify(notify::event::ModifyKind::Name(_)) + ) { + observed.fetch_add(1, Ordering::Relaxed); + } + } + if let Some(path) = update_path(&event) + && !pending_updates.lock().unwrap().insert(path.clone()) + { + return; + } + if let Err(error) = events.try_send(event) { + let event = match error { + mpsc::TrySendError::Full(event) + | mpsc::TrySendError::Disconnected(event) => event, + }; + if let Some(path) = update_path(&event) { + pending_updates.lock().unwrap().remove(path); + } + event_overflow.store(true, Ordering::Release); + } + }, + notify::Config::default(), + ) + .ok(); + let mut file: Option = None; + let mut tree: Option = None; + while let Some(command) = command_rx.take() { + match command { + Command::File(path) => { + if let Some(watcher) = watcher.as_mut() { + if let Some(old) = file.take() { + let _ = watcher.unwatch(&old); + } + if let Some(path) = path { + let canonical = path.canonicalize().unwrap_or(path); + if watcher + .watch(&canonical, RecursiveMode::NonRecursive) + .is_ok() + { + file = Some(canonical.clone()); + } + // Close the gap between recording the buffer's + // mtime and installing its asynchronous watch. + let kind = if canonical.exists() { + EventKind::Modify(notify::event::ModifyKind::Any) + } else { + EventKind::Remove(notify::event::RemoveKind::Any) + }; + if notices + .try_send(Ok(notify::Event::new(kind).add_path(canonical))) + .is_err() + { + worker_overflow.store(true, Ordering::Release); + } + } + } + } + Command::Tree(path, generation) => { + if let Some(watcher) = watcher.as_mut() + && let Some(old) = tree.take() + { + let _ = watcher.unwatch(&old); + } + if let Some(root) = path { + let root = root.canonicalize().unwrap_or(root); + { + let _timing = Timing::new("tree_watch_setup"); + if let Some(watcher) = watcher.as_mut() { + let _ = watcher.watch(&root, RecursiveMode::Recursive); + } + } + tree = Some(root.clone()); + scan_tree( + &root, + generation, + &changes, + &worker_overflow, + &worker_result, + ); + } + } + Command::Scan(root, generation) => { + scan_tree( + &root, + generation, + &changes, + &worker_overflow, + &worker_result, + ); + } + } + } + // Backend teardown may traverse thousands of watches; it stays here. + }); Ok(Self { - watcher, + commands, + coalesced, rx, + overflow, + tree_result, watched_file: None, watched_tree: None, + generation: 0, + scanning: false, }) } + fn enqueue(&self, command: Command) -> Result<()> { + self.commands.submit(command); + Ok(()) + } + pub fn next_event(&self) -> Option> { + let event = self.rx.try_recv().ok()?; + if let Some(path) = update_path(&event) { + self.coalesced.lock().unwrap().remove(path); + } + Some(event) + } pub fn watch_file(&mut self, path: &Path) -> Result<()> { - self.unwatch_file(); - let canonical = path.canonicalize().unwrap_or_else(|_| path.to_path_buf()); - self.watcher - .watch(&canonical, RecursiveMode::NonRecursive)?; - self.watched_file = Some(canonical); + self.enqueue(Command::File(Some(path.to_path_buf())))?; + self.watched_file = Some(path.to_path_buf()); Ok(()) } - pub fn unwatch_file(&mut self) { - if let Some(path) = self.watched_file.take() { - let _ = self.watcher.unwatch(&path); - } + let _ = self.enqueue(Command::File(None)); + self.watched_file = None; } - pub fn watch_tree(&mut self, root: &Path) -> Result<()> { - self.unwatch_tree(); - let canonical = root.canonicalize().unwrap_or_else(|_| root.to_path_buf()); - self.watcher.watch(&canonical, RecursiveMode::Recursive)?; - self.watched_tree = Some(canonical); + self.generation = self.generation.wrapping_add(1); + self.enqueue(Command::Tree(Some(root.to_path_buf()), self.generation))?; + self.watched_tree = Some(root.to_path_buf()); + self.scanning = true; Ok(()) } - pub fn unwatch_tree(&mut self) { - if let Some(path) = self.watched_tree.take() { - let _ = self.watcher.unwatch(&path); - } + self.generation = self.generation.wrapping_add(1); + let _ = self.enqueue(Command::Tree(None, self.generation)); + self.watched_tree = None; + self.scanning = false; + } + pub fn watched_tree(&self) -> Option<&Path> { + self.watched_tree.as_deref() } - pub fn watched_file(&self) -> Option<&Path> { self.watched_file.as_deref() } + pub fn take_tree(&mut self) -> Option> { + let (generation, result) = self.tree_result.lock().unwrap().take()?; + if generation != self.generation { + return None; + } + self.scanning = false; + Some(result) + } + pub fn take_overflow(&self) -> bool { + let overflow = self.overflow.swap(false, Ordering::AcqRel); + if overflow { + self.coalesced.lock().unwrap().clear(); + for _ in 0..EVENT_CAPACITY { + if self.rx.try_recv().is_err() { + break; + } + } + } + overflow + } + pub fn reconcile_tree(&mut self) -> Result<()> { + if !self.scanning + && let Some(root) = &self.watched_tree + { + self.enqueue(Command::Scan(root.clone(), self.generation))?; + self.scanning = true; + } else if self.scanning { + self.overflow.store(true, Ordering::Release); + } + Ok(()) + } +} + +impl Drop for FsWatcher { + fn drop(&mut self) { + self.commands.state.lock().unwrap().shutdown = true; + self.commands.changed.notify_one(); + } +} + +fn scan_tree( + root: &Path, + generation: u64, + changes: &AtomicU64, + overflow: &AtomicBool, + result: &Mutex, +) { + let mut timing = Timing::new("tree_scan"); + let before = changes.load(Ordering::Acquire); + let snapshot = FileTree::from_dir(root); + timing.set_count(snapshot.as_ref().map_or(0, |tree| tree.entries.len())); + if changes.load(Ordering::Acquire) != before { + overflow.store(true, Ordering::Release); + } + *result.lock().unwrap() = Some((generation, snapshot)); } #[cfg(test)] @@ -95,4 +365,62 @@ mod tests { "watched_file should be None after unwatch_file" ); } + fn await_snapshot(watcher: &mut FsWatcher) -> crate::data::file_tree::FileTree { + let deadline = std::time::Instant::now() + Duration::from_secs(5); + loop { + if let Some(result) = watcher.take_tree() { + return result.unwrap(); + } + assert!( + std::time::Instant::now() < deadline, + "background scan did not finish" + ); + std::thread::sleep(Duration::from_millis(5)); + } + } + + #[test] + fn watch_scan_filters_access_events_and_overflow_reconciles_final_tree() { + let root = tempfile::tempdir().unwrap(); + let mut watcher = FsWatcher::new().unwrap(); + watcher.watch_tree(root.path()).unwrap(); + assert!(await_snapshot(&mut watcher).entries.is_empty()); + for i in 0..3000 { + std::fs::write(root.path().join(format!("{i}.txt")), "x").unwrap(); + } + let deadline = std::time::Instant::now() + Duration::from_secs(5); + while !watcher.overflow.load(Ordering::Acquire) && std::time::Instant::now() < deadline { + std::thread::sleep(Duration::from_millis(5)); + } + assert!( + watcher.take_overflow(), + "bounded event queue must report overflow" + ); + watcher.reconcile_tree().unwrap(); + let snapshot = await_snapshot(&mut watcher); + assert_eq!(snapshot.files_only().count(), 3000); + while let Some(Ok(event)) = watcher.next_event() { + assert!(!matches!(event.kind, EventKind::Access(_))); + } + } + + #[test] + fn watch_commands_coalesce_to_latest_desired_paths() { + let commands = Commands { + state: Mutex::new(Desired::default()), + changed: Condvar::new(), + }; + for i in 0..10_000 { + commands.submit(Command::File(Some(PathBuf::from(i.to_string())))); + } + assert!( + matches!(commands.take(), Some(Command::File(Some(path))) if path == Path::new("9999")) + ); + commands.submit(Command::Scan(PathBuf::from("old"), 1)); + commands.submit(Command::Tree(Some(PathBuf::from("new")), 2)); + assert!(commands.state.lock().unwrap().scan.is_none()); + assert!( + matches!(commands.take(), Some(Command::Tree(Some(path), 2)) if path == Path::new("new")) + ); + } } diff --git a/src/frontend/tui/mod.rs b/src/frontend/tui/mod.rs index c989517..be60a62 100644 --- a/src/frontend/tui/mod.rs +++ b/src/frontend/tui/mod.rs @@ -8,3 +8,5 @@ pub mod status_bar; pub mod title_bar; pub mod tree_pane; pub mod tui_frontend; + +mod background_chord; diff --git a/src/frontend/tui/title_bar.rs b/src/frontend/tui/title_bar.rs index e2a1157..ab0ed17 100644 --- a/src/frontend/tui/title_bar.rs +++ b/src/frontend/tui/title_bar.rs @@ -28,7 +28,7 @@ pub fn render(frame: &mut Frame, area: Rect, state: &EditorState) { .file_name() .and_then(|n| n.to_str()) .unwrap_or("untitled"); - let loc = compute_loc(&buf.lines); + let loc = state.cached_loc; let lines_count = buf.lines.len(); (name.to_string(), buf.dirty, loc, lines_count) } diff --git a/src/frontend/tui/tree_pane.rs b/src/frontend/tui/tree_pane.rs index 139610d..2f5cdbc 100644 --- a/src/frontend/tui/tree_pane.rs +++ b/src/frontend/tui/tree_pane.rs @@ -39,7 +39,11 @@ pub fn render(frame: &mut Frame, area: Rect, state: &EditorState) { let block = Block::default() .borders(Borders::ALL) .border_type(BorderType::Rounded) - .title(format!(" {title} ")) + .title(if state.tree_loading { + format!(" {title} (loading) ") + } else { + format!(" {title} ") + }) .border_style(border_style); if state.tree_view.is_empty() { @@ -241,18 +245,17 @@ pub fn expand(state: &mut EditorState, idx: usize) { None => return, }; - let children: Vec<_> = tree - .entries + let start = tree.entries.partition_point(|e| e.path <= entry.path); + let children: Vec<_> = tree.entries[start..] .iter() + .take_while(|e| e.path.starts_with(&entry.path)) .filter(|e| { e.depth == entry.depth + 1 && e.path.parent().map(|p| p == entry.path).unwrap_or(false) }) .cloned() .collect(); - for (offset, child) in children.into_iter().enumerate() { - state.tree_view.insert(idx + 1 + offset, child); - } + state.tree_view.splice(idx + 1..idx + 1, children); } pub fn collapse(state: &mut EditorState, idx: usize) { @@ -330,6 +333,7 @@ mod tests { EditorState { buffers: Vec::new(), active_buffer: 0, + tree_loading: false, file_tree: Some(FileTree { root: root.clone(), entries: all_entries, @@ -357,6 +361,8 @@ mod tests { selection: None, list_dialog: None, cached_token_count: 0, + cached_loc: 0, + buffer_generation: 0, disk_changed_path: None, pending_rewatch_path: None, tree_rename_state: None, diff --git a/src/frontend/tui/tui_frontend.rs b/src/frontend/tui/tui_frontend.rs index 7aa3b9b..3810c04 100644 --- a/src/frontend/tui/tui_frontend.rs +++ b/src/frontend/tui/tui_frontend.rs @@ -6,17 +6,153 @@ use crate::data::state::{EditorState, ListDialogState, Mode}; use crate::frontend::traits::ApplyChordAction; -pub struct TuiFrontend; +use super::background_chord::{ChordResult, ChordWorker, Job}; +use std::path::PathBuf; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, mpsc}; + +struct PendingChord { + input: String, + generation: u64, + active_buffer: usize, + path: Option, + cursor: (usize, usize), + mode: Mode, + focus_tree: bool, + cancel: Arc, + result: mpsc::Receiver, +} +pub struct TuiFrontend { + worker: ChordWorker, + pending: Option, +} impl Default for TuiFrontend { fn default() -> Self { - Self + Self::new() } } impl TuiFrontend { pub fn new() -> Self { - Self + Self { + worker: ChordWorker::new(), + pending: None, + } + } + + pub fn has_pending(&self) -> bool { + self.pending.is_some() + } + + pub fn cancel_chord(&mut self, state: &mut EditorState) { + if let Some(pending) = self.pending.take() { + pending.cancel.store(true, Ordering::Release); + state.chord_running = false; + if state.status_msg == "chord running — Esc to cancel" { + state.status_msg = "chord cancelled".into(); + } + } + } + + pub fn submit_chord( + &mut self, + state: &mut EditorState, + input: &str, + query: crate::commands::chord_engine::types::ChordQuery, + provider: Box, + cancel: Arc, + ) { + self.cancel_chord(state); + let mut buffers = std::collections::HashMap::new(); + if let Some(buffer) = state.current_buffer() { + buffers.insert(buffer.path.to_string_lossy().into_owned(), buffer.clone()); + } + let (reply, result) = mpsc::sync_channel(1); + self.worker.submit(Job { + query, + buffers, + provider, + cancel: Arc::clone(&cancel), + reply, + }); + self.pending = Some(PendingChord { + input: input.into(), + generation: state.buffer_generation, + active_buffer: state.active_buffer, + path: state.current_buffer().map(|b| b.path.clone()), + cursor: (state.cursor_line, state.cursor_col), + mode: state.mode, + focus_tree: state.focus_tree, + cancel, + result, + }); + state.chord_running = true; + state.status_msg = "chord running — Esc to cancel".into(); + } + + /// Returns true after applying a completed chord; the caller refreshes caches. + pub fn poll_chord(&mut self, state: &mut EditorState) -> bool { + if state.show_exit_modal + || state.pending_open_path.is_some() + || state.list_dialog.is_some() + || state.tree_rename_state.is_some() + || state.tree_delete_confirm.is_some() + || state.tree_new_file_state.is_some() + { + return false; + } + let Some(pending) = &self.pending else { + return false; + }; + if pending.generation != state.buffer_generation + || pending.active_buffer != state.active_buffer + || pending.path.as_deref() != state.current_buffer().map(|b| b.path.as_path()) + || pending.cursor != (state.cursor_line, state.cursor_col) + || pending.mode != state.mode + || pending.focus_tree != state.focus_tree + { + self.cancel_chord(state); + return false; + } + let result = match pending.result.try_recv() { + Ok(result) => result, + Err(mpsc::TryRecvError::Empty) => return false, + Err(mpsc::TryRecvError::Disconnected) => Err("chord worker stopped".into()), + }; + let pending = self.pending.take().unwrap(); + state.chord_running = false; + state.status_msg.clear(); + match result { + Ok(actions) => { + for action in actions.values() { + match self.apply(state, action) { + Ok(message) if !message.is_empty() && state.status_msg.is_empty() => { + state.status_msg = message + } + Err(error) => { + state.status_msg = format!("error: {error}"); + return false; + } + _ => {} + } + } + state.chord_history.push(pending.input); + true + } + Err(error) => { + state.status_msg = error; + false + } + } + } +} + +impl Drop for TuiFrontend { + fn drop(&mut self) { + if let Some(pending) = &self.pending { + pending.cancel.store(true, Ordering::Release); + } } } @@ -191,4 +327,49 @@ mod tests { assert_eq!(state.cursor_line, 0); assert_eq!(state.cursor_col, 2); } + #[test] + fn editing_cancels_a_pending_chord_and_discards_its_result() { + use crate::commands::lsp_engine::LspProvider; + use crate::data::lsp::types::{DocumentSymbol, SelectionRange}; + use std::path::Path; + use std::time::Duration; + struct Blocking { + entered: mpsc::SyncSender<()>, + release: mpsc::Receiver<()>, + } + impl LspProvider for Blocking { + fn document_symbols(&mut self, _: &Path) -> Result> { + self.entered.send(()).unwrap(); + self.release.recv_timeout(Duration::from_secs(3)).unwrap(); + anyhow::bail!("delayed response") + } + fn selection_range(&mut self, _: &Path, _: usize, _: usize) -> Result { + anyhow::bail!("unused") + } + } + let (_f, mut state) = make_state("fn main() {}\n"); + let mut query = crate::commands::chord_engine::ChordEngine::parse("cifn").unwrap(); + query.args.cursor_pos = Some((0, 3)); + let (entered, started) = mpsc::sync_channel(1); + let (release, released) = mpsc::sync_channel(1); + let mut frontend = TuiFrontend::new(); + frontend.submit_chord( + &mut state, + "cifn", + query, + Box::new(Blocking { + entered, + release: released, + }), + Arc::new(AtomicBool::new(false)), + ); + started.recv_timeout(Duration::from_secs(3)).unwrap(); + state.buffer_generation += 1; + state.buffers[0].lines[0] = "edited while waiting".into(); + assert!(!frontend.poll_chord(&mut state)); + assert!(!frontend.has_pending()); + release.send(()).unwrap(); + assert!(!frontend.poll_chord(&mut state)); + assert_eq!(state.buffers[0].lines[0], "edited while waiting"); + } } diff --git a/tests/lsp_integration.rs b/tests/lsp_integration.rs index 568da25..fa93911 100644 --- a/tests/lsp_integration.rs +++ b/tests/lsp_integration.rs @@ -327,3 +327,98 @@ fn document_symbols_works_without_explicit_did_open() { .expect("document_symbols should succeed (auto-opens file)"); assert!(!symbols.is_empty()); } + +// WI-16: response deadlines apply after initialization, and workers do not +// retain the engine lock while awaiting transport I/O. +fn failure_config(flags: &[&str]) -> LspEngineConfig { + mock_config() + .with_server_override( + MOCK_SERVER, + flags.iter().map(|flag| (*flag).into()).collect(), + "true", + ) + .with_request_timeout(Duration::from_millis(300)) +} + +#[test] +fn silent_and_partial_semantic_responses_obey_request_deadline() { + for flag in ["--ignore-semantic", "--partial-semantic"] { + let mut engine = LspEngine::new(failure_config(&[flag])); + start_and_wait(&mut engine); + let mut client = engine.request_client(); + let error = client + .semantic_tokens(rs_file(), "fn main() {}") + .unwrap_err(); + assert!(error.to_string().contains("timed out"), "{flag}: {error}"); + } +} + +#[test] +fn late_response_cannot_satisfy_the_next_request() { + use ane::commands::lsp_engine::LspProvider; + let mut engine = LspEngine::new(failure_config(&["--delay-first-symbol"])); + start_and_wait(&mut engine); + let mut client = engine.request_client(); + assert!( + client + .document_symbols(rs_file()) + .unwrap_err() + .to_string() + .contains("timed out") + ); + let symbols = client.document_symbols(rs_file()).unwrap(); + assert_eq!( + symbols[0].name, "main", + "the late response has a different request ID" + ); +} + +#[test] +fn server_request_with_colliding_id_is_answered_before_initialize() { + let mut engine = LspEngine::new(failure_config(&["--client-request"])); + assert_eq!(start_and_wait(&mut engine), ServerState::Running); +} + +#[test] +fn client_queries_use_the_unsaved_buffer_snapshot() { + use ane::commands::lsp_engine::LspProvider; + let mut engine = LspEngine::new(failure_config(&["--echo-document"])); + start_and_wait(&mut engine); + let mut client = engine + .request_client() + .with_contents(std::collections::HashMap::from([( + rs_file().to_path_buf(), + "unsaved content".into(), + )])); + assert_eq!( + client.document_symbols(rs_file()).unwrap()[0].name, + "unsaved content" + ); +} + +#[test] +fn cancellation_unblocks_a_silent_request_and_releases_interactive_priority() { + use ane::commands::lsp_engine::LspProvider; + use std::sync::{ + Arc, + atomic::{AtomicBool, Ordering}, + }; + let mut engine = LspEngine::new(failure_config(&["--ignore-symbols"])); + start_and_wait(&mut engine); + let cancel = Arc::new(AtomicBool::new(false)); + let mut client = engine + .request_client() + .with_cancellation(Arc::clone(&cancel)); + assert!(engine.request_client().interactive_pending()); + let worker = std::thread::spawn(move || client.document_symbols(rs_file())); + cancel.store(true, Ordering::Release); + assert!( + worker + .join() + .unwrap() + .unwrap_err() + .to_string() + .contains("cancelled") + ); + assert!(!engine.request_client().interactive_pending()); +} From 722b467e5c3df8004dc98bb4c5efbec88b11b6ab Mon Sep 17 00:00:00 2001 From: Connor Hicks Date: Thu, 1 Oct 2026 17:08:47 -0400 Subject: [PATCH 2/4] Fix new Clippy warnings on Rust 1.99 --- src/commands/chord_engine/resolver.rs | 13 +++++-------- src/commands/lsp_engine/engine.rs | 5 ++--- 2 files changed, 7 insertions(+), 11 deletions(-) diff --git a/src/commands/chord_engine/resolver.rs b/src/commands/chord_engine/resolver.rs index a2b873d..e986179 100644 --- a/src/commands/chord_engine/resolver.rs +++ b/src/commands/chord_engine/resolver.rs @@ -2155,15 +2155,12 @@ fn shrink_range(buffer: &Buffer, range: &TextRange) -> TextRange { let start_line = range.start_line; let start_col = range.start_col + 1; - let end_line; - let end_col; - if inner_lines == 0 { - end_line = start_line; - end_col = range.end_col.saturating_sub(1); + let end_line = if inner_lines == 0 { + start_line } else { - end_line = range.end_line; - end_col = range.end_col.saturating_sub(1); - } + range.end_line + }; + let end_col = range.end_col.saturating_sub(1); return TextRange { start_line, diff --git a/src/commands/lsp_engine/engine.rs b/src/commands/lsp_engine/engine.rs index 47fa7df..4f5d4b4 100644 --- a/src/commands/lsp_engine/engine.rs +++ b/src/commands/lsp_engine/engine.rs @@ -1424,7 +1424,8 @@ fn parse_hover(value: &Value) -> Option { s.to_string() } else if let Some(obj) = contents.as_object() { obj.get("value")?.as_str()?.to_string() - } else if let Some(arr) = contents.as_array() { + } else { + let arr = contents.as_array()?; arr.iter() .filter_map(|v| { v.as_str() @@ -1433,8 +1434,6 @@ fn parse_hover(value: &Value) -> Option { }) .collect::>() .join("\n") - } else { - return None; }; Some(HoverInfo { contents: text }) From 780785d976467344dbcd34eaade667e7d401f454 Mon Sep 17 00:00:00 2001 From: Connor Hicks Date: Thu, 1 Oct 2026 17:16:41 -0400 Subject: [PATCH 3/4] Publish canonical watched paths from the filesystem worker --- src/frontend/tui/app.rs | 4 +-- src/frontend/tui/fs_watcher.rs | 54 ++++++++++++++++++++++++++++++++-- 2 files changed, 53 insertions(+), 5 deletions(-) diff --git a/src/frontend/tui/app.rs b/src/frontend/tui/app.rs index 4e8ca26..f744495 100644 --- a/src/frontend/tui/app.rs +++ b/src/frontend/tui/app.rs @@ -2929,8 +2929,8 @@ mod tests { state.buffers[0].last_disk_mtime = Some(SystemTime::UNIX_EPOCH); let mut watcher = super::super::fs_watcher::FsWatcher::new().unwrap(); - watcher.watch_file(f.path()).unwrap(); let canonical = f.path().canonicalize().unwrap(); + watcher.watch_file(&canonical).unwrap(); let event = notify::Event::new(notify::EventKind::Modify(NModifyKind::Data( DataChange::Content, @@ -3129,8 +3129,8 @@ mod tests { state.buffers[0].disk_changed = true; let mut watcher = super::super::fs_watcher::FsWatcher::new().unwrap(); - watcher.watch_file(f.path()).unwrap(); let canonical = f.path().canonicalize().unwrap(); + watcher.watch_file(&canonical).unwrap(); std::fs::remove_file(f.path()).unwrap(); diff --git a/src/frontend/tui/fs_watcher.rs b/src/frontend/tui/fs_watcher.rs index 17cee4f..e67b942 100644 --- a/src/frontend/tui/fs_watcher.rs +++ b/src/frontend/tui/fs_watcher.rs @@ -83,6 +83,7 @@ pub struct FsWatcher { overflow: Arc, tree_result: Arc>, watched_file: Option, + installed_file: Arc>>, watched_tree: Option, generation: u64, scanning: bool, @@ -102,6 +103,8 @@ impl FsWatcher { let tree_result = Arc::new(Mutex::new(None)); let worker_overflow = Arc::clone(&overflow); let worker_result = Arc::clone(&tree_result); + let installed_file = Arc::new(Mutex::new(None)); + let worker_file = Arc::clone(&installed_file); std::thread::spawn(move || { let changes = Arc::new(AtomicU64::new(0)); let observed = Arc::clone(&changes); @@ -158,7 +161,9 @@ impl FsWatcher { let _ = watcher.unwatch(&old); } if let Some(path) = path { - let canonical = path.canonicalize().unwrap_or(path); + let canonical = + path.canonicalize().unwrap_or_else(|_| path.clone()); + *worker_file.lock().unwrap() = Some((path, canonical.clone())); if watcher .watch(&canonical, RecursiveMode::NonRecursive) .is_ok() @@ -225,6 +230,7 @@ impl FsWatcher { overflow, tree_result, watched_file: None, + installed_file, watched_tree: None, generation: 0, scanning: false, @@ -235,7 +241,14 @@ impl FsWatcher { self.commands.submit(command); Ok(()) } - pub fn next_event(&self) -> Option> { + pub fn next_event(&mut self) -> Option> { + // Publish the worker's canonical namespace before consuming its events. + // Resolving symlinks here would put filesystem I/O back on the UI thread. + if let Some((requested, canonical)) = self.installed_file.lock().unwrap().take() + && self.watched_file.as_ref() == Some(&requested) + { + self.watched_file = Some(canonical); + } let event = self.rx.try_recv().ok()?; if let Some(path) = update_path(&event) { self.coalesced.lock().unwrap().remove(path); @@ -343,6 +356,17 @@ mod tests { let mut watcher = FsWatcher::new().unwrap(); watcher.watch_file(f.path()).unwrap(); + let deadline = std::time::Instant::now() + Duration::from_secs(5); + loop { + if watcher.next_event().is_some() { + break; + } + assert!( + std::time::Instant::now() < deadline, + "watch installation timed out" + ); + std::thread::sleep(Duration::from_millis(5)); + } let expected_canonical = f.path().canonicalize().unwrap(); assert_eq!( watcher.watched_file().map(|p| p.to_path_buf()), @@ -350,7 +374,6 @@ mod tests { "watched_file should be set to the canonical path after watch_file" ); - std::thread::sleep(Duration::from_millis(50)); std::fs::write(f.path(), b"modified\n").unwrap(); let result = watcher.rx.recv_timeout(Duration::from_secs(1)); @@ -365,6 +388,31 @@ mod tests { "watched_file should be None after unwatch_file" ); } + #[cfg(unix)] + #[test] + fn canonical_watch_path_is_published_for_a_symlinked_parent() { + let root = tempfile::tempdir().unwrap(); + let real = root.path().join("real"); + std::fs::create_dir(&real).unwrap(); + std::fs::write(real.join("file"), "initial").unwrap(); + let alias = root.path().join("alias"); + std::os::unix::fs::symlink(&real, &alias).unwrap(); + let requested = alias.join("file"); + let canonical = requested.canonicalize().unwrap(); + let mut watcher = FsWatcher::new().unwrap(); + watcher.watch_file(&requested).unwrap(); + let deadline = std::time::Instant::now() + Duration::from_secs(5); + loop { + if let Some(Ok(event)) = watcher.next_event() { + assert_eq!(watcher.watched_file(), Some(canonical.as_path())); + assert!(event.paths.contains(&canonical)); + break; + } + assert!(std::time::Instant::now() < deadline); + std::thread::sleep(Duration::from_millis(5)); + } + } + fn await_snapshot(watcher: &mut FsWatcher) -> crate::data::file_tree::FileTree { let deadline = std::time::Instant::now() + Duration::from_secs(5); loop { From bec2efa7e76ea9172647f2d87aeb393e54572260 Mon Sep 17 00:00:00 2001 From: Connor Hicks Date: Thu, 1 Oct 2026 17:24:13 -0400 Subject: [PATCH 4/4] Synchronize syntax worker tests instead of relying on sleeps --- src/commands/lsp_engine/engine.rs | 23 ++++ src/commands/lsp_engine/mod.rs | 3 + src/commands/syntax_engine/mod.rs | 172 ++++++++---------------------- 3 files changed, 69 insertions(+), 129 deletions(-) diff --git a/src/commands/lsp_engine/engine.rs b/src/commands/lsp_engine/engine.rs index 4f5d4b4..816d66d 100644 --- a/src/commands/lsp_engine/engine.rs +++ b/src/commands/lsp_engine/engine.rs @@ -138,6 +138,12 @@ fn try_transition( Some(old) } +#[cfg(test)] +pub(crate) struct SemanticTestGate { + pub entered: mpsc::Sender<()>, + pub release: Mutex>, +} + pub struct LspEngine { servers: HashMap, config: LspEngineConfig, @@ -156,6 +162,8 @@ pub struct LspEngine { pub did_open_log: Vec<(PathBuf, String)>, #[cfg(any(test, feature = "test-support"))] pub test_semantic_tokens_delay: Option, + #[cfg(test)] + pub(crate) test_semantic_gate: Option>, } impl LspEngine { @@ -179,6 +187,8 @@ impl LspEngine { did_open_log: Vec::new(), #[cfg(any(test, feature = "test-support"))] test_semantic_tokens_delay: None, + #[cfg(test)] + test_semantic_gate: None, } } @@ -457,6 +467,8 @@ impl LspEngine { test_semantic_tokens: self.test_semantic_tokens.clone(), #[cfg(any(test, feature = "test-support"))] test_semantic_tokens_delay: self.test_semantic_tokens_delay, + #[cfg(test)] + test_semantic_gate: self.test_semantic_gate.clone(), } } @@ -887,6 +899,8 @@ pub struct LspClient { test_semantic_tokens: HashMap>, #[cfg(any(test, feature = "test-support"))] test_semantic_tokens_delay: Option, + #[cfg(test)] + test_semantic_gate: Option>, } struct InteractiveLease(Arc); @@ -968,6 +982,15 @@ impl LspClient { pub fn semantic_tokens(&mut self, path: &Path, content: &str) -> Result> { #[cfg(any(test, feature = "test-support"))] if let Some(tokens) = self.test_semantic_tokens.get(path) { + #[cfg(test)] + if let Some(gate) = &self.test_semantic_gate { + gate.entered.send(()).unwrap(); + gate.release + .lock() + .unwrap() + .recv_timeout(Duration::from_secs(5)) + .unwrap(); + } if let Some(delay) = self.test_semantic_tokens_delay { thread::sleep(delay); } diff --git a/src/commands/lsp_engine/mod.rs b/src/commands/lsp_engine/mod.rs index c8e0a3f..83f75a0 100644 --- a/src/commands/lsp_engine/mod.rs +++ b/src/commands/lsp_engine/mod.rs @@ -6,3 +6,6 @@ mod transport; pub use engine::{LspClient, LspEngine, LspEngineConfig, LspProvider, NoLsp, ShutdownControl}; pub use installer::InstallProgress; + +#[cfg(test)] +pub(crate) use engine::SemanticTestGate; diff --git a/src/commands/syntax_engine/mod.rs b/src/commands/syntax_engine/mod.rs index 3d26f8f..61c0e92 100644 --- a/src/commands/syntax_engine/mod.rs +++ b/src/commands/syntax_engine/mod.rs @@ -521,149 +521,63 @@ mod tests { assert_eq!(total, 11, "worker should fire exactly once after debounce"); } - #[test] - fn staleness_check_discards_outdated_lsp_tokens() { - use crate::data::lsp::types::SemanticToken as ST; - - // Strategy: inject slow LSP tokens (400ms delay). After the debounce - // window (300ms), the worker calls semantic_tokens and SLEEPS for 400ms. - // During that sleep the test thread calls compute(B), updating - // content_hashes to hash(B). When the worker wakes and checks staleness, - // hash(A) ≠ hash(B) → SKIP. Only the two sync deliveries occur. - let path = std::path::PathBuf::from("staleness_test.rs"); - + fn assert_latest_semantic_delivery(queued: &[&str]) { + use crate::commands::lsp_engine::SemanticTestGate; + use std::sync::mpsc; + + let path = PathBuf::from("latest_wins.rs"); + let (entered_tx, entered_rx) = mpsc::channel(); + let (release_tx, release_rx) = mpsc::channel(); + let (delivery_tx, delivery_rx) = mpsc::channel(); + struct Counter(mpsc::Sender<()>); + impl SyntaxFrontend for Counter { + fn set_semantic_tokens(&self, _: &Path, _: Vec) { + self.0.send(()).unwrap(); + } + } let mut lsp = LspEngine::new(LspEngineConfig::default()); lsp.inject_test_semantic_tokens( path.clone(), - vec![ST { + vec![SemanticToken { line: 0, start_col: 0, length: 2, - token_type: "keyword".to_string(), + token_type: "keyword".into(), }], ); - lsp.test_semantic_tokens_delay = Some(Duration::from_millis(400)); - - let call_count = Arc::new(Mutex::new(0usize)); - let cc = Arc::clone(&call_count); - let counter_frontend = Arc::new({ - struct Counter(Arc>); - impl SyntaxFrontend for Counter { - fn set_semantic_tokens(&self, _: &Path, _: Vec) { - *self.0.lock().unwrap() += 1; - } - } - Counter(cc) - }); - - let lsp_arc = Arc::new(Mutex::new(lsp)); - let mut engine = SyntaxEngine::new( - Arc::clone(&lsp_arc), - counter_frontend as Arc, - ); - - // compute(A): queued, worker starts 300ms debounce - engine.compute(path.as_path(), "fn foo() {}"); - assert_eq!( - *call_count.lock().unwrap(), - 1, - "sync delivery for content A" - ); - - // Wait for debounce to expire so the worker starts the slow LSP call - std::thread::sleep(Duration::from_millis(350)); - - // compute(B): worker is now sleeping inside semantic_tokens (400ms delay). - // This updates content_hashes to hash(B), and req(B) is queued. - engine.compute(path.as_path(), "fn bar() {}"); - assert_eq!( - *call_count.lock().unwrap(), - 2, - "sync delivery for content B" + lsp.test_semantic_gate = Some(Arc::new(SemanticTestGate { + entered: entered_tx, + release: Mutex::new(release_rx), + })); + let mut engine = + SyntaxEngine::new(Arc::new(Mutex::new(lsp)), Arc::new(Counter(delivery_tx))); + let timeout = Duration::from_secs(5); + engine.compute(&path, "fn a() {}"); + delivery_rx.recv_timeout(timeout).unwrap(); // synchronous A + entered_rx.recv_timeout(timeout).unwrap(); // semantic A is blocked + for content in queued { + engine.compute(&path, content); + delivery_rx.recv_timeout(timeout).unwrap(); // synchronous edit + } + release_tx.send(()).unwrap(); + entered_rx.recv_timeout(timeout).unwrap(); // newest snapshot only + assert!( + delivery_rx.try_recv().is_err(), + "stale A must not be delivered" ); + release_tx.send(()).unwrap(); + delivery_rx.recv_timeout(timeout).unwrap(); // newest semantic result + assert!(delivery_rx.try_recv().is_err()); + } - // Timeline after compute(B) at t=350ms: - // t≈700ms – slow LSP for req(A) returns; staleness check → SKIP - // t≈700ms – worker picks up req(B), 300ms debounce fires at t≈1000ms - // t≈1400ms – slow LSP for req(B) returns; staleness OK → delivery #3 - // Sleep 2100ms after compute(B) (total ~2450ms) to ensure delivery #3 has fired. - std::thread::sleep(Duration::from_millis(2100)); - - // delivery #3 comes from req(B) processed correctly; req(A) was discarded - let final_count = *call_count.lock().unwrap(); - assert_eq!( - final_count, 3, - "req(A) stale → skip; req(B) not stale → deliver; total = 3" - ); + #[test] + fn staleness_check_discards_outdated_lsp_tokens() { + assert_latest_semantic_delivery(&["fn b() {}"]); } #[test] fn latest_wins_during_slow_lsp_call() { - // True latest-wins: while the worker is inside a slow semantic_tokens - // call for content A, two more computes happen (B then C). Both B and C - // are submitted to the slot; with latest-wins semantics, C overwrites B - // before the worker takes them out. After A's call returns (stale → - // skip), the worker picks up C directly — B is never processed. - use crate::data::lsp::types::SemanticToken as ST; - let path = std::path::PathBuf::from("latest_wins.rs"); - - let mut lsp = LspEngine::new(LspEngineConfig::default()); - lsp.inject_test_semantic_tokens( - path.clone(), - vec![ST { - line: 0, - start_col: 0, - length: 2, - token_type: "keyword".to_string(), - }], - ); - lsp.test_semantic_tokens_delay = Some(Duration::from_millis(500)); - - let call_count = Arc::new(Mutex::new(0usize)); - let cc = Arc::clone(&call_count); - let counter_frontend = Arc::new({ - struct Counter(Arc>); - impl SyntaxFrontend for Counter { - fn set_semantic_tokens(&self, _: &Path, _: Vec) { - *self.0.lock().unwrap() += 1; - } - } - Counter(cc) - }); - - let lsp_arc = Arc::new(Mutex::new(lsp)); - let mut engine = SyntaxEngine::new( - Arc::clone(&lsp_arc), - counter_frontend as Arc, - ); - - // compute(A): sync delivery #1, worker debounces 300ms, then starts - // a 500ms LSP call at t≈300ms (finishes at t≈800ms). - engine.compute(path.as_path(), "fn a() {}"); - assert_eq!(*call_count.lock().unwrap(), 1); - - // Wait past the debounce so the worker is mid-LSP-call. - std::thread::sleep(Duration::from_millis(400)); - - // compute(B) then compute(C) — both arrive while worker is in LSP call. - // With latest-wins, C overwrites B in the slot before the worker - // takes them. The worker should pick up C (not B) next. - engine.compute(path.as_path(), "fn b() {}"); - engine.compute(path.as_path(), "fn c() {}"); - assert_eq!(*call_count.lock().unwrap(), 3, "three sync deliveries"); - - // Timeline: - // t≈800ms – LSP(A) returns; content_hash is now hash(C) → SKIP - // t≈800ms – worker takes C, debounces 300ms (t≈1100ms) - // t≈1600ms – LSP(C) returns; staleness OK → delivery #4 - // Sleep 1500ms after compute(C) (total ~1900ms after start). - std::thread::sleep(Duration::from_millis(1500)); - - let total = *call_count.lock().unwrap(); - assert_eq!( - total, 4, - "exactly one worker delivery (for C); B was overwritten before worker took it" - ); + assert_latest_semantic_delivery(&["fn b() {}", "fn c() {}"]); } #[test]