From b7d6e72ff608ef2f79f067f91932110e9d30a78c Mon Sep 17 00:00:00 2001 From: Vasily Kulikov Date: Sat, 5 Sep 2026 16:54:30 +0300 Subject: [PATCH 01/18] todo --- TODO.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/TODO.md b/TODO.md index 4d03809..92633cd 100644 --- a/TODO.md +++ b/TODO.md @@ -9,6 +9,8 @@ - man +- fill filetypes + # Features From bfd3f16c673a851b5bfba3e684cb02d2afd26c62 Mon Sep 17 00:00:00 2001 From: Vasily Kulikov Date: Sat, 5 Sep 2026 17:05:58 +0300 Subject: [PATCH 02/18] Add E2E test plan and harness foundation --- .../e2e-supported-language-scope.md | 11 + .../tmp-git-affects-workspace-root-tests.md | 10 + E2E_TESTS.md | 428 ++++++++++++++++++ tests/e2e.rs | 4 + tests/e2e/catalog.rs | 46 ++ tests/e2e/harness.rs | 49 ++ 6 files changed, 548 insertions(+) create mode 100644 .memory/unprocessed/e2e-supported-language-scope.md create mode 100644 .memory/unprocessed/tmp-git-affects-workspace-root-tests.md create mode 100644 E2E_TESTS.md create mode 100644 tests/e2e.rs create mode 100644 tests/e2e/catalog.rs create mode 100644 tests/e2e/harness.rs diff --git a/.memory/unprocessed/e2e-supported-language-scope.md b/.memory/unprocessed/e2e-supported-language-scope.md new file mode 100644 index 0000000..e39ca55 --- /dev/null +++ b/.memory/unprocessed/e2e-supported-language-scope.md @@ -0,0 +1,11 @@ +# Configured and detectable language scopes differ + +At pinned `lsp-cli-data` revision `013a75f6412917b710aa4683b9c5761c0c679975`, there are 362 +filetype YAML files, but only 16 have a non-empty `extensions` or `patterns` list. The other 346 +cannot be detected from a project even though the `languages` command currently collects every +loaded filetype ID without filtering for usable detection rules. + +This makes “all supported languages” ambiguous for E2E coverage. The proposed executable scope is +the 16 detectable IDs and their 57 relevant LSP configurations (141 compatible pairs), while the +full catalog receives configuration-validation coverage. Product ownership must confirm whether +the `languages` command is intended to expose catalog entries or only detectable languages. diff --git a/.memory/unprocessed/tmp-git-affects-workspace-root-tests.md b/.memory/unprocessed/tmp-git-affects-workspace-root-tests.md new file mode 100644 index 0000000..3799826 --- /dev/null +++ b/.memory/unprocessed/tmp-git-affects-workspace-root-tests.md @@ -0,0 +1,10 @@ +# A parent `/tmp/.git` changes workspace-root tests + +Several `suggest` tests create temporary workspaces directly below `/tmp` and configure `.git` as +an LSP root marker. When the host provides `/tmp/.git`, root-marker discovery resolves `/tmp` +instead of the test's temporary workspace, causing otherwise unrelated assertions to fail. + +Running the tests with `TMPDIR=/dev/shm` avoids that host-specific parent marker. A durable defense +would make these tests control every ancestor up to the search boundary, or let root discovery stop +at an explicit test boundary, rather than assuming the system temporary directory has no matching +root marker. diff --git a/E2E_TESTS.md b/E2E_TESTS.md new file mode 100644 index 0000000..809470b --- /dev/null +++ b/E2E_TESTS.md @@ -0,0 +1,428 @@ +# End-to-end test plan + +## Goal + +Exercise the released `lsp-cli` binary against every supported language, every compatible +supported LSP server, and every top-level subcommand. Keep the suite useful both as a fast pull +request check and as an exhaustive compatibility check. + +The tests must validate user-visible behavior: exit status, stdout, stderr, filesystem effects, +server lifecycle, and semantically relevant LSP results. They must not depend on private Rust APIs. + +## Working definition of supported + +At pinned `lsp-cli-data` revision `013a75f6412917b710aa4683b9c5761c0c679975`, the data tree has: + +- 362 filetype configurations; +- 362 LSP configurations; +- 16 detectable filetype IDs (a non-empty `extensions` or `patterns` list); +- 57 distinct LSP configurations associated with those detectable filetypes; +- 141 compatible detectable-filetype/LSP pairs. + +The working E2E scope is the 16 detectable IDs: + +```text +c cpp cs cuda go gomod gowork java javascript kotlin lua objc objcpp python rust typescript +``` + +The remaining 346 filetype configurations have no detection rules. They are configuration catalog +entries, but cannot currently drive a project-based E2E test. Separately test that the whole data +tree parses and that catalog commands describe it consistently. + +This is a product-policy boundary rather than an implementation fact. Before declaring the suite +complete, the product owner must confirm one of these definitions: + +1. **Detectable support (recommended):** exhaustive real-server tests cover the 16 detectable IDs, + 57 relevant servers, and 141 compatible pairs. +2. **Configured support:** all 362 filetype and server configs are considered supported. This first + requires adding detection rules, test projects, and provisioning for the presently inactive + catalog entries. + +Pros of detectable support: executable now, objective, and automatically derived from the shipped +data. Cons: `lsp-cli languages` currently appears to expose a wider catalog than this definition. + +Pros of configured support: the word “supported” matches every shipped YAML entry. Cons: most of +the required projects and provisioning do not exist, and millions of incompatible Cartesian cases +would still need to be excluded. + +## Non-goals + +- Do not run a Cartesian product of every language, server, command, and option. Only configured + language/server relationships are meaningful. +- Do not require a server to implement an optional LSP capability. +- Do not put language-specific parsing or source-code knowledge into production `lsp-cli` code. +- Do not make tracked playground files writable test state. +- Do not add a Rust dependency without explicit permission. The existing `tempfile`, `serde`, + `serde_json`, and `serde_yaml` dependencies are sufficient for the planned harness. + +## Test projects + +### Existing projects + +Reuse the projects under `playground/` for: + +| Filetype ID | Directory | +|---|---| +| `c` | `playground/c` | +| `cpp` | `playground/cpp` | +| `cs` | `playground/csharp` | +| `go` | `playground/go` | +| `java` | `playground/java` | +| `javascript` | `playground/js` | +| `lua` | `playground/lua` | +| `python` | `playground/python` | +| `rust` | `playground/rust` | +| `typescript` | `playground/typescript` | + +### Projects to add + +Add source projects for: + +- `playground/cuda` +- `playground/kotlin` +- `playground/objc` +- `playground/objcpp` + +Add minimal detection fixtures for `gomod` and `gowork`. These IDs describe Go workspace metadata, +not source languages, so they can cover detection, file listing, server selection, initialization, +and lifecycle, but cannot independently provide meaningful symbol or call-hierarchy assertions. + +Every source-language project should be small, valid, and multi-file. Where the language permits, +it should contain: + +- one stable workspace symbol; +- functions and methods; +- a declaration separated from its definition; +- references from more than one file; +- a caller and callee chain; +- types and fields; +- a file whose formatting can be made deterministically incorrect; +- a deterministic source mutation that produces one diagnostic. + +Prefer equivalent domain concepts and symbol names across projects when natural. Do not force a +language into constructs it does not support merely to make fixtures textually identical. + +Tests copy a project into a temporary directory before formatting it or introducing diagnostics. +This keeps the repository clean and allows safe parallel execution. + +Pros of committed playgrounds: humans can reproduce failures with the same projects. Cons: each +language fixture must evolve with its toolchain and server ecosystem. + +An alternative is to generate every project during test setup. That reduces committed files, but +makes failures harder to inspect and manual reproduction less convenient; do not use it for the +baseline projects. + +## Coverage model + +`lsp-cli` currently has 24 canonical top-level subcommands. Factor them by responsibility instead +of multiplying all commands by all language/server pairs. + +| Scope | Subcommands | Required coverage | +|---|---|---| +| Global CLI | `commands`, `languages`, `servers`, `completion`, `agent-skill`, `update` | Focused binary-level cases, independent of real servers | +| Detection and filesystem | `detect`, `list-files` | Every detectable filetype ID | +| LSP requests | `server-capabilities`, `diagnostics`, `format`, `grep`, `list-symbols`, `list-functions`, `references`, `callers`, `callees`, `definition`, `declaration`, `build-index` | Every compatible language/server pair, capability-aware | +| Process lifecycle | `run`, `daemon`, `stop`, `stop-all` | Every distinct relevant server where applicable, with grouped lifecycle scenarios | + +### Capability-aware expectations + +For each compatible pair, first record or inspect the server's initialized capabilities. A command +passes if it either: + +- succeeds and returns the expected semantic result; or +- returns the documented, user-facing unsupported-capability error when the server does not + advertise the required capability. + +Formatting, declarations, diagnostics, workspace symbols, and call hierarchy are optional or vary +substantially between servers. Treating every unsupported operation as a suite failure would test +an assumption the LSP specification does not make. + +Capability advertisement is not enough by itself: when a server advertises a capability, exercise +the corresponding command and assert its behavior. + +### Option coverage + +Distribute option variants across the matrix using explicit cases; do not create another full +cross-product. Cover at least: + +- automatic selection, `--lang`, and `--lsp`; +- text and `--json` output; +- direct execution, `--detach`, and `--no-detach`; +- `--limit`, `--files-with-matches`, and `--full`; +- `--wait-for-index`; +- `format`, `format --check`, and `format --stdout`; +- successful operations, unsupported capabilities, missing executables, server crashes, malformed + replies, and timeouts; +- `--download` once per supported installation mechanism, rather than redundantly for every query. + +JSON assertions should deserialize and compare stable semantic fields. Text assertions should +avoid full snapshots when server versions can legitimately change ordering, signatures, or detail. + +## Harness design + +Use a normal Cargo integration-test crate that invokes the built binary through +`CARGO_BIN_EXE_lsp-cli`. + +Proposed layout: + +```text +tests/ + e2e.rs + e2e/ + harness.rs + manifest.rs + catalog.rs + detection.rs + queries.rs + lifecycle.rs + update.rs + cases.yaml +``` + +Keep every Rust file under 600 lines. Move repeated process setup and assertions into helpers as +soon as a second test needs them. + +`harness.rs` should provide methods for actions on an E2E context, for example: + +- create an isolated home, configuration root, runtime root, and workspace copy; +- construct an `lsp-cli` process with deterministic environment variables; +- run a command with a deadline and capture stdout/stderr/status; +- parse JSON output; +- introduce a formatting or diagnostic mutation; +- find and terminate remaining child processes; +- stop daemons and report their runtime state after failure. + +Each test process should set at least: + +- `HOME` to an isolated temporary home; +- `XDG_CONFIG_HOME` to an isolated configuration directory; +- `XDG_RUNTIME_DIR` to an isolated daemon directory; +- `LSP_DATA` to the pinned repository submodule; +- `PATH` to the explicitly provisioned toolchain/server environment. + +Do not rely on a developer's user configuration, downloaded server cache, daemon sockets, current +shell, or ambient server versions. + +The manifest should include stable case data, provisioning metadata, expected capabilities, and +documented exclusions. A validation test should fail when: + +- a detectable filetype lacks a project; +- a compatible pair lacks a manifest entry; +- a manifest entry names a missing data config; +- an exclusion lacks a reason; +- two cases select the same user-visible server ambiguously; +- a new top-level subcommand has no assigned coverage class. + +## Special command strategies + +### `run` + +`run` replaces the current process with the language server on Unix. Start it with piped stdio, +send a minimal LSP `initialize` / `initialized` / `shutdown` / `exit` exchange, and verify that the +selected real server took over. Test selection and exec errors separately. + +### `daemon`, `stop`, and `stop-all` + +For every applicable server, run a grouped lifecycle scenario: + +1. start a daemon in an isolated runtime directory; +2. issue at least two queries with `--detach` and verify reuse; +3. stop the exact daemon; +4. verify that a later query starts or connects according to documented behavior; +5. start multiple isolated daemons and verify `stop-all` removes all of them. + +On failure, print socket paths, process state, selected command line, workspace root, and bounded +server stderr. Cleanup must run even after an assertion failure. + +### `update` + +The update repository and release endpoints are currently hardcoded. A deterministic success-path +binary E2E test therefore needs a small production seam that redirects HTTP to a local fixture +server. This change improves testability but is an architectural decision and must be approved +before implementation. + +Without that seam, only argument/error behavior can be deterministic; a live GitHub success test +would be slow, mutable, rate-limited, and capable of replacing test data from the network. + +Recommended decision: introduce a narrowly scoped test-only or configurable repository endpoint, +while keeping the production default unchanged. + +### Diagnostics and formatting + +Start from a valid temporary workspace. Apply one language-specific mutation recorded in the +manifest, run the command, assert the expected file/range/message class, and discard the temporary +copy. Do not commit permanently broken source files that could interfere with unrelated queries. + +### Indexing + +`build-index` should be tested against every server that has a usable background-work completion +signal. For other servers, assert the intended bounded timeout or no-op policy. Do not infer +completion from a fixed sleep. + +## Real-server provisioning + +Pin every server and required toolchain version. The manifest should distinguish: + +- directly installed executables; +- npm, PyPI, Cargo, Go, or other package-manager installations; +- archive-based installations; +- servers requiring a language SDK or compiler; +- servers not installable through the current downloader. + +Do not silently skip a required pair because its executable is absent. A CI lane either provisions +the server or reports the pair as an explicit, reviewed exclusion. + +Adding and pinning these external test tools requires product-owner approval under the repository's +dependency policy. They need not become Rust package dependencies, but they are still operational +dependencies with maintenance, security, licensing, storage, and network consequences. + +Pros of pinned versions: reproducible failures and controlled upgrades. Cons: compatibility with +new upstream releases is detected only when pins are deliberately refreshed. + +An unpinned “latest” lane can complement the pinned suite on a schedule. Its advantage is early +warning of upstream breakage; its disadvantage is nondeterminism, so it must not be the only merge +gate. + +## CI plan + +### Pull requests + +Run: + +- all existing unit tests and checks through `make test`; +- global and detection E2E tests; +- one preferred, pinned server per source language; +- every relevant subcommand across that smoke matrix; +- manifest/data consistency checks. + +### Nightly exhaustive matrix + +Run all 141 compatible pairs, sharded by language and server installation family. Use fail-fast +disabled so one broken server does not hide the rest of the compatibility report. + +Cache downloaded toolchains and server packages using keys that include the pinned version. Do not +share homes, daemon runtime directories, or mutable workspaces between parallel jobs. + +### Scheduled latest-version compatibility + +Optionally run supported servers at current upstream versions. Report failures separately from the +pinned merge gate until a human confirms whether the server or `lsp-cli` needs adaptation. + +### Manual workflow + +Allow selection of one language, one server, or one installation family. This is needed to debug a +nightly failure without rerunning the complete matrix. + +Pros of split CI: fast merge feedback plus exhaustive coverage. Cons: a regression affecting a +non-preferred server may be found the following night rather than on the originating pull request. + +Running all 141 pairs on every pull request gives earlier detection, but has much higher latency, +cost, rate-limit exposure, and upstream-flake risk. It is not the recommended default. + +## Failure policy + +Classify failures as: + +1. `lsp-cli` regression; +2. playground/manifest drift; +3. provisioning or network failure; +4. upstream server behavior change; +5. known server limitation; +6. unsupported LSP capability with the expected user-facing response. + +Only category 6 is an immediate passing outcome. Known limitations must be explicit manifest +entries and, when they concern protocol or server behavior, documented in `GOTCHAS.md`. Do not add +unbounded retries. A retry may cover an identified transient installation/network step, but must +not conceal query or protocol failures. + +If a hard-to-debug defect is fixed, add a focused regression test in addition to the broad matrix. +Also consider whether a type invariant, runtime check, clearer trace, or state-dump helper can make +that class of defect easier to diagnose. + +## Execution phases + +### Phase 0: approve boundaries + +- [x] Confirm detectable support versus configured support: use the 16 detectable IDs. +- [x] Confirm that unsupported optional capabilities count as a passing, asserted outcome. +- [x] Approve pinned external server/toolchain provisioning. +- [x] Approve a narrow HTTP endpoint seam for deterministic `update` E2E coverage. +- [x] Confirm PR smoke plus nightly exhaustive CI cadence. + +### Phase 1: foundation + +- [x] Add `tests/e2e.rs` and compact harness modules. +- [ ] Add the initial manifest schema and validation. +- [ ] Isolate all environment and runtime state. +- [ ] Implement deadlines, cleanup guards, JSON helpers, and useful failure diagnostics. +- [ ] Prove the harness with Rust/rust-analyzer. +- [ ] Cover all 24 subcommand paths with either a real server or a deterministic local fixture. + +### Phase 2: projects + +- [ ] Audit the ten existing playgrounds against the common semantic requirements. +- [ ] Remove duplicated setup patterns within each class of fixture. +- [ ] Add CUDA, Kotlin, Objective-C, and Objective-C++ projects. +- [ ] Add `gomod` and `gowork` detection fixtures. +- [ ] Update `playground/README.md` with manual reproduction commands. +- [ ] Run each relevant command manually against every new project. + +### Phase 3: preferred-server smoke matrix + +- [ ] Select and pin one preferred server for each source language. +- [ ] Add provisioning scripts without new Rust dependencies. +- [ ] Implement capability-aware query assertions. +- [ ] Implement direct/detached lifecycle scenarios. +- [ ] Add the pull-request E2E job. + +### Phase 4: exhaustive compatibility + +- [ ] Populate manifest entries for all 141 compatible pairs. +- [ ] Provision every non-excluded server and required SDK. +- [ ] Record reviewed exceptions and platform constraints. +- [ ] Add sharded nightly and manual workflows. +- [ ] Verify failures retain server version, command line, capabilities, stderr summary, and cleanup + state. + +### Phase 5: hardening + +- [ ] Run `make test`. +- [ ] Run the full pinned E2E matrix from a clean environment. +- [ ] Check every new or edited test file for boilerplate and duplication. +- [ ] Check every source file remains below 600 lines. +- [ ] Add regression tests for every bug uncovered during rollout. +- [ ] Add LSP/server-specific discoveries to `GOTCHAS.md`. +- [ ] Document how to refresh server pins and triage nightly failures. + +## Definition of done + +The work is complete when: + +- every accepted supported language has a committed project or justified metadata-only fixture; +- every compatible supported language/server pair has an executable manifest entry; +- every top-level subcommand has binary-level E2E coverage in its appropriate scope; +- advertised capabilities are exercised and unsupported capabilities have asserted user-facing + behavior; +- direct, detached, stop, and stop-all lifecycle paths are covered; +- formatting and diagnostics cannot dirty tracked files; +- required servers and toolchains are pinned and reproducibly provisioned; +- PR smoke, nightly exhaustive, and manual targeted workflows are documented and passing; +- `make test` passes; +- known protocol/server deviations are recorded in `GOTCHAS.md`; +- no required case is silently skipped. + +## Architectural consequences and limitations + +- The data catalog becomes an enforceable compatibility contract: adding a detectable filetype or + compatible LSP config requires E2E ownership. +- Real-server E2E tests are inherently slower and less hermetic than protocol tests. Unit tests and + fake-server integration tests remain necessary for precise edge cases. +- Capability-aware results mean “all commands tested” does not mean “all commands succeed on every + server.” It means every applicable success path and every inapplicable user-facing response is + verified. +- Third-party server pinning creates a recurring upgrade and security-review obligation. +- Some servers require proprietary, platform-specific, or unusually heavy SDKs. Their treatment + must be an explicit product decision rather than an automatic skip. +- The current difference between the 362 configured filetypes and 16 detectable filetypes may need + a future terminology or behavior change in `languages`; this plan exposes but does not decide + that product question. diff --git a/tests/e2e.rs b/tests/e2e.rs new file mode 100644 index 0000000..5a80752 --- /dev/null +++ b/tests/e2e.rs @@ -0,0 +1,4 @@ +#[path = "e2e/catalog.rs"] +mod catalog; +#[path = "e2e/harness.rs"] +mod harness; diff --git a/tests/e2e/catalog.rs b/tests/e2e/catalog.rs new file mode 100644 index 0000000..cd57e27 --- /dev/null +++ b/tests/e2e/catalog.rs @@ -0,0 +1,46 @@ +use crate::harness::E2eContext; + +#[test] +fn commands_lists_every_canonical_subcommand() { + let output = E2eContext::new() + .expect("E2E context should initialize") + .run(&["commands"]) + .expect("lsp-cli commands should run"); + + assert!( + output.status.success(), + "lsp-cli commands failed: {}", + String::from_utf8_lossy(&output.stderr) + ); + assert_eq!( + String::from_utf8(output.stdout) + .expect("lsp-cli commands output should be UTF-8") + .trim_end(), + concat!( + "commands\n", + "daemon\n", + "stop\n", + "stop-all\n", + "languages\n", + "servers\n", + "server-capabilities\n", + "detect\n", + "diagnostics\n", + "format\n", + "grep\n", + "list-symbols\n", + "list-functions\n", + "list-files\n", + "references\n", + "callers\n", + "callees\n", + "definition\n", + "declaration\n", + "build-index\n", + "update\n", + "completion\n", + "agent-skill\n", + "run" + ) + ); +} diff --git a/tests/e2e/harness.rs b/tests/e2e/harness.rs new file mode 100644 index 0000000..e6e477c --- /dev/null +++ b/tests/e2e/harness.rs @@ -0,0 +1,49 @@ +use std::fs; +use std::io; +use std::path::PathBuf; +use std::process::{Command, Output}; + +use tempfile::TempDir; + +pub(crate) struct E2eContext { + _sandbox: TempDir, + home: PathBuf, + config_home: PathBuf, + runtime_dir: PathBuf, + data_dir: PathBuf, +} + +impl E2eContext { + pub(crate) fn new() -> io::Result { + let sandbox = tempfile::Builder::new().prefix("lsp-cli-e2e-").tempdir()?; + let home = sandbox.path().join("home"); + let config_home = sandbox.path().join("config"); + let runtime_dir = sandbox.path().join("runtime"); + + for directory in [&home, &config_home, &runtime_dir] { + fs::create_dir(directory)?; + } + + Ok(Self { + _sandbox: sandbox, + home, + config_home, + runtime_dir, + data_dir: PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("data"), + }) + } + + pub(crate) fn run(&self, args: &[&str]) -> io::Result { + self.command().args(args).output() + } + + fn command(&self) -> Command { + let mut command = Command::new(env!("CARGO_BIN_EXE_lsp-cli")); + command + .env("HOME", &self.home) + .env("XDG_CONFIG_HOME", &self.config_home) + .env("XDG_RUNTIME_DIR", &self.runtime_dir) + .env("LSP_DATA", &self.data_dir); + command + } +} From c3a03d34f01e743e1146c276b59bb7b4b9fd479c Mon Sep 17 00:00:00 2001 From: Vasily Kulikov Date: Sat, 5 Sep 2026 17:11:11 +0300 Subject: [PATCH 03/18] cleanup --- .../e2e-supported-language-scope.md | 11 -- .../unprocessed/references-lua-performance.md | 32 ----- .../unprocessed/release-reference-profile.md | 117 ------------------ .../request-window-implementation.md | 27 ---- .../request-window-validation-difficulties.md | 27 ---- .../tmp-git-affects-workspace-root-tests.md | 10 -- 6 files changed, 224 deletions(-) delete mode 100644 .memory/unprocessed/e2e-supported-language-scope.md delete mode 100644 .memory/unprocessed/references-lua-performance.md delete mode 100644 .memory/unprocessed/release-reference-profile.md delete mode 100644 .memory/unprocessed/request-window-implementation.md delete mode 100644 .memory/unprocessed/request-window-validation-difficulties.md delete mode 100644 .memory/unprocessed/tmp-git-affects-workspace-root-tests.md diff --git a/.memory/unprocessed/e2e-supported-language-scope.md b/.memory/unprocessed/e2e-supported-language-scope.md deleted file mode 100644 index e39ca55..0000000 --- a/.memory/unprocessed/e2e-supported-language-scope.md +++ /dev/null @@ -1,11 +0,0 @@ -# Configured and detectable language scopes differ - -At pinned `lsp-cli-data` revision `013a75f6412917b710aa4683b9c5761c0c679975`, there are 362 -filetype YAML files, but only 16 have a non-empty `extensions` or `patterns` list. The other 346 -cannot be detected from a project even though the `languages` command currently collects every -loaded filetype ID without filtering for usable detection rules. - -This makes “all supported languages” ambiguous for E2E coverage. The proposed executable scope is -the 16 detectable IDs and their 57 relevant LSP configurations (141 compatible pairs), while the -full catalog receives configuration-validation coverage. Product ownership must confirm whether -the `languages` command is intended to expose catalog entries or only detectable languages. diff --git a/.memory/unprocessed/references-lua-performance.md b/.memory/unprocessed/references-lua-performance.md deleted file mode 100644 index 54a4a50..0000000 --- a/.memory/unprocessed/references-lua-performance.md +++ /dev/null @@ -1,32 +0,0 @@ -# References query performance investigation (2026-09-05) - -User requested investigation of a 17.715-second detached references query for -`normalize_timestamp` in `/home/segoon/projects/parley.nvim`. - -Installed `/home/segoon/.cargo/bin/lsp-cli` reproduced the same result in 19.744 and -22.107 seconds without debug logging. A timestamped debug run took 20.303 seconds: -initialize 0.076 s, workspace/symbol 0.040 s, 188 sequential documentSymbol requests -17.368 s combined, references 0.063 s, shutdown 0.024 s. Debug logging adds overhead, -so these are phase measurements, not a controlled estimate of every source of latency. - -`select_named_anchors` always scans all matching documents when document symbols are -supported. It is not merely a fallback for empty workspace-symbol results. Here -workspace/symbol returned null, so preferring workspace results alone would not help. -The scanned files were 100 under lua/, 86 under tests/, one plugin and one script. -Only lua/parley/timestamp.lua contains the literal query (checked with rg). - -Daemon::serve sleeps 25 ms after each iteration, draining upstream before downstream. -This adds latency to sequential exchanges, independently of server computation. ---limit truncates final output and does not bound discovery work. - -Direct runs used an isolated XDG_RUNTIME_DIR because connect_lsp_client reuses existing -sockets even with --no-detach. They took 45.521 s with debug and 44.738 s without: -query work finished, but server exit timed out after another 30 seconds. This is a -separate unresolved behavior, documented in GOTCHAS.md, not a successful workaround. - -Potential changes: event-driven daemon wakeups preserve discovery semantics but need -transport work; literal source prefiltering greatly reduces candidates here but could -miss server-provided names absent literally from source. No runtime code was changed. -Future regressions should exercise delayed/local-symbol discovery and verify protocol -request counts; daemon latency checks should use a fake immediate-response server. -Timestamped traces and result files are in /tmp/lsp-cli-{detached,direct}* for this session. diff --git a/.memory/unprocessed/release-reference-profile.md b/.memory/unprocessed/release-reference-profile.md deleted file mode 100644 index 18345f6..0000000 --- a/.memory/unprocessed/release-reference-profile.md +++ /dev/null @@ -1,117 +0,0 @@ -# Release references benchmark and remaining latency (2026-09-05) - -## User correction - -The user asked to investigate remaining latency and explicitly corrected the use of -**debug** benchmarks: use **release**. Future performance comparisons must build with -`cargo build --release` and invoke `target/release/lsp-cli`. No new product preference -or architectural decision was requested in this investigation. - -## Measurements - -Built the current uncommitted implementation in release. Ran without --debug against -/home/segoon/projects/parley.nvim, using isolated config/runtime directories and the -installed LuaLS executable. All six reference outputs were identical: - -| Request window | Run 1 | Run 2 | Run 3 | Median | -|---|---:|---:|---:|---:| -| 1 | 19.207 s | 21.177 s | 20.266 s | 20.266 s | -| 20 | 12.982 s | 13.438 s | 13.003 s | 13.003 s | - -These are repeated runs, not guaranteed warm-server runs. The project currently has -193 Lua files, compared with 188 in the earlier debug investigation, so the old and -new series are not a controlled comparison of compiler profiles alone. - -## Remaining time - -A temporary Python pass-through wrapper recorded message timestamps, payload sizes, -and Linux /proc CPU counters for LuaLS without verbose JSON logging. Two release -queries with a window of 20 took 12.809/13.211 seconds: - -- 193 documentSymbol requests: span 12.425/12.774 seconds at the server boundary. -- LuaLS CPU consumed across that span: 12.16/12.61 seconds. -- Foreground CLI user+system CPU: 0.574/0.626 seconds (excludes daemon CPU). -- Each run returned 5.509 MB of document-symbol JSON and about 300 diagnostic notifications. -- Actual references request at the server boundary: 0.0034/0.0030 seconds. -- Maximum observed outstanding documentSymbol requests: 20. - -Request latencies overlap and must not be summed as elapsed time. Server CPU and -foreground CPU also overlap; the figures are not additive phase timings. Wrapper -measurements have instrumentation overhead; the uninstrumented series above is the -baseline. - -## Controlled diagnostics comparison - -Used LuaLS --configpath pointing to a temporary copy of the project's .luarc.json, -with an explicit diagnostics.enable value. The real project configuration was not -edited. Both controls used the same wrapper, release binary, and request window 20. - -| diagnostics.enable | Run 1 wall | Run 2 wall | LuaLS CPU during symbol span | -|---|---:|---:|---:| -| true | 12.573 s | 12.375 s | 11.90 / 11.71 s | -| false | 4.502 s | 2.837 s | 3.62 / 2.45 s | - -All reference matches were identical to baseline. Disabled runs emitted no diagnostic -notifications and reused the same daemon/server PID. Thus file-open-triggered -background diagnostics explain much of the remaining cost; symbol generation, parsing, -and transport still remain. This experiment does not establish that disabling -server diagnostics is a generally acceptable behavior change. - -Installed source confirms the trigger: -- libexec/script/provider/provider.lua:271 handles didOpen, files.open, and compileState. -- libexec/script/provider/diagnostic.lua:678 watches file events; the open branch calls - doDiagnostic when the workspace is ready. -- libexec/script/provider/provider.lua:825 handles documentSymbol and converts all - returned symbols; requests still require file-wide symbol generation. - -## Daemon reuse defect - -The trace initially showed a new daemon and server on each command. Capturing daemon -stderr in a separate foreground-managed experiment reproduced: - - failed to write daemon client message: failed to write JSON-RPC message: Broken pipe (os error 32) - -The query itself succeeded. The daemon exited with status 1 and left a stale socket. -The coordinator drains upstream traffic before client events, and downstream write -errors propagate out of serve. Notifications racing with client disconnect can thus -terminate the daemon. This is intermittent: a manually managed daemon also survived -queries that still took 12.9/12.7 seconds, so lost reuse is a separate contributor, -not a complete explanation of the 13-second latency. For a fix, use a fake-server -regression that sends a notification after downstream disconnect and then verifies -that another client can use the same upstream server. - -## Implications and tradeoffs - -- Fixing disconnect handling should preserve reuse without changing query semantics, - but does not remove diagnostics triggered by opening every file again. -- Reducing file opens or caching discovery could avoid more work, but requires an - explicit completeness/invalidation design and server compatibility checks. -- Explicit user-selected LuaLS diagnostic settings give a concrete speed tradeoff: - faster query-only sessions at the cost of diagnostic availability in that session. - Do not hardcode Lua-specific configuration into generic lsp-cli logic. -- Runtime code and normal project/user configuration were not changed here. - -## Artifacts from this session - -- /tmp/lsp-release-bench-0z8zssym/results.json: uninstrumented release baseline. -- /tmp/lsp-release-trace-ab66lhav/{wire.jsonl,runs.json}: normal-config profile. - The wire log also has later foreground-daemon observations; filter by run timestamps. -- /tmp/lsp-diag-control-g7gfa7nw/{wire.jsonl,runs.json}: diagnostics enabled. -- /tmp/lsp-diag-control-h7mzmx60/{wire.jsonl,runs.json}: diagnostics disabled. -- /tmp/lsp-proxy-daemon-exit-v2jzda7x/daemon.stderr: captured broken-pipe failure. -- Benchmark and reproducer scripts: /tmp/lsp-cli-release-benchmark.py, - /tmp/lsp-cli-release-trace.py, /tmp/lsp-lua-trace-proxy.py, - /tmp/lsp-cli-diagnostics-control.py, /tmp/lsp-cli-proxy-daemon-exit-check.py. - -# Encountered difficulties - -## What confused me - -Repeated commands were initially assumed to be warm. PID traces and captured stderr -showed intermittent daemon termination. The live project also gained five Lua files -since the earlier measurements; release/debug cross-series comparisons need that caveat. - -## Where to report - -If you're sure the reported difficulties above are related to techplatform (e.g. userver, c35), -please report to [aisuite](https://nda.ya.ru/t/EcUMOwSH7eudWX). diff --git a/.memory/unprocessed/request-window-implementation.md b/.memory/unprocessed/request-window-implementation.md deleted file mode 100644 index 7baebdd..0000000 --- a/.memory/unprocessed/request-window-implementation.md +++ /dev/null @@ -1,27 +0,0 @@ -# Request-window implementation decisions and validation (2026-09-05) - -The user requested a default of 20 concurrent per-file requests, configurable as -`max-requests-in-flight` in lsp-cli.yaml. During planning the user selected: -- Named queries only (references, definition, declaration, callers, callees). -- Fail on document-symbol timeout instead of silently skipping it. - -The implementation uses bounded document-symbol scheduling in the client, matches -response IDs, decodes immediately, and preserves scan order. Synchronous request -transmission is shared. Do not invoke the unbounded notification-drain helper inside -window scheduling: continuous traffic could prevent deadline checks. Regression tests -exercise continuous notifications, refill with an older outstanding request, reversed -responses, server requests, cancellation, and deterministic named-query results. - -Debug-build benchmark, without verbose logging, with separate temporary configuration -and daemon runtime directories for each limit: -- parley.nvim, limit 1: 20.012 s cold / 22.382 s warm. -- parley.nvim, limit 20: 13.519 s cold / 13.200 s warm. -- Lua playground, limit 1: 1.139 s cold / 0.503 s warm. -- Lua playground, limit 20: 1.148 s cold / 0.352 s warm. -All four outputs for each workspace were identical. These are two measurements per -limit, not a statistical performance guarantee. Artifacts are in -/tmp/lsp-window-bench-7mvr24xb/results.json for this session. - -A direct-process fake server also verified the default window, 23 file requests, -reversed responses, all 23 anchors, and clean shutdown. Its temporary script and -fixture are /tmp/lsp-cli-window-stdio-check.py and /tmp/lsp-window-stdio-msmp85nt. diff --git a/.memory/unprocessed/request-window-validation-difficulties.md b/.memory/unprocessed/request-window-validation-difficulties.md deleted file mode 100644 index cb6237a..0000000 --- a/.memory/unprocessed/request-window-validation-difficulties.md +++ /dev/null @@ -1,27 +0,0 @@ -# Encountered difficulties - -## What confused me - -- The environment contains `/tmp/.git`. Three existing workspace-root tests then - discover `/tmp` as their project root. Running `TMPDIR=/dev/shm make test` isolates - fixture ancestry and passes those tests without changing unrelated code. -- Tests change PATH while other tests run. The new daemon echo-helper test must use - `/bin/cat`, not a PATH lookup. This avoids introducing another environment race. -- Unix socket creation is blocked in the sandbox; socket regression tests needed - execution outside it. -- The rust-analyzer command is a rustup proxy with no installed component. LuaLS was - available, so a small Lua playground was added for real-server validation. -- One isolated benchmark daemon returned an unexpected stop-response ID. Retrying - cleanup removed its stale socket; this is recorded in GOTCHAS.md and remains - separate from request-window scheduling. - -# Missing tools - -`make test` reaches `cargo deny check`, but cargo-deny is not installed. Formatting, -270 active tests (one ignored subprocess helper), Clippy, and README generation -consistency checks pass. No new dependency or tool was installed. - -## Where to report - -If you're sure the reported difficulties above are related to techplatform (e.g. userver, c35), -please report to [aisuite](https://nda.ya.ru/t/EcUMOwSH7eudWX). diff --git a/.memory/unprocessed/tmp-git-affects-workspace-root-tests.md b/.memory/unprocessed/tmp-git-affects-workspace-root-tests.md deleted file mode 100644 index 3799826..0000000 --- a/.memory/unprocessed/tmp-git-affects-workspace-root-tests.md +++ /dev/null @@ -1,10 +0,0 @@ -# A parent `/tmp/.git` changes workspace-root tests - -Several `suggest` tests create temporary workspaces directly below `/tmp` and configure `.git` as -an LSP root marker. When the host provides `/tmp/.git`, root-marker discovery resolves `/tmp` -instead of the test's temporary workspace, causing otherwise unrelated assertions to fail. - -Running the tests with `TMPDIR=/dev/shm` avoids that host-specific parent marker. A durable defense -would make these tests control every ancestor up to the search boundary, or let root discovery stop -at an explicit test boundary, rather than assuming the system temporary directory has no matching -root marker. From 2683f08d6696689d520f25a1adf0761e26d3830e Mon Sep 17 00:00:00 2001 From: Vasily Kulikov Date: Sat, 5 Sep 2026 17:19:15 +0300 Subject: [PATCH 04/18] Create test sandboxes outside tmp --- .../unprocessed/secure-test-temp-location.md | 9 ++++++++ ...temp-root-must-allow-short-socket-paths.md | 13 +++++++++++ E2E_TESTS.md | 7 ++++-- src/suggest.rs | 2 +- src/test_support.rs | 20 ++++++++++++++-- src/test_support/temp_root.rs | 23 +++++++++++++++++++ tests/e2e/harness.rs | 11 ++++++++- 7 files changed, 79 insertions(+), 6 deletions(-) create mode 100644 .memory/unprocessed/secure-test-temp-location.md create mode 100644 .memory/unprocessed/test-temp-root-must-allow-short-socket-paths.md create mode 100644 src/test_support/temp_root.rs diff --git a/.memory/unprocessed/secure-test-temp-location.md b/.memory/unprocessed/secure-test-temp-location.md new file mode 100644 index 0000000..0ad0471 --- /dev/null +++ b/.memory/unprocessed/secure-test-temp-location.md @@ -0,0 +1,9 @@ +# Real test sandboxes must not use ambient `/tmp` + +The user clarified that real test state must not be created under `/tmp`, even through the secure +default `tempfile::Builder::tempdir` API. Unit and E2E test helpers should use the existing +`tempfile` crate to create randomized exclusive directories under `XDG_RUNTIME_DIR`, +`XDG_CACHE_HOME`, or `$HOME/.cache`, in that order. + +Synthetic `/tmp` strings used only to test URI or path parsing do not create files and are outside +this policy. diff --git a/.memory/unprocessed/test-temp-root-must-allow-short-socket-paths.md b/.memory/unprocessed/test-temp-root-must-allow-short-socket-paths.md new file mode 100644 index 0000000..3f7523d --- /dev/null +++ b/.memory/unprocessed/test-temp-root-must-allow-short-socket-paths.md @@ -0,0 +1,13 @@ +# Test temp roots must leave room for Unix socket names + +Creating secure test sandboxes below this worktree's `target/test-tmp/` made the absolute paths long +enough that existing Unix listener tests failed with `path must be shorter than SUN_LEN`. Workspace +and daemon test helpers must select a short per-user runtime or cache root, not merely any secure +repository-local directory. + +This should be defended by keeping test directory prefixes compact and by retaining socket tests +that bind the longest production-shaped daemon path. + +The root must also be selected from the build-time environment. Reading `XDG_RUNTIME_DIR` or +`HOME` while tests run races with existing tests that temporarily replace process-wide environment +variables and can place one test's sandbox inside another test's short-lived directory. diff --git a/E2E_TESTS.md b/E2E_TESTS.md index 809470b..b4687a3 100644 --- a/E2E_TESTS.md +++ b/E2E_TESTS.md @@ -102,8 +102,11 @@ it should contain: Prefer equivalent domain concepts and symbol names across projects when natural. Do not force a language into constructs it does not support merely to make fixtures textually identical. -Tests copy a project into a temporary directory before formatting it or introducing diagnostics. -This keeps the repository clean and allows safe parallel execution. +Tests securely create randomized sandboxes with the `tempfile` crate under the user's +`XDG_RUNTIME_DIR`, `XDG_CACHE_HOME`, or `$HOME/.cache`, in that order, then copy a project there +before formatting it or introducing diagnostics. Real test state must not use the ambient system +`/tmp`. This keeps the repository clean, avoids uncontrolled parent root markers, keeps Unix socket +paths short, and allows safe parallel execution. Pros of committed playgrounds: humans can reproduce failures with the same projects. Cons: each language fixture must evolve with its toolchain and server ecosystem. diff --git a/src/suggest.rs b/src/suggest.rs index f7a2ed9..47b6fcb 100644 --- a/src/suggest.rs +++ b/src/suggest.rs @@ -191,7 +191,7 @@ mod tests { LspConfig { id: "example_lsp".to_string(), filetypes: vec!["alpha".to_string(), "beta".to_string()], - root_markers: vec![".workspace-root".to_string(), ".git".to_string()], + root_markers: vec![".workspace-root".to_string()], name: "example-lsp".to_string(), cmdline: "example-lsp --stdio $WORKSPACE".to_string(), wait_for_index: false, diff --git a/src/test_support.rs b/src/test_support.rs index 52c38cb..c9530a4 100644 --- a/src/test_support.rs +++ b/src/test_support.rs @@ -1,4 +1,5 @@ pub(crate) mod lsp_peer; +mod temp_root; use crate::detect::DetectionResult; use crate::mason::registry::{MasonDownload, MasonNeovim, MasonPackage, MasonSource, OneOrMany}; @@ -12,6 +13,8 @@ use std::sync::{Mutex, OnceLock}; use std::time::{SystemTime, UNIX_EPOCH}; use tempfile::TempDir; +use self::temp_root::test_temp_root; + pub(crate) const LOCAL_SHARE_LSP_CLI: &str = ".local/share/lsp-cli"; #[cfg(test)] pub(crate) const SUBPROCESS_HELPER_MODE_ENV: &str = "LSP_CLI_TEST_HELPER_MODE"; @@ -30,9 +33,11 @@ pub(crate) struct TestDir { impl TestDir { pub(crate) fn new(prefix: &str) -> Self { + let test_temp_root = test_temp_root().expect("secure test temp root should be selected"); + fs::create_dir_all(&test_temp_root).expect("test temp root should be created"); let dir = tempfile::Builder::new() .prefix(&format!("lsp-cli-{prefix}-test-")) - .tempdir() + .tempdir_in(test_temp_root) .expect("temp dir should be created"); Self { dir } } @@ -279,11 +284,22 @@ pub(crate) fn make_executable(path: &Path) { mod tests { use super::{ SUBPROCESS_HELPER_EXIT_CODE_ENV, SUBPROCESS_HELPER_MODE_ENV, - SUBPROCESS_HELPER_OUTPUT_PATH_ENV, SUBPROCESS_HELPER_STDERR_ENV, + SUBPROCESS_HELPER_OUTPUT_PATH_ENV, SUBPROCESS_HELPER_STDERR_ENV, TestDir, test_temp_root, }; use std::fs; use std::io::Write as _; + #[test] + fn creates_test_directories_in_secure_selected_root() { + let dir = TestDir::new("secure-root"); + + assert!( + dir.path() + .starts_with(test_temp_root().expect("secure test temp root should be selected")) + ); + assert!(!dir.path().starts_with("/tmp")); + } + #[test] #[ignore = "subprocess helper"] fn subprocess_helper() { diff --git a/src/test_support/temp_root.rs b/src/test_support/temp_root.rs new file mode 100644 index 0000000..64362b0 --- /dev/null +++ b/src/test_support/temp_root.rs @@ -0,0 +1,23 @@ +use std::io; +use std::path::{Path, PathBuf}; + +pub(crate) fn test_temp_root() -> io::Result { + let base = option_env!("XDG_RUNTIME_DIR") + .map(PathBuf::from) + .filter(|path| acceptable_test_temp_base(path)) + .or_else(|| option_env!("XDG_CACHE_HOME").map(PathBuf::from)) + .filter(|path| acceptable_test_temp_base(path)) + .or_else(|| option_env!("HOME").map(|home| PathBuf::from(home).join(".cache"))) + .filter(|path| acceptable_test_temp_base(path)) + .ok_or_else(|| { + io::Error::new( + io::ErrorKind::NotFound, + "an absolute XDG_RUNTIME_DIR, XDG_CACHE_HOME, or HOME outside /tmp is required", + ) + })?; + Ok(base.join("lsp-cli/test-tmp")) +} + +fn acceptable_test_temp_base(path: &Path) -> bool { + path.is_absolute() && !path.starts_with("/tmp") +} diff --git a/tests/e2e/harness.rs b/tests/e2e/harness.rs index e6e477c..e64ebbf 100644 --- a/tests/e2e/harness.rs +++ b/tests/e2e/harness.rs @@ -5,6 +5,11 @@ use std::process::{Command, Output}; use tempfile::TempDir; +#[path = "../../src/test_support/temp_root.rs"] +mod temp_root; + +use self::temp_root::test_temp_root; + pub(crate) struct E2eContext { _sandbox: TempDir, home: PathBuf, @@ -15,7 +20,11 @@ pub(crate) struct E2eContext { impl E2eContext { pub(crate) fn new() -> io::Result { - let sandbox = tempfile::Builder::new().prefix("lsp-cli-e2e-").tempdir()?; + let test_temp_root = test_temp_root()?; + fs::create_dir_all(&test_temp_root)?; + let sandbox = tempfile::Builder::new() + .prefix("lsp-cli-e2e-") + .tempdir_in(test_temp_root)?; let home = sandbox.path().join("home"); let config_home = sandbox.path().join("config"); let runtime_dir = sandbox.path().join("runtime"); From c8a66fa1c9e5b7225776cc37d69b2015049bb65f Mon Sep 17 00:00:00 2001 From: Vasily Kulikov Date: Sat, 5 Sep 2026 17:46:27 +0300 Subject: [PATCH 05/18] Add E2E manifest validation --- E2E_TESTS.md | 7 +- tests/e2e.rs | 2 + tests/e2e/cases.yaml | 11 ++ tests/e2e/manifest.rs | 380 ++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 399 insertions(+), 1 deletion(-) create mode 100644 tests/e2e/cases.yaml create mode 100644 tests/e2e/manifest.rs diff --git a/E2E_TESTS.md b/E2E_TESTS.md index b4687a3..c3ad364 100644 --- a/E2E_TESTS.md +++ b/E2E_TESTS.md @@ -216,6 +216,11 @@ documented exclusions. A validation test should fail when: - two cases select the same user-visible server ambiguously; - a new top-level subcommand has no assigned coverage class. +The version 1 manifest starts with `coverage: partial`, which validates every declared entry +against the pinned data without requiring unfinished matrix entries. Phase 4 adds the remaining +entries and switches it to `coverage: complete`; complete mode enforces every detectable language +and compatible pair. + ## Special command strategies ### `run` @@ -355,7 +360,7 @@ that class of defect easier to diagnose. ### Phase 1: foundation - [x] Add `tests/e2e.rs` and compact harness modules. -- [ ] Add the initial manifest schema and validation. +- [x] Add the initial manifest schema and validation. - [ ] Isolate all environment and runtime state. - [ ] Implement deadlines, cleanup guards, JSON helpers, and useful failure diagnostics. - [ ] Prove the harness with Rust/rust-analyzer. diff --git a/tests/e2e.rs b/tests/e2e.rs index 5a80752..fc380de 100644 --- a/tests/e2e.rs +++ b/tests/e2e.rs @@ -2,3 +2,5 @@ mod catalog; #[path = "e2e/harness.rs"] mod harness; +#[path = "e2e/manifest.rs"] +mod manifest; diff --git a/tests/e2e/cases.yaml b/tests/e2e/cases.yaml new file mode 100644 index 0000000..3865b6f --- /dev/null +++ b/tests/e2e/cases.yaml @@ -0,0 +1,11 @@ +schema-version: 1 +coverage: partial + +languages: + - id: rust + kind: source + project: playground/rust + +pairs: + - language: rust + server: rust_analyzer diff --git a/tests/e2e/manifest.rs b/tests/e2e/manifest.rs new file mode 100644 index 0000000..c404ddb --- /dev/null +++ b/tests/e2e/manifest.rs @@ -0,0 +1,380 @@ +use std::collections::BTreeSet; +use std::fs; +use std::path::{Component, Path, PathBuf}; + +use serde::Deserialize; +use serde::de::DeserializeOwned; + +const MANIFEST_SCHEMA_VERSION: u32 = 1; + +#[derive(Clone, Debug, Deserialize)] +#[serde(rename_all = "kebab-case", deny_unknown_fields)] +struct Manifest { + schema_version: u32, + coverage: Coverage, + languages: Vec, + pairs: Vec, +} + +#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq)] +#[serde(rename_all = "kebab-case")] +enum Coverage { + Partial, + Complete, +} + +#[derive(Clone, Debug, Deserialize)] +#[serde(rename_all = "kebab-case", deny_unknown_fields)] +struct LanguageCase { + id: String, + kind: ProjectKind, + project: PathBuf, +} + +#[derive(Clone, Copy, Debug, Deserialize)] +#[serde(rename_all = "kebab-case")] +enum ProjectKind { + Source, + Metadata, +} + +impl ProjectKind { + fn label(self) -> &'static str { + match self { + Self::Source => "source", + Self::Metadata => "metadata", + } + } +} + +#[derive(Clone, Debug, Deserialize, Eq, Ord, PartialEq, PartialOrd)] +#[serde(rename_all = "kebab-case", deny_unknown_fields)] +struct PairCase { + language: String, + server: String, +} + +#[derive(Deserialize)] +struct FiletypeConfig { + #[serde(default)] + extensions: Vec, + #[serde(default)] + patterns: Vec, +} + +impl FiletypeConfig { + fn is_detectable(&self) -> bool { + !self.extensions.is_empty() || !self.patterns.is_empty() + } +} + +#[derive(Deserialize)] +struct LspConfig { + #[serde(default)] + filetypes: Vec, +} + +impl Manifest { + fn load() -> Result { + serde_yaml::from_str(include_str!("cases.yaml")) + .map_err(|error| format!("failed to parse E2E manifest: {error}")) + } + + fn validate(&self, repository: &Path) -> Result<(), String> { + if self.schema_version != MANIFEST_SCHEMA_VERSION { + return Err(format!( + "unsupported E2E manifest schema version {}; expected {MANIFEST_SCHEMA_VERSION}", + self.schema_version + )); + } + if self.languages.is_empty() { + return Err("E2E manifest must declare at least one language".to_string()); + } + if self.pairs.is_empty() { + return Err("E2E manifest must declare at least one language/server pair".to_string()); + } + + let data = repository.join("data"); + let declared_languages = self.validate_languages(repository, &data)?; + let declared_pairs = self.validate_pairs(&data, &declared_languages)?; + + if self.coverage == Coverage::Complete { + Self::validate_complete_coverage(&data, &declared_languages, &declared_pairs)?; + } + Ok(()) + } + + fn validate_languages( + &self, + repository: &Path, + data: &Path, + ) -> Result, String> { + let mut declared = BTreeSet::new(); + for language in &self.languages { + validate_config_id("language", &language.id)?; + if !declared.insert(language.id.clone()) { + return Err(format!( + "E2E manifest declares language {:?} more than once", + language.id + )); + } + + let filetype_path = data.join("filetypes").join(format!("{}.yaml", language.id)); + let filetype: FiletypeConfig = read_yaml(&filetype_path)?; + if !filetype.is_detectable() { + return Err(format!( + "E2E language {:?} has no extension or filename-pattern detection rule", + language.id + )); + } + language.validate_project(repository)?; + } + Ok(declared) + } + + fn validate_pairs( + &self, + data: &Path, + declared_languages: &BTreeSet, + ) -> Result, String> { + let mut declared = BTreeSet::new(); + for pair in &self.pairs { + validate_config_id("server", &pair.server)?; + if !declared_languages.contains(&pair.language) { + return Err(format!( + "E2E pair {}/{} references an undeclared language", + pair.language, pair.server + )); + } + if !declared.insert(pair.clone()) { + return Err(format!( + "E2E manifest declares pair {}/{} more than once", + pair.language, pair.server + )); + } + + let lsp_path = data.join("lsp").join(format!("{}.yaml", pair.server)); + let lsp: LspConfig = read_yaml(&lsp_path)?; + if !lsp.filetypes.contains(&pair.language) { + return Err(format!( + "LSP config {:?} does not support language {:?}", + pair.server, pair.language + )); + } + } + Ok(declared) + } + + fn validate_complete_coverage( + data: &Path, + declared_languages: &BTreeSet, + declared_pairs: &BTreeSet, + ) -> Result<(), String> { + let detectable = detectable_languages(data)?; + let missing_languages = detectable + .difference(declared_languages) + .cloned() + .collect::>(); + if !missing_languages.is_empty() { + return Err(format!( + "complete E2E manifest is missing languages: {}", + missing_languages.join(", ") + )); + } + + let compatible = compatible_pairs(data, &detectable)?; + let missing_pairs = compatible + .difference(declared_pairs) + .map(|pair| format!("{}/{}", pair.language, pair.server)) + .collect::>(); + if !missing_pairs.is_empty() { + return Err(format!( + "complete E2E manifest is missing pairs: {}", + missing_pairs.join(", ") + )); + } + Ok(()) + } +} + +impl LanguageCase { + fn validate_project(&self, repository: &Path) -> Result<(), String> { + if self.project.is_absolute() + || self + .project + .components() + .any(|component| !matches!(component, Component::Normal(_))) + { + return Err(format!( + "E2E project path {} must be a normalized relative path", + self.project.display() + )); + } + + let project = repository.join(&self.project); + if !project.is_dir() { + return Err(format!( + "E2E {} project {} for language {:?} is not a directory", + self.kind.label(), + self.project.display(), + self.id + )); + } + let canonical_repository = repository + .canonicalize() + .map_err(|error| format!("failed to resolve {}: {error}", repository.display()))?; + let canonical_project = project + .canonicalize() + .map_err(|error| format!("failed to resolve {}: {error}", project.display()))?; + if !canonical_project.starts_with(canonical_repository) { + return Err(format!( + "E2E project {} resolves outside the repository", + self.project.display() + )); + } + Ok(()) + } +} + +fn detectable_languages(data: &Path) -> Result, String> { + let mut languages = BTreeSet::new(); + for path in yaml_paths(&data.join("filetypes"))? { + let config: FiletypeConfig = read_yaml(&path)?; + if config.is_detectable() { + languages.insert(file_stem(&path)?); + } + } + Ok(languages) +} + +fn compatible_pairs( + data: &Path, + detectable: &BTreeSet, +) -> Result, String> { + let mut pairs = BTreeSet::new(); + for path in yaml_paths(&data.join("lsp"))? { + let config: LspConfig = read_yaml(&path)?; + let server = file_stem(&path)?; + for language in config.filetypes { + if detectable.contains(&language) { + pairs.insert(PairCase { + language, + server: server.clone(), + }); + } + } + } + Ok(pairs) +} + +fn yaml_paths(directory: &Path) -> Result, String> { + let entries = fs::read_dir(directory) + .map_err(|error| format!("failed to read {}: {error}", directory.display()))?; + let mut paths = entries + .map(|entry| { + entry + .map(|entry| entry.path()) + .map_err(|error| format!("failed to read {}: {error}", directory.display())) + }) + .collect::, _>>()?; + paths.retain(|path| path.extension().and_then(|extension| extension.to_str()) == Some("yaml")); + paths.sort(); + Ok(paths) +} + +fn read_yaml(path: &Path) -> Result { + let contents = fs::read_to_string(path) + .map_err(|error| format!("failed to read {}: {error}", path.display()))?; + serde_yaml::from_str(&contents) + .map_err(|error| format!("failed to parse {}: {error}", path.display())) +} + +fn file_stem(path: &Path) -> Result { + path.file_stem() + .and_then(|stem| stem.to_str()) + .map(str::to_owned) + .ok_or_else(|| format!("{} has no UTF-8 file stem", path.display())) +} + +fn validate_config_id(kind: &str, value: &str) -> Result<(), String> { + let mut components = Path::new(value).components(); + if value.is_empty() + || !matches!(components.next(), Some(Component::Normal(_))) + || components.next().is_some() + { + return Err(format!( + "E2E {kind} config ID {value:?} must be one normalized path component" + )); + } + Ok(()) +} + +fn repository_root() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")) +} + +#[test] +fn partial_manifest_matches_pinned_data() { + Manifest::load() + .expect("E2E manifest should parse") + .validate(&repository_root()) + .expect("E2E manifest should be valid"); +} + +#[test] +fn complete_mode_rejects_the_partial_matrix() { + let mut manifest = Manifest::load().expect("E2E manifest should parse"); + manifest.coverage = Coverage::Complete; + + let error = manifest + .validate(&repository_root()) + .expect_err("partial matrix should not satisfy complete coverage"); + + assert!(error.contains("complete E2E manifest is missing languages")); +} + +#[test] +fn complete_mode_rejects_missing_server_pairs() { + let data = repository_root().join("data"); + let detectable = detectable_languages(&data).expect("filetype configs should load"); + let declared_pairs = BTreeSet::from([PairCase { + language: "rust".to_string(), + server: "rust_analyzer".to_string(), + }]); + + let error = Manifest::validate_complete_coverage(&data, &detectable, &declared_pairs) + .expect_err("partial server matrix should not satisfy complete coverage"); + + assert!(error.contains("complete E2E manifest is missing pairs")); +} + +#[test] +fn manifest_rejects_unknown_fields() { + let error = serde_yaml::from_str::( + "schema-version: 1\ncoverage: partial\nlanguages: []\npairs: []\nunknown: true\n", + ) + .expect_err("unknown manifest fields should fail"); + + assert!(error.to_string().contains("unknown field `unknown`")); +} + +#[test] +fn manifest_rejects_config_path_traversal() { + let mut manifest = Manifest::load().expect("E2E manifest should parse"); + manifest + .languages + .first_mut() + .expect("manifest should contain a language") + .id = "../rust".to_string(); + manifest + .pairs + .first_mut() + .expect("manifest should contain a pair") + .language = "../rust".to_string(); + + let error = manifest + .validate(&repository_root()) + .expect_err("config path traversal should fail"); + + assert!(error.contains("must be one normalized path component")); +} From 905c1b06e66c94ec4d221b9704cc6ebedd93acc8 Mon Sep 17 00:00:00 2001 From: Vasily Kulikov Date: Sat, 5 Sep 2026 17:56:23 +0300 Subject: [PATCH 06/18] Isolate E2E process state --- .../e2e-runtime-root-constraints.md | 11 +++ E2E_TESTS.md | 2 +- src/test_support/temp_root.rs | 9 +- tests/e2e/harness.rs | 86 ++++++++++++++++++- 4 files changed, 100 insertions(+), 8 deletions(-) create mode 100644 .memory/unprocessed/e2e-runtime-root-constraints.md diff --git a/.memory/unprocessed/e2e-runtime-root-constraints.md b/.memory/unprocessed/e2e-runtime-root-constraints.md new file mode 100644 index 0000000..9d9d644 --- /dev/null +++ b/.memory/unprocessed/e2e-runtime-root-constraints.md @@ -0,0 +1,11 @@ +# E2E runtime-root constraints + +The E2E daemon runtime directory cannot be nested below the ordinary test root. The production +socket suffix (`lsp-cli/`, a server slug of up to 32 characters, a separator, a 24-character hash, +and `.sock`) makes that layout exceed the Unix-domain socket path limit in the current environment. +Create the runtime `TempDir` directly below the selected secure base with a short prefix. + +The configured build-time `XDG_RUNTIME_DIR` may also be mounted read-only by an execution sandbox +even though it is writable in the host environment. Tests using the secure root can therefore need +the test command to run with the sandbox's filesystem restriction lifted; falling back to `/tmp` +would violate the project requirement and conceal the actual path behavior. diff --git a/E2E_TESTS.md b/E2E_TESTS.md index c3ad364..c555455 100644 --- a/E2E_TESTS.md +++ b/E2E_TESTS.md @@ -361,7 +361,7 @@ that class of defect easier to diagnose. - [x] Add `tests/e2e.rs` and compact harness modules. - [x] Add the initial manifest schema and validation. -- [ ] Isolate all environment and runtime state. +- [x] Isolate all environment and runtime state. - [ ] Implement deadlines, cleanup guards, JSON helpers, and useful failure diagnostics. - [ ] Prove the harness with Rust/rust-analyzer. - [ ] Cover all 24 subcommand paths with either a real server or a deterministic local fixture. diff --git a/src/test_support/temp_root.rs b/src/test_support/temp_root.rs index 64362b0..244b7ff 100644 --- a/src/test_support/temp_root.rs +++ b/src/test_support/temp_root.rs @@ -2,7 +2,11 @@ use std::io; use std::path::{Path, PathBuf}; pub(crate) fn test_temp_root() -> io::Result { - let base = option_env!("XDG_RUNTIME_DIR") + Ok(test_temp_base()?.join("lsp-cli/test-tmp")) +} + +pub(crate) fn test_temp_base() -> io::Result { + option_env!("XDG_RUNTIME_DIR") .map(PathBuf::from) .filter(|path| acceptable_test_temp_base(path)) .or_else(|| option_env!("XDG_CACHE_HOME").map(PathBuf::from)) @@ -14,8 +18,7 @@ pub(crate) fn test_temp_root() -> io::Result { io::ErrorKind::NotFound, "an absolute XDG_RUNTIME_DIR, XDG_CACHE_HOME, or HOME outside /tmp is required", ) - })?; - Ok(base.join("lsp-cli/test-tmp")) + }) } fn acceptable_test_temp_base(path: &Path) -> bool { diff --git a/tests/e2e/harness.rs b/tests/e2e/harness.rs index e64ebbf..8365a34 100644 --- a/tests/e2e/harness.rs +++ b/tests/e2e/harness.rs @@ -8,13 +8,16 @@ use tempfile::TempDir; #[path = "../../src/test_support/temp_root.rs"] mod temp_root; -use self::temp_root::test_temp_root; +use self::temp_root::{test_temp_base, test_temp_root}; pub(crate) struct E2eContext { _sandbox: TempDir, + _runtime_sandbox: TempDir, home: PathBuf, config_home: PathBuf, runtime_dir: PathBuf, + workspace: PathBuf, + bin_dir: PathBuf, data_dir: PathBuf, } @@ -25,19 +28,30 @@ impl E2eContext { let sandbox = tempfile::Builder::new() .prefix("lsp-cli-e2e-") .tempdir_in(test_temp_root)?; + let test_temp_base = test_temp_base()?; + fs::create_dir_all(&test_temp_base)?; + // Keep this prefix and hierarchy short: daemon socket paths have a small OS limit. + let runtime_sandbox = tempfile::Builder::new() + .prefix("e-") + .tempdir_in(test_temp_base)?; let home = sandbox.path().join("home"); let config_home = sandbox.path().join("config"); - let runtime_dir = sandbox.path().join("runtime"); + let workspace = sandbox.path().join("workspace"); + let bin_dir = sandbox.path().join("bin"); + let runtime_dir = runtime_sandbox.path().to_path_buf(); - for directory in [&home, &config_home, &runtime_dir] { + for directory in [&home, &config_home, &workspace, &bin_dir] { fs::create_dir(directory)?; } Ok(Self { _sandbox: sandbox, + _runtime_sandbox: runtime_sandbox, home, config_home, runtime_dir, + workspace, + bin_dir, data_dir: PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("data"), }) } @@ -49,10 +63,74 @@ impl E2eContext { fn command(&self) -> Command { let mut command = Command::new(env!("CARGO_BIN_EXE_lsp-cli")); command + .env_clear() .env("HOME", &self.home) .env("XDG_CONFIG_HOME", &self.config_home) .env("XDG_RUNTIME_DIR", &self.runtime_dir) - .env("LSP_DATA", &self.data_dir); + .env("LSP_DATA", &self.data_dir) + .env("PATH", &self.bin_dir) + .env("LANG", "C") + .env("LC_ALL", "C") + .env("TZ", "UTC") + .current_dir(&self.workspace); command } } + +#[cfg(test)] +mod tests { + use std::collections::BTreeMap; + use std::ffi::OsString; + + use super::*; + + #[test] + fn command_isolated_from_ambient_process_state() { + let context = E2eContext::new().expect("E2E context should initialize"); + let command = context.command(); + let actual = command + .get_envs() + .map(|(name, value)| (name.to_os_string(), value.map(OsString::from))) + .collect::>(); + let expected = [ + ("HOME", context.home.as_os_str().to_os_string()), + ("LANG", OsString::from("C")), + ("LC_ALL", OsString::from("C")), + ("LSP_DATA", context.data_dir.as_os_str().to_os_string()), + ("PATH", context.bin_dir.as_os_str().to_os_string()), + ("TZ", OsString::from("UTC")), + ( + "XDG_CONFIG_HOME", + context.config_home.as_os_str().to_os_string(), + ), + ( + "XDG_RUNTIME_DIR", + context.runtime_dir.as_os_str().to_os_string(), + ), + ] + .into_iter() + .map(|(name, value)| (OsString::from(name), Some(value))) + .collect::>(); + + assert_eq!(actual, expected); + assert_eq!(command.get_current_dir(), Some(context.workspace.as_path())); + assert!(!context.runtime_dir.starts_with(context._sandbox.path())); + } + + #[cfg(unix)] + #[test] + fn runtime_directory_has_room_for_daemon_socket_name() { + use std::os::unix::net::UnixListener; + + let context = E2eContext::new().expect("E2E context should initialize"); + let daemon_root = context.runtime_dir.join("lsp-cli"); + fs::create_dir(&daemon_root).expect("daemon root should be created"); + let socket_path = daemon_root.join(format!("{}-{}.sock", "s".repeat(32), "f".repeat(24))); + let _listener = UnixListener::bind(&socket_path).unwrap_or_else(|error| { + panic!( + "E2E runtime path {} cannot hold a daemon socket: {error}", + socket_path.display() + ) + }); + } +} From 5453c32df2d997b4a75b65808d606c45bb8bbb56 Mon Sep 17 00:00:00 2001 From: Vasily Kulikov Date: Sat, 5 Sep 2026 18:18:07 +0300 Subject: [PATCH 07/18] Add E2E execution safeguards --- ...ocess-deadline-includes-inherited-pipes.md | 7 + .../process-group-cleanup-reaping.md | 10 + Cargo.lock | 54 +++ Cargo.toml | 4 + E2E_TESTS.md | 2 +- tests/e2e.rs | 7 + tests/e2e/catalog.rs | 14 +- tests/e2e/harness.rs | 281 ++++++++++++- tests/e2e/process.rs | 397 ++++++++++++++++++ 9 files changed, 759 insertions(+), 17 deletions(-) create mode 100644 .memory/unprocessed/process-deadline-includes-inherited-pipes.md create mode 100644 .memory/unprocessed/process-group-cleanup-reaping.md create mode 100644 tests/e2e/process.rs diff --git a/.memory/unprocessed/process-deadline-includes-inherited-pipes.md b/.memory/unprocessed/process-deadline-includes-inherited-pipes.md new file mode 100644 index 0000000..6232050 --- /dev/null +++ b/.memory/unprocessed/process-deadline-includes-inherited-pipes.md @@ -0,0 +1,7 @@ +# Process deadline must include inherited output pipes + +Waiting with a deadline only on the process-group leader is insufficient. The leader can exit while +a descendant remains alive with inherited stdout or stderr descriptors, causing output-reader joins +to block forever after the nominal deadline. Apply the same absolute deadline to both leader exit +and pipe closure; if either pipe remains open, kill the still-addressable process group before +joining readers. diff --git a/.memory/unprocessed/process-group-cleanup-reaping.md b/.memory/unprocessed/process-group-cleanup-reaping.md new file mode 100644 index 0000000..8becee6 --- /dev/null +++ b/.memory/unprocessed/process-group-cleanup-reaping.md @@ -0,0 +1,10 @@ +# Process-group cleanup and descendant reaping + +`command-group` successfully sends the kill signal to the E2E command and its descendants, but a +killed grandchild can remain briefly visible in `/proc` as a zombie until its new parent reaps it. +A cleanup regression test must use a bounded wait for the process entry to disappear instead of +interpreting its immediate presence as a live orphan. + +After killing and reaping a process group, the RAII guard must discard its `GroupChild` handle. +Leaving the handle armed makes `Drop` attempt a second group kill; in the unlikely event that the +process-group ID has already been reused, that could signal an unrelated process. diff --git a/Cargo.lock b/Cargo.lock index e577390..e577c1a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -195,6 +195,16 @@ version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" +[[package]] +name = "command-group" +version = "5.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a68fa787550392a9d58f44c21a3022cfb3ea3e2458b7f85d3b399d0ceeccf409" +dependencies = [ + "nix", + "winapi", +] + [[package]] name = "const-oid" version = "0.10.2" @@ -762,6 +772,7 @@ version = "0.1.5" dependencies = [ "clap", "clap_complete", + "command-group", "flate2", "humantime", "lsp-types", @@ -776,6 +787,7 @@ dependencies = [ "tempfile", "thiserror", "url", + "wait-timeout", "zip", ] @@ -819,6 +831,17 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "nix" +version = "0.27.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2eb04e9c688eff1c89d72b407f168cf79bb9e867a9d3323ed6c01519eb9cc053" +dependencies = [ + "bitflags 2.11.1", + "cfg-if", + "libc", +] + [[package]] name = "once_cell" version = "1.21.4" @@ -1568,6 +1591,15 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" +[[package]] +name = "wait-timeout" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ac3b126d3914f9849036f826e054cbabdc8519970b8998ddaf3b5bd3c65f11" +dependencies = [ + "libc", +] + [[package]] name = "want" version = "0.3.1" @@ -1719,6 +1751,28 @@ dependencies = [ "rustls-pki-types", ] +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + [[package]] name = "windows-link" version = "0.2.1" diff --git a/Cargo.toml b/Cargo.toml index 59f1230..4faf213 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -30,6 +30,10 @@ thiserror = "2.0.17" url = "2.5.7" zip = { version = "8.6.0", default-features = false, features = ["deflate"] } +[dev-dependencies] +command-group = "5.0.1" +wait-timeout = "0.2.1" + [lints] workspace = true diff --git a/E2E_TESTS.md b/E2E_TESTS.md index c555455..6d52154 100644 --- a/E2E_TESTS.md +++ b/E2E_TESTS.md @@ -362,7 +362,7 @@ that class of defect easier to diagnose. - [x] Add `tests/e2e.rs` and compact harness modules. - [x] Add the initial manifest schema and validation. - [x] Isolate all environment and runtime state. -- [ ] Implement deadlines, cleanup guards, JSON helpers, and useful failure diagnostics. +- [x] Implement deadlines, cleanup guards, JSON helpers, and useful failure diagnostics. - [ ] Prove the harness with Rust/rust-analyzer. - [ ] Cover all 24 subcommand paths with either a real server or a deterministic local fixture. diff --git a/tests/e2e.rs b/tests/e2e.rs index fc380de..b518555 100644 --- a/tests/e2e.rs +++ b/tests/e2e.rs @@ -1,6 +1,13 @@ +#![expect( + clippy::panic, + reason = "E2E assertion helpers panic with captured process diagnostics." +)] + #[path = "e2e/catalog.rs"] mod catalog; #[path = "e2e/harness.rs"] mod harness; #[path = "e2e/manifest.rs"] mod manifest; +#[path = "e2e/process.rs"] +mod process; diff --git a/tests/e2e/catalog.rs b/tests/e2e/catalog.rs index cd57e27..7a9683b 100644 --- a/tests/e2e/catalog.rs +++ b/tests/e2e/catalog.rs @@ -4,18 +4,12 @@ use crate::harness::E2eContext; fn commands_lists_every_canonical_subcommand() { let output = E2eContext::new() .expect("E2E context should initialize") - .run(&["commands"]) - .expect("lsp-cli commands should run"); + .run(&["commands"]); - assert!( - output.status.success(), - "lsp-cli commands failed: {}", - String::from_utf8_lossy(&output.stderr) - ); + output.assert_success(); + assert!(output.stderr_text().is_empty()); assert_eq!( - String::from_utf8(output.stdout) - .expect("lsp-cli commands output should be UTF-8") - .trim_end(), + output.stdout_text().trim_end(), concat!( "commands\n", "daemon\n", diff --git a/tests/e2e/harness.rs b/tests/e2e/harness.rs index 8365a34..401ec8c 100644 --- a/tests/e2e/harness.rs +++ b/tests/e2e/harness.rs @@ -1,15 +1,23 @@ +use std::ffi::OsStr; use std::fs; use std::io; use std::path::PathBuf; -use std::process::{Command, Output}; +use std::process::Command; +use std::time::Duration; +use serde::de::DeserializeOwned; use tempfile::TempDir; +use crate::process::{self, ProcessOutput}; + #[path = "../../src/test_support/temp_root.rs"] mod temp_root; use self::temp_root::{test_temp_base, test_temp_root}; +const DEFAULT_COMMAND_DEADLINE: Duration = Duration::from_secs(30); +const DAEMON_CLEANUP_DEADLINE: Duration = Duration::from_secs(5); + pub(crate) struct E2eContext { _sandbox: TempDir, _runtime_sandbox: TempDir, @@ -21,6 +29,11 @@ pub(crate) struct E2eContext { data_dir: PathBuf, } +pub(crate) struct E2eOutput { + process: ProcessOutput, + runtime_dir: PathBuf, +} + impl E2eContext { pub(crate) fn new() -> io::Result { let test_temp_root = test_temp_root()?; @@ -56,12 +69,23 @@ impl E2eContext { }) } - pub(crate) fn run(&self, args: &[&str]) -> io::Result { - self.command().args(args).output() + pub(crate) fn run(&self, args: &[&str]) -> E2eOutput { + self.run_with_deadline(args, DEFAULT_COMMAND_DEADLINE) + } + + pub(crate) fn run_with_deadline(&self, args: &[&str], deadline: Duration) -> E2eOutput { + let mut command = self.command(); + command.args(args); + self.run_command(&mut command, deadline) + .unwrap_or_else(|diagnostic| panic!("{diagnostic}")) } fn command(&self) -> Command { - let mut command = Command::new(env!("CARGO_BIN_EXE_lsp-cli")); + self.command_for(env!("CARGO_BIN_EXE_lsp-cli")) + } + + fn command_for(&self, program: impl AsRef) -> Command { + let mut command = Command::new(program); command .env_clear() .env("HOME", &self.home) @@ -75,18 +99,162 @@ impl E2eContext { .current_dir(&self.workspace); command } + + fn run_command(&self, command: &mut Command, deadline: Duration) -> Result { + process::run(command, deadline) + .map(|process| E2eOutput { + process, + runtime_dir: self.runtime_dir.clone(), + }) + .map_err(|failure| failure.diagnostic(&runtime_state(&self.runtime_dir))) + } + + #[cfg(test)] + fn run_test_program( + &self, + program: impl AsRef, + args: &[&str], + deadline: Duration, + ) -> Result { + let mut command = self.command_for(program); + command.args(args); + self.run_command(&mut command, deadline) + } +} + +impl Drop for E2eContext { + fn drop(&mut self) { + let daemon_root = self.runtime_dir.join("lsp-cli"); + if !daemon_root.exists() { + return; + } + + // Detached daemons outlive command process groups, so the context must stop them explicitly. + let mut command = self.command(); + command.args(["stop-all", "--debug"]); + let cleanup = process::run(&mut command, DAEMON_CLEANUP_DEADLINE); + let diagnostic = match cleanup { + Ok(output) if output.status().success() => return, + Ok(output) => output.diagnostic( + "E2E daemon cleanup exited unsuccessfully", + &runtime_state(&self.runtime_dir), + ), + Err(failure) => failure.diagnostic(&runtime_state(&self.runtime_dir)), + }; + if std::thread::panicking() { + eprintln!("E2E daemon cleanup failed:\n{diagnostic}"); + } else { + panic!("E2E daemon cleanup failed:\n{diagnostic}"); + } + } +} + +impl E2eOutput { + pub(crate) fn assert_success(&self) { + assert!( + self.process.status().success(), + "{}", + self.diagnostic("lsp-cli exited unsuccessfully") + ); + } + + pub(crate) fn stdout_text(&self) -> &str { + std::str::from_utf8(self.process.stdout()).unwrap_or_else(|error| { + panic!( + "{}", + self.diagnostic(&format!("stdout is not valid UTF-8: {error}")) + ) + }) + } + + pub(crate) fn stderr_text(&self) -> &str { + std::str::from_utf8(self.process.stderr()).unwrap_or_else(|error| { + panic!( + "{}", + self.diagnostic(&format!("stderr is not valid UTF-8: {error}")) + ) + }) + } + + pub(crate) fn json(&self) -> T { + self.try_json() + .unwrap_or_else(|diagnostic| panic!("{diagnostic}")) + } + + fn diagnostic(&self, reason: &str) -> String { + self.process + .diagnostic(reason, &runtime_state(&self.runtime_dir)) + } + + fn try_json(&self) -> Result { + serde_json::from_slice(self.process.stdout()) + .map_err(|error| self.diagnostic(&format!("stdout is not valid JSON: {error}"))) + } +} + +fn runtime_state(runtime_dir: &std::path::Path) -> String { + let daemon_root = runtime_dir.join("lsp-cli"); + let entries = match fs::read_dir(&daemon_root) { + Ok(entries) => entries, + Err(error) if error.kind() == io::ErrorKind::NotFound => { + return format!("{} does not exist", daemon_root.display()); + } + Err(error) => return format!("failed to read {}: {error}", daemon_root.display()), + }; + let mut paths = entries + .map(|entry| match entry { + Ok(entry) => entry.path().display().to_string(), + Err(error) => format!(""), + }) + .collect::>(); + paths.sort(); + if paths.is_empty() { + format!("{} is empty", daemon_root.display()) + } else { + paths.join("\n") + } } #[cfg(test)] mod tests { use std::collections::BTreeMap; use std::ffi::OsString; + #[cfg(target_os = "linux")] + use std::path::Path; + #[cfg(target_os = "linux")] + use std::time::Instant; use super::*; + fn context() -> E2eContext { + E2eContext::new().expect("E2E context should initialize") + } + + fn run_shell( + context: &E2eContext, + script: &str, + deadline: Duration, + ) -> Result { + context.run_test_program("/bin/sh", &["-c", script], deadline) + } + + #[cfg(target_os = "linux")] + fn assert_recorded_process_is_gone(pid_file: &Path) { + let pid = fs::read_to_string(pid_file).expect("descendant PID should be recorded"); + let process = Path::new("/proc").join(&pid); + let reaping_deadline = Instant::now() + Duration::from_secs(1); + while process.exists() && Instant::now() < reaping_deadline { + std::thread::sleep(Duration::from_millis(10)); + } + assert!( + !process.exists(), + "descendant process {pid} survived group cleanup" + ); + } + #[test] fn command_isolated_from_ambient_process_state() { - let context = E2eContext::new().expect("E2E context should initialize"); + let context = context(); let command = context.command(); let actual = command .get_envs() @@ -122,7 +290,7 @@ mod tests { fn runtime_directory_has_room_for_daemon_socket_name() { use std::os::unix::net::UnixListener; - let context = E2eContext::new().expect("E2E context should initialize"); + let context = context(); let daemon_root = context.runtime_dir.join("lsp-cli"); fs::create_dir(&daemon_root).expect("daemon root should be created"); let socket_path = daemon_root.join(format!("{}-{}.sock", "s".repeat(32), "f".repeat(24))); @@ -133,4 +301,105 @@ mod tests { ) }); } + + #[test] + fn captures_large_stdout_and_stderr_without_deadlock() { + let context = context(); + let output = run_shell( + &context, + "i=0; while [ \"$i\" -lt 100000 ]; do printf o; printf e >&2; i=$((i + 1)); done", + Duration::from_secs(5), + ) + .expect("output fixture should finish"); + + assert_eq!(output.process.stdout().len(), 100_000); + assert_eq!(output.process.stderr().len(), 100_000); + } + + #[test] + fn deadline_kills_the_command_process_group() { + let context = context(); + let pid_file = context.workspace.join("descendant.pid"); + let script = format!( + "/bin/sleep 30 & child=$!; printf '%s' \"$child\" > {}; wait", + pid_file.display() + ); + let diagnostic = run_shell(&context, &script, Duration::from_millis(100)) + .err() + .expect("stalled fixture should exceed its deadline"); + + assert!(diagnostic.contains("process exceeded its deadline")); + assert!(diagnostic.contains("process group killed and reaped")); + #[cfg(target_os = "linux")] + assert_recorded_process_is_gone(&pid_file); + } + + #[test] + fn deadline_includes_output_pipes_held_by_descendants() { + let context = context(); + let pid_file = context.workspace.join("pipe-holder.pid"); + let script = format!( + "/bin/sleep 30 & child=$!; printf '%s' \"$child\" > {}", + pid_file.display() + ); + let diagnostic = run_shell(&context, &script, Duration::from_millis(100)) + .err() + .expect("inherited pipe should keep the process group beyond its deadline"); + + assert!(diagnostic.contains("remained open after the command deadline")); + assert!(diagnostic.contains("process group killed and reaped")); + #[cfg(target_os = "linux")] + assert_recorded_process_is_gone(&pid_file); + } + + #[test] + fn parses_json_output_into_requested_type() { + let context = context(); + let output = run_shell( + &context, + "printf '%s' '{\"answer\":42}'", + Duration::from_secs(1), + ) + .expect("JSON fixture should finish"); + let value: serde_json::Value = output.json(); + + assert_eq!(value, serde_json::json!({"answer": 42})); + } + + #[test] + fn invalid_json_reports_command_and_captured_output() { + let context = context(); + let output = run_shell(&context, "printf not-json", Duration::from_secs(1)) + .expect("invalid JSON fixture should finish"); + let diagnostic = output + .try_json::() + .expect_err("invalid JSON should be rejected"); + + assert!(diagnostic.contains("stdout is not valid JSON")); + assert!(diagnostic.contains("command: \"/bin/sh\" \"-c\"")); + assert!(diagnostic.contains("not-json")); + } + + #[test] + fn failed_command_diagnostic_includes_execution_context_and_output() { + let context = context(); + let output = run_shell( + &context, + "printf stdout-marker; printf stderr-marker >&2; exit 7", + Duration::from_secs(1), + ) + .expect("failure fixture should finish"); + let diagnostic = output.diagnostic("fixture failed"); + + assert!(diagnostic.contains("fixture failed")); + assert!(diagnostic.contains("command: \"/bin/sh\" \"-c\"")); + assert!(diagnostic.contains(&format!( + "working directory: {}", + context.workspace.display() + ))); + assert!(diagnostic.contains("status: exit status: 7")); + assert!(diagnostic.contains("stdout-marker")); + assert!(diagnostic.contains("stderr-marker")); + assert!(diagnostic.contains("runtime state:")); + } } diff --git a/tests/e2e/process.rs b/tests/e2e/process.rs new file mode 100644 index 0000000..3f3c97d --- /dev/null +++ b/tests/e2e/process.rs @@ -0,0 +1,397 @@ +use std::ffi::OsStr; +use std::io; +use std::path::PathBuf; +use std::process::{Command, ExitStatus, Stdio}; +use std::sync::mpsc::{self, Receiver, RecvTimeoutError}; +use std::thread::{self, JoinHandle}; +use std::time::{Duration, Instant}; + +use command_group::{CommandGroup as _, GroupChild}; +use wait_timeout::ChildExt as _; + +const OUTPUT_EXCERPT_LIMIT: usize = 16 * 1024; +const READER_CLEANUP_DEADLINE: Duration = Duration::from_secs(1); + +pub(crate) struct ProcessOutput { + details: ProcessDetails, + status: ExitStatus, +} + +pub(crate) struct ProcessFailure { + details: Box, + reason: String, +} + +struct ProcessDetails { + command: String, + cwd: PathBuf, + deadline: Duration, + elapsed: Duration, + stdout: Vec, + stderr: Vec, +} + +struct RunningGroup { + child: Option, + stdout_reader: OutputReader, + stderr_reader: OutputReader, +} + +struct OutputReader { + result: Option>>>, + thread: Option>, + output: Option>, +} + +pub(crate) fn run( + command: &mut Command, + deadline: Duration, +) -> Result { + let command_line = render_command(command); + let cwd = command + .get_current_dir() + .map_or_else(|| PathBuf::from(""), PathBuf::from); + let started = Instant::now(); + let absolute_deadline = started.checked_add(deadline).unwrap_or(started); + command + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + + let child = command.group_spawn().map_err(|error| ProcessFailure { + details: Box::new(ProcessDetails::empty( + command_line.clone(), + cwd.clone(), + deadline, + started.elapsed(), + )), + reason: format!("failed to start process group: {error}"), + })?; + let mut running = RunningGroup::new(child).map_err(|(mut child, error)| { + let cleanup = terminate_group(&mut child); + ProcessFailure { + details: Box::new(ProcessDetails::empty( + command_line.clone(), + cwd.clone(), + deadline, + started.elapsed(), + )), + reason: format!("failed to capture process output: {error}; {cleanup}"), + } + })?; + + // `wait-timeout` extends `Child`, so borrow the group leader only for the bounded wait. + // If it expires, `GroupChild::kill` still terminates the complete process group. + let wait_result = running.child_mut().and_then(|child| { + child + .inner() + .wait_timeout(deadline.saturating_sub(started.elapsed())) + .map_err(|error| format!("failed while waiting for process group: {error}")) + }); + + match wait_result { + Ok(Some(status)) => { + let (stdout, stderr) = match running.finish_readers_before(absolute_deadline) { + Ok(output) => output, + Err(reason) => { + let cleanup = running.terminate(); + let (stdout, stderr) = running.finish_readers_after_termination(); + return Err(ProcessFailure { + details: Box::new(ProcessDetails { + command: command_line, + cwd, + deadline, + elapsed: started.elapsed(), + stdout, + stderr, + }), + reason: format!("{reason}; {cleanup}"), + }); + } + }; + running.disarm_child(); + Ok(ProcessOutput { + details: ProcessDetails { + command: command_line, + cwd, + deadline, + elapsed: started.elapsed(), + stdout, + stderr, + }, + status, + }) + } + Ok(None) => { + let cleanup = running.terminate(); + let (stdout, stderr) = running.finish_readers_after_termination(); + Err(ProcessFailure { + details: Box::new(ProcessDetails { + command: command_line, + cwd, + deadline, + elapsed: started.elapsed(), + stdout, + stderr, + }), + reason: format!("process exceeded its deadline; {cleanup}"), + }) + } + Err(reason) => { + let cleanup = running.terminate(); + let (stdout, stderr) = running.finish_readers_after_termination(); + Err(ProcessFailure { + details: Box::new(ProcessDetails { + command: command_line, + cwd, + deadline, + elapsed: started.elapsed(), + stdout, + stderr, + }), + reason: format!("{reason}; {cleanup}"), + }) + } + } +} + +impl ProcessOutput { + pub(crate) fn status(&self) -> ExitStatus { + self.status + } + + pub(crate) fn stdout(&self) -> &[u8] { + &self.details.stdout + } + + pub(crate) fn stderr(&self) -> &[u8] { + &self.details.stderr + } + + pub(crate) fn diagnostic(&self, reason: &str, runtime_state: &str) -> String { + self.details + .diagnostic(reason, Some(self.status), runtime_state) + } +} + +impl ProcessFailure { + pub(crate) fn diagnostic(&self, runtime_state: &str) -> String { + self.details.diagnostic(&self.reason, None, runtime_state) + } +} + +impl ProcessDetails { + fn empty(command: String, cwd: PathBuf, deadline: Duration, elapsed: Duration) -> Self { + Self { + command, + cwd, + deadline, + elapsed, + stdout: Vec::new(), + stderr: Vec::new(), + } + } + + fn diagnostic(&self, reason: &str, status: Option, runtime_state: &str) -> String { + let status = status.map_or_else(|| "not available".to_string(), |value| value.to_string()); + format!( + "{reason}\ncommand: {}\nworking directory: {}\nstatus: {status}\nelapsed: {:?}\ndeadline: {:?}\nruntime state:\n{runtime_state}\nstdout:\n{}\nstderr:\n{}", + self.command, + self.cwd.display(), + self.elapsed, + self.deadline, + output_excerpt(&self.stdout), + output_excerpt(&self.stderr), + ) + } +} + +impl RunningGroup { + fn new(mut child: GroupChild) -> Result { + let Some(stdout) = child.inner().stdout.take() else { + return Err((child, io::Error::other("stdout pipe was not created"))); + }; + let Some(stderr) = child.inner().stderr.take() else { + return Err((child, io::Error::other("stderr pipe was not created"))); + }; + + Ok(Self { + child: Some(child), + stdout_reader: OutputReader::new(stdout), + stderr_reader: OutputReader::new(stderr), + }) + } + + fn child_mut(&mut self) -> Result<&mut GroupChild, String> { + self.child + .as_mut() + .ok_or_else(|| "running process group lost its child handle".to_string()) + } + + fn disarm_child(&mut self) { + let _child = self.child.take(); + } + + fn terminate(&mut self) -> String { + let Some(mut child) = self.child.take() else { + return "process group already reaped".to_string(); + }; + terminate_group(&mut child) + } + + fn finish_readers_before(&mut self, deadline: Instant) -> Result<(Vec, Vec), String> { + let stdout = self.stdout_reader.finish_before(deadline, "stdout"); + let stderr = self.stderr_reader.finish_before(deadline, "stderr"); + match (stdout, stderr) { + (Ok(()), Ok(())) => Ok(self.take_output()), + (Err(stdout), Ok(())) => Err(stdout), + (Ok(()), Err(stderr)) => Err(stderr), + (Err(stdout), Err(stderr)) => Err(format!("{stdout}; {stderr}")), + } + } + + fn finish_readers_after_termination(&mut self) -> (Vec, Vec) { + let deadline = Instant::now() + .checked_add(READER_CLEANUP_DEADLINE) + .unwrap_or_else(Instant::now); + let error = self.finish_readers_before(deadline).err(); + let (stdout, mut stderr) = self.take_output(); + if let Some(error) = error { + stderr.extend_from_slice(format!("\nfailed to capture output: {error}").as_bytes()); + } + (stdout, stderr) + } + + fn take_output(&mut self) -> (Vec, Vec) { + ( + self.stdout_reader.take_output(), + self.stderr_reader.take_output(), + ) + } +} + +impl Drop for RunningGroup { + fn drop(&mut self) { + // This guard is necessary because test assertions may unwind while descendants are alive. + if self.child.is_some() { + let _cleanup_result = self.terminate(); + } + let _captured_output = self.finish_readers_after_termination(); + } +} + +impl OutputReader { + fn new(reader: impl io::Read + Send + 'static) -> Self { + let (sender, result) = mpsc::channel(); + let thread = thread::spawn(move || { + let _send_result = sender.send(read_all(reader)); + }); + Self { + result: Some(result), + thread: Some(thread), + output: None, + } + } + + fn finish_before(&mut self, deadline: Instant, stream: &str) -> Result<(), String> { + if self.output.is_some() { + return Ok(()); + } + let Some(result) = self.result.as_ref() else { + return Err(format!("{stream} capture result is unavailable")); + }; + let remaining = deadline.saturating_duration_since(Instant::now()); + let output = match result.recv_timeout(remaining) { + Ok(output) => output.map_err(|error| format!("failed to read {stream}: {error}"))?, + Err(RecvTimeoutError::Timeout) => { + return Err(format!("{stream} remained open after the command deadline")); + } + Err(RecvTimeoutError::Disconnected) => { + return Err(format!("{stream} capture thread exited without a result")); + } + }; + self.result = None; + if let Some(thread) = self.thread.take() + && thread.join().is_err() + { + return Err(format!("{stream} capture thread panicked")); + } + self.output = Some(output); + Ok(()) + } + + fn take_output(&mut self) -> Vec { + self.output.take().unwrap_or_default() + } +} + +fn terminate_group(child: &mut GroupChild) -> String { + let kill_result = child.kill(); + let wait_result = child.wait(); + match (kill_result, wait_result) { + (Ok(()), Ok(status)) => format!("process group killed and reaped with {status}"), + (Err(kill_error), Ok(status)) if kill_error.kind() == io::ErrorKind::InvalidInput => { + format!("process group exited during cleanup with {status}") + } + (Err(kill_error), Ok(status)) => { + format!("process group reaped with {status}, but kill failed: {kill_error}") + } + (Ok(()), Err(wait_error)) => { + format!("process group was killed but could not be reaped: {wait_error}") + } + (Err(kill_error), Err(wait_error)) => { + format!("process group could not be killed ({kill_error}) or reaped ({wait_error})") + } + } +} + +fn read_all(mut reader: impl io::Read) -> io::Result> { + let mut output = Vec::new(); + reader.read_to_end(&mut output)?; + Ok(output) +} + +fn render_command(command: &Command) -> String { + std::iter::once(command.get_program()) + .chain(command.get_args()) + .map(render_argument) + .collect::>() + .join(" ") +} + +fn render_argument(argument: &OsStr) -> String { + format!("{argument:?}") +} + +fn output_excerpt(output: &[u8]) -> String { + let omitted = output.len().saturating_sub(OUTPUT_EXCERPT_LIMIT); + let excerpt = output + .get(omitted..) + .map_or(output, |bounded_output| bounded_output); + if omitted == 0 { + String::from_utf8_lossy(excerpt).into_owned() + } else { + format!( + "<{} earlier bytes omitted>\n{}", + omitted, + String::from_utf8_lossy(excerpt) + ) + } +} + +#[cfg(test)] +mod tests { + use super::{OUTPUT_EXCERPT_LIMIT, output_excerpt}; + + #[test] + fn diagnostic_output_keeps_only_a_bounded_tail() { + let output = vec![b'x'; OUTPUT_EXCERPT_LIMIT + 7]; + let excerpt = output_excerpt(&output); + + assert!(excerpt.starts_with("<7 earlier bytes omitted>\n")); + assert_eq!( + excerpt.bytes().filter(|byte| *byte == b'x').count(), + OUTPUT_EXCERPT_LIMIT + ); + } +} From d69f466f756b66226adb85c718ccc9b498f5529f Mon Sep 17 00:00:00 2001 From: Vasily Kulikov Date: Sat, 5 Sep 2026 18:50:05 +0300 Subject: [PATCH 08/18] Add manifest-driven real-server E2E proof --- .github/workflows/ci.yml | 7 + ...e-real-server-cases-are-manifest-driven.md | 6 + ...-config-id-differs-from-cli-server-name.md | 6 + Cargo.lock | 7 + Cargo.toml | 1 + E2E_TESTS.md | 20 +- tests/e2e.rs | 2 + tests/e2e/cases.yaml | 17 ++ tests/e2e/harness.rs | 133 ++++++++- tests/e2e/manifest.rs | 262 +++++++++++++----- tests/e2e/manifest_tests.rs | 112 ++++++++ tests/e2e/real_servers.rs | 159 +++++++++++ 12 files changed, 646 insertions(+), 86 deletions(-) create mode 100644 .memory/unprocessed/e2e-real-server-cases-are-manifest-driven.md create mode 100644 .memory/unprocessed/lsp-config-id-differs-from-cli-server-name.md create mode 100644 tests/e2e/manifest_tests.rs create mode 100644 tests/e2e/real_servers.rs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fb1abf0..73312ca 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -44,6 +44,13 @@ jobs: - name: Run tests run: cargo test --locked + - name: Run real-server E2E smoke tests + if: matrix.rust == 'stable' + run: >- + cargo test --locked --test e2e + real_servers::manifest_real_server_smoke_cases + -- --ignored --exact --nocapture + - name: Run clippy if: matrix.rust == 'stable' run: cargo clippy --all-targets --all-features -- -D warnings diff --git a/.memory/unprocessed/e2e-real-server-cases-are-manifest-driven.md b/.memory/unprocessed/e2e-real-server-cases-are-manifest-driven.md new file mode 100644 index 0000000..7025e49 --- /dev/null +++ b/.memory/unprocessed/e2e-real-server-cases-are-manifest-driven.md @@ -0,0 +1,6 @@ +# Real-server E2E cases are manifest-driven + +The first proposed real-server proof was shaped as a Rust/rust-analyzer-specific test. The user +corrected that direction: the harness must extend to any LSP server and must not hardcode one +server. Keep language IDs, server config IDs, expected symbols, and runtime prerequisites in the +E2E manifest. Runner code may dispatch only on generic operation and provisioning types. diff --git a/.memory/unprocessed/lsp-config-id-differs-from-cli-server-name.md b/.memory/unprocessed/lsp-config-id-differs-from-cli-server-name.md new file mode 100644 index 0000000..6d77bce --- /dev/null +++ b/.memory/unprocessed/lsp-config-id-differs-from-cli-server-name.md @@ -0,0 +1,6 @@ +# LSP config ID differs from CLI server name + +The E2E manifest identifies an LSP configuration by the YAML filename stem, such as +`rust_analyzer`. The `--lsp` CLI option does not accept that ID; it selects the configured +user-visible `name`, such as `rust-analyzer`. Generic E2E code must load the name from the selected +LSP YAML instead of passing the manifest ID or duplicating a server-specific name in test code. diff --git a/Cargo.lock b/Cargo.lock index e577c1a..6afd233 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -334,6 +334,12 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "fs_extra" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" + [[package]] name = "futures-channel" version = "0.3.32" @@ -774,6 +780,7 @@ dependencies = [ "clap_complete", "command-group", "flate2", + "fs_extra", "humantime", "lsp-types", "regex", diff --git a/Cargo.toml b/Cargo.toml index 4faf213..681aa0c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -32,6 +32,7 @@ zip = { version = "8.6.0", default-features = false, features = ["deflate"] } [dev-dependencies] command-group = "5.0.1" +fs_extra = "1.3.0" wait-timeout = "0.2.1" [lints] diff --git a/E2E_TESTS.md b/E2E_TESTS.md index 6d52154..b84338e 100644 --- a/E2E_TESTS.md +++ b/E2E_TESTS.md @@ -221,6 +221,24 @@ against the pinned data without requiring unfinished matrix entries. Phase 4 add entries and switches it to `coverage: complete`; complete mode enforces every detectable language and compatible pair. +### Extending the manifest + +To cover an existing filetype, add its small project under `playground/`, declare it once under +`languages`, then add a `pairs` entry for each compatible server. To introduce a genuinely new +filetype or server, first add its YAML config and commit it in the `data` submodule, then update the +submodule revision and the E2E manifest in this repository. + +Pair entries use the LSP YAML filename stem as their stable config ID. The test runner loads the +configured user-visible server name for `--lsp`; do not duplicate it in the manifest. Each optional +`smoke` block declares a generic provisioning method, query kind, semantic expectations, runtime +host programs, and deadlines. Language-specific prerequisites and expected symbols belong in YAML, +not in the Rust runner. The first provisioning method is `download`; add other mechanisms as typed +methods when needed instead of branching on server names. + +In `coverage: complete` mode, manifest validation makes a new detectable filetype or compatible +filetype/server relationship fail until its project and pair are declared. Partial mode intentionally +allows the matrix to grow incrementally. + ## Special command strategies ### `run` @@ -363,7 +381,7 @@ that class of defect easier to diagnose. - [x] Add the initial manifest schema and validation. - [x] Isolate all environment and runtime state. - [x] Implement deadlines, cleanup guards, JSON helpers, and useful failure diagnostics. -- [ ] Prove the harness with Rust/rust-analyzer. +- [x] Prove the harness with Rust/rust-analyzer. - [ ] Cover all 24 subcommand paths with either a real server or a deterministic local fixture. ### Phase 2: projects diff --git a/tests/e2e.rs b/tests/e2e.rs index b518555..ec41815 100644 --- a/tests/e2e.rs +++ b/tests/e2e.rs @@ -11,3 +11,5 @@ mod harness; mod manifest; #[path = "e2e/process.rs"] mod process; +#[path = "e2e/real_servers.rs"] +mod real_servers; diff --git a/tests/e2e/cases.yaml b/tests/e2e/cases.yaml index 3865b6f..0a0749f 100644 --- a/tests/e2e/cases.yaml +++ b/tests/e2e/cases.yaml @@ -9,3 +9,20 @@ languages: pairs: - language: rust server: rust_analyzer + smoke: + provision: + method: download + query: + kind: list-symbols + expected-names: + - Order + - OrderItem + - sample_order + - format_order + host-programs: + - name: cargo + resolve: [rustup, which, cargo] + - name: rustc + resolve: [rustup, which, rustc] + lsp-timeout-seconds: 30 + deadline-seconds: 180 diff --git a/tests/e2e/harness.rs b/tests/e2e/harness.rs index 401ec8c..0a71ca9 100644 --- a/tests/e2e/harness.rs +++ b/tests/e2e/harness.rs @@ -1,10 +1,11 @@ use std::ffi::OsStr; use std::fs; use std::io; -use std::path::PathBuf; +use std::path::{Path, PathBuf}; use std::process::Command; use std::time::Duration; +use fs_extra::dir::CopyOptions; use serde::de::DeserializeOwned; use tempfile::TempDir; @@ -26,6 +27,7 @@ pub(crate) struct E2eContext { runtime_dir: PathBuf, workspace: PathBuf, bin_dir: PathBuf, + build_dir: PathBuf, data_dir: PathBuf, } @@ -51,9 +53,10 @@ impl E2eContext { let config_home = sandbox.path().join("config"); let workspace = sandbox.path().join("workspace"); let bin_dir = sandbox.path().join("bin"); + let build_dir = sandbox.path().join("build"); let runtime_dir = runtime_sandbox.path().to_path_buf(); - for directory in [&home, &config_home, &workspace, &bin_dir] { + for directory in [&home, &config_home, &workspace, &bin_dir, &build_dir] { fs::create_dir(directory)?; } @@ -65,19 +68,121 @@ impl E2eContext { runtime_dir, workspace, bin_dir, + build_dir, data_dir: PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("data"), }) } + pub(crate) fn copy_project(&self, source: &Path) -> Result<(), String> { + let options = CopyOptions::new().content_only(true); + fs_extra::dir::copy(source, &self.workspace, &options) + .map(|_copied_bytes| ()) + .map_err(|error| { + format!( + "failed to copy E2E project {} into {}: {error}", + source.display(), + self.workspace.display() + ) + }) + } + + pub(crate) fn stage_host_program( + &self, + name: &str, + resolver: &[String], + deadline: Duration, + ) -> Result<(), String> { + let (program, args) = resolver + .split_first() + .ok_or_else(|| format!("host program {name:?} has no resolver command"))?; + let mut command = Command::new(program); + command.args(args).current_dir(env!("CARGO_MANIFEST_DIR")); + let output = process::run(&mut command, deadline) + .map_err(|failure| failure.diagnostic(&runtime_state(&self.runtime_dir)))?; + if !output.status().success() { + return Err(output.diagnostic( + &format!("failed to resolve required host program {name:?}"), + &runtime_state(&self.runtime_dir), + )); + } + let stdout = std::str::from_utf8(output.stdout()).map_err(|error| { + output.diagnostic( + &format!("resolver output for host program {name:?} is not UTF-8: {error}"), + &runtime_state(&self.runtime_dir), + ) + })?; + let paths = stdout + .lines() + .filter(|line| !line.is_empty()) + .collect::>(); + let [resolved] = paths.as_slice() else { + return Err(output.diagnostic( + &format!( + "resolver for host program {name:?} must print exactly one non-empty path" + ), + &runtime_state(&self.runtime_dir), + )); + }; + let resolved = Path::new(resolved); + if !resolved.is_file() { + return Err(output.diagnostic( + &format!( + "resolver for host program {name:?} returned {}, which is not a file", + resolved.display() + ), + &runtime_state(&self.runtime_dir), + )); + } + let resolved = resolved.canonicalize().map_err(|error| { + output.diagnostic( + &format!( + "failed to resolve host program {name:?} path {}: {error}", + resolved.display() + ), + &runtime_state(&self.runtime_dir), + ) + })?; + + self.link_host_program(&resolved, &self.bin_dir.join(name)) + .map_err(|error| { + format!( + "failed to stage host program {name:?} from {}: {error}", + resolved.display() + ) + }) + } + + #[cfg(unix)] + fn link_host_program(&self, source: &Path, destination: &Path) -> io::Result<()> { + std::os::unix::fs::symlink(source, destination) + } + + #[cfg(not(unix))] + fn link_host_program(&self, source: &Path, destination: &Path) -> io::Result<()> { + fs::copy(source, destination).map(|_copied_bytes| ()) + } + + pub(crate) fn home(&self) -> &Path { + &self.home + } + pub(crate) fn run(&self, args: &[&str]) -> E2eOutput { self.run_with_deadline(args, DEFAULT_COMMAND_DEADLINE) } pub(crate) fn run_with_deadline(&self, args: &[&str], deadline: Duration) -> E2eOutput { + self.try_run_with_deadline(args, deadline) + .unwrap_or_else(|diagnostic| panic!("{diagnostic}")) + } + + pub(crate) fn try_run_with_deadline( + &self, + args: &[&str], + deadline: Duration, + ) -> Result { let mut command = self.command(); command.args(args); self.run_command(&mut command, deadline) - .unwrap_or_else(|diagnostic| panic!("{diagnostic}")) } fn command(&self) -> Command { @@ -89,6 +194,7 @@ impl E2eContext { command .env_clear() .env("HOME", &self.home) + .env("CARGO_TARGET_DIR", &self.build_dir) .env("XDG_CONFIG_HOME", &self.config_home) .env("XDG_RUNTIME_DIR", &self.runtime_dir) .env("LSP_DATA", &self.data_dir) @@ -151,11 +257,16 @@ impl Drop for E2eContext { impl E2eOutput { pub(crate) fn assert_success(&self) { - assert!( - self.process.status().success(), - "{}", - self.diagnostic("lsp-cli exited unsuccessfully") - ); + self.ensure_success() + .unwrap_or_else(|diagnostic| panic!("{diagnostic}")); + } + + pub(crate) fn ensure_success(&self) -> Result<(), String> { + if self.process.status().success() { + Ok(()) + } else { + Err(self.diagnostic("lsp-cli exited unsuccessfully")) + } } pub(crate) fn stdout_text(&self) -> &str { @@ -186,7 +297,7 @@ impl E2eOutput { .diagnostic(reason, &runtime_state(&self.runtime_dir)) } - fn try_json(&self) -> Result { + pub(crate) fn try_json(&self) -> Result { serde_json::from_slice(self.process.stdout()) .map_err(|error| self.diagnostic(&format!("stdout is not valid JSON: {error}"))) } @@ -261,6 +372,10 @@ mod tests { .map(|(name, value)| (name.to_os_string(), value.map(OsString::from))) .collect::>(); let expected = [ + ( + "CARGO_TARGET_DIR", + context.build_dir.as_os_str().to_os_string(), + ), ("HOME", context.home.as_os_str().to_os_string()), ("LANG", OsString::from("C")), ("LC_ALL", OsString::from("C")), diff --git a/tests/e2e/manifest.rs b/tests/e2e/manifest.rs index c404ddb..753f675 100644 --- a/tests/e2e/manifest.rs +++ b/tests/e2e/manifest.rs @@ -9,7 +9,7 @@ const MANIFEST_SCHEMA_VERSION: u32 = 1; #[derive(Clone, Debug, Deserialize)] #[serde(rename_all = "kebab-case", deny_unknown_fields)] -struct Manifest { +pub(crate) struct Manifest { schema_version: u32, coverage: Coverage, languages: Vec, @@ -47,11 +47,67 @@ impl ProjectKind { } } -#[derive(Clone, Debug, Deserialize, Eq, Ord, PartialEq, PartialOrd)] +#[derive(Clone, Debug, Deserialize)] #[serde(rename_all = "kebab-case", deny_unknown_fields)] struct PairCase { language: String, server: String, + smoke: Option, +} + +#[derive(Clone, Debug, Eq, Ord, PartialEq, PartialOrd)] +struct PairKey { + language: String, + server: String, +} + +#[derive(Clone, Debug, Deserialize)] +#[serde(rename_all = "kebab-case", deny_unknown_fields)] +struct SmokeCase { + provision: Provision, + query: Query, + expected_names: Vec, + #[serde(default)] + host_programs: Vec, + lsp_timeout_seconds: u64, + deadline_seconds: u64, +} + +#[derive(Clone, Copy, Debug, Deserialize)] +#[serde(rename_all = "kebab-case", deny_unknown_fields)] +struct Provision { + method: ProvisionMethod, +} + +#[derive(Clone, Copy, Debug, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub(crate) enum ProvisionMethod { + Download, +} + +#[derive(Clone, Copy, Debug, Deserialize)] +#[serde(rename_all = "kebab-case", deny_unknown_fields)] +struct Query { + kind: QueryKind, +} + +#[derive(Clone, Copy, Debug, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub(crate) enum QueryKind { + ListSymbols, +} + +#[derive(Clone, Debug, Deserialize)] +#[serde(rename_all = "kebab-case", deny_unknown_fields)] +struct HostProgram { + name: String, + resolve: Vec, +} + +pub(crate) struct RealServerCase<'a> { + language: &'a LanguageCase, + pair: &'a PairCase, + smoke: &'a SmokeCase, } #[derive(Deserialize)] @@ -72,6 +128,7 @@ impl FiletypeConfig { struct LspConfig { #[serde(default)] filetypes: Vec, + name: String, } impl Manifest { @@ -104,6 +161,27 @@ impl Manifest { Ok(()) } + pub(crate) fn load_validated(repository: &Path) -> Result { + let manifest = Self::load()?; + manifest.validate(repository)?; + Ok(manifest) + } + + pub(crate) fn real_server_smoke_cases(&self) -> impl Iterator> { + self.pairs.iter().filter_map(|pair| { + let smoke = pair.smoke.as_ref()?; + let language = self + .languages + .iter() + .find(|language| language.id == pair.language)?; + Some(RealServerCase { + language, + pair, + smoke, + }) + }) + } + fn validate_languages( &self, repository: &Path, @@ -136,7 +214,7 @@ impl Manifest { &self, data: &Path, declared_languages: &BTreeSet, - ) -> Result, String> { + ) -> Result, String> { let mut declared = BTreeSet::new(); for pair in &self.pairs { validate_config_id("server", &pair.server)?; @@ -146,7 +224,7 @@ impl Manifest { pair.language, pair.server )); } - if !declared.insert(pair.clone()) { + if !declared.insert(pair.key()) { return Err(format!( "E2E manifest declares pair {}/{} more than once", pair.language, pair.server @@ -161,6 +239,9 @@ impl Manifest { pair.server, pair.language )); } + if let Some(smoke) = &pair.smoke { + smoke.validate(pair)?; + } } Ok(declared) } @@ -168,7 +249,7 @@ impl Manifest { fn validate_complete_coverage( data: &Path, declared_languages: &BTreeSet, - declared_pairs: &BTreeSet, + declared_pairs: &BTreeSet, ) -> Result<(), String> { let detectable = detectable_languages(data)?; let missing_languages = detectable @@ -197,6 +278,101 @@ impl Manifest { } } +impl PairCase { + fn key(&self) -> PairKey { + PairKey { + language: self.language.clone(), + server: self.server.clone(), + } + } +} + +impl SmokeCase { + fn validate(&self, pair: &PairCase) -> Result<(), String> { + let label = format!("{}/{}", pair.language, pair.server); + if self.expected_names.is_empty() || self.expected_names.iter().any(String::is_empty) { + return Err(format!( + "E2E smoke case {label} must declare non-empty expected names" + )); + } + if self.lsp_timeout_seconds == 0 || self.deadline_seconds == 0 { + return Err(format!("E2E smoke case {label} deadlines must be positive")); + } + if self.deadline_seconds < self.lsp_timeout_seconds { + return Err(format!( + "E2E smoke case {label} deadline must not be shorter than its LSP timeout" + )); + } + + let mut names = BTreeSet::new(); + for program in &self.host_programs { + validate_config_id("host program", &program.name)?; + if !names.insert(&program.name) { + return Err(format!( + "E2E smoke case {label} declares host program {:?} more than once", + program.name + )); + } + if program.resolve.is_empty() || program.resolve.iter().any(String::is_empty) { + return Err(format!( + "E2E host program {:?} for {label} must have a non-empty resolver command", + program.name + )); + } + } + Ok(()) + } +} + +impl RealServerCase<'_> { + pub(crate) fn label(&self) -> String { + format!("{}/{}", self.pair.language, self.pair.server) + } + + pub(crate) fn language(&self) -> &str { + &self.language.id + } + + pub(crate) fn server_name(&self, repository: &Path) -> Result { + let path = repository + .join("data/lsp") + .join(format!("{}.yaml", self.pair.server)); + let config: LspConfig = read_yaml(&path)?; + Ok(config.name) + } + + pub(crate) fn project(&self) -> &Path { + &self.language.project + } + + pub(crate) fn provision_method(&self) -> ProvisionMethod { + self.smoke.provision.method + } + + pub(crate) fn query_kind(&self) -> QueryKind { + self.smoke.query.kind + } + + pub(crate) fn expected_names(&self) -> &[String] { + &self.smoke.expected_names + } + + pub(crate) fn host_programs(&self) -> impl Iterator { + self.smoke + .host_programs + .iter() + .map(|program| (program.name.as_str(), program.resolve.as_slice())) + } + + pub(crate) fn lsp_timeout_seconds(&self) -> u64 { + self.smoke.lsp_timeout_seconds + } + + pub(crate) fn deadline_seconds(&self) -> u64 { + self.smoke.deadline_seconds + } +} + impl LanguageCase { fn validate_project(&self, repository: &Path) -> Result<(), String> { if self.project.is_absolute() @@ -250,14 +426,14 @@ fn detectable_languages(data: &Path) -> Result, String> { fn compatible_pairs( data: &Path, detectable: &BTreeSet, -) -> Result, String> { +) -> Result, String> { let mut pairs = BTreeSet::new(); for path in yaml_paths(&data.join("lsp"))? { let config: LspConfig = read_yaml(&path)?; let server = file_stem(&path)?; for language in config.filetypes { if detectable.contains(&language) { - pairs.insert(PairCase { + pairs.insert(PairKey { language, server: server.clone(), }); @@ -309,72 +485,6 @@ fn validate_config_id(kind: &str, value: &str) -> Result<(), String> { Ok(()) } -fn repository_root() -> PathBuf { - PathBuf::from(env!("CARGO_MANIFEST_DIR")) -} - -#[test] -fn partial_manifest_matches_pinned_data() { - Manifest::load() - .expect("E2E manifest should parse") - .validate(&repository_root()) - .expect("E2E manifest should be valid"); -} - -#[test] -fn complete_mode_rejects_the_partial_matrix() { - let mut manifest = Manifest::load().expect("E2E manifest should parse"); - manifest.coverage = Coverage::Complete; - - let error = manifest - .validate(&repository_root()) - .expect_err("partial matrix should not satisfy complete coverage"); - - assert!(error.contains("complete E2E manifest is missing languages")); -} - -#[test] -fn complete_mode_rejects_missing_server_pairs() { - let data = repository_root().join("data"); - let detectable = detectable_languages(&data).expect("filetype configs should load"); - let declared_pairs = BTreeSet::from([PairCase { - language: "rust".to_string(), - server: "rust_analyzer".to_string(), - }]); - - let error = Manifest::validate_complete_coverage(&data, &detectable, &declared_pairs) - .expect_err("partial server matrix should not satisfy complete coverage"); - - assert!(error.contains("complete E2E manifest is missing pairs")); -} - -#[test] -fn manifest_rejects_unknown_fields() { - let error = serde_yaml::from_str::( - "schema-version: 1\ncoverage: partial\nlanguages: []\npairs: []\nunknown: true\n", - ) - .expect_err("unknown manifest fields should fail"); - - assert!(error.to_string().contains("unknown field `unknown`")); -} - -#[test] -fn manifest_rejects_config_path_traversal() { - let mut manifest = Manifest::load().expect("E2E manifest should parse"); - manifest - .languages - .first_mut() - .expect("manifest should contain a language") - .id = "../rust".to_string(); - manifest - .pairs - .first_mut() - .expect("manifest should contain a pair") - .language = "../rust".to_string(); - - let error = manifest - .validate(&repository_root()) - .expect_err("config path traversal should fail"); - - assert!(error.contains("must be one normalized path component")); -} +#[cfg(test)] +#[path = "manifest_tests.rs"] +mod tests; diff --git a/tests/e2e/manifest_tests.rs b/tests/e2e/manifest_tests.rs new file mode 100644 index 0000000..f8f1597 --- /dev/null +++ b/tests/e2e/manifest_tests.rs @@ -0,0 +1,112 @@ +use super::*; + +fn repository_root() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")) +} + +fn first_smoke(manifest: &mut Manifest) -> &mut SmokeCase { + manifest + .pairs + .first_mut() + .expect("manifest should contain a pair") + .smoke + .as_mut() + .expect("first pair should have a smoke case") +} + +#[test] +fn partial_manifest_matches_pinned_data() { + Manifest::load() + .expect("E2E manifest should parse") + .validate(&repository_root()) + .expect("E2E manifest should be valid"); +} + +#[test] +fn complete_mode_rejects_the_partial_matrix() { + let mut manifest = Manifest::load().expect("E2E manifest should parse"); + manifest.coverage = Coverage::Complete; + + let error = manifest + .validate(&repository_root()) + .expect_err("partial matrix should not satisfy complete coverage"); + + assert!(error.contains("complete E2E manifest is missing languages")); +} + +#[test] +fn complete_mode_rejects_missing_server_pairs() { + let data = repository_root().join("data"); + let detectable = detectable_languages(&data).expect("filetype configs should load"); + let declared_pairs = BTreeSet::from([PairKey { + language: "rust".to_string(), + server: "rust_analyzer".to_string(), + }]); + + let error = Manifest::validate_complete_coverage(&data, &detectable, &declared_pairs) + .expect_err("partial server matrix should not satisfy complete coverage"); + + assert!(error.contains("complete E2E manifest is missing pairs")); +} + +#[test] +fn manifest_rejects_unknown_fields() { + let error = serde_yaml::from_str::( + "schema-version: 1\ncoverage: partial\nlanguages: []\npairs: []\nunknown: true\n", + ) + .expect_err("unknown manifest fields should fail"); + + assert!(error.to_string().contains("unknown field `unknown`")); +} + +#[test] +fn manifest_rejects_config_path_traversal() { + let mut manifest = Manifest::load().expect("E2E manifest should parse"); + manifest + .languages + .first_mut() + .expect("manifest should contain a language") + .id = "../rust".to_string(); + manifest + .pairs + .first_mut() + .expect("manifest should contain a pair") + .language = "../rust".to_string(); + + let error = manifest + .validate(&repository_root()) + .expect_err("config path traversal should fail"); + + assert!(error.contains("must be one normalized path component")); +} + +#[test] +fn manifest_rejects_invalid_smoke_deadlines() { + let mut manifest = Manifest::load().expect("E2E manifest should parse"); + let smoke = first_smoke(&mut manifest); + smoke.deadline_seconds = smoke.lsp_timeout_seconds.saturating_sub(1); + + let error = manifest + .validate(&repository_root()) + .expect_err("short overall deadline should fail"); + + assert!(error.contains("deadline must not be shorter")); +} + +#[test] +fn manifest_rejects_duplicate_host_programs() { + let mut manifest = Manifest::load().expect("E2E manifest should parse"); + let smoke = first_smoke(&mut manifest); + let duplicate = smoke + .host_programs + .first() + .expect("smoke case should have a host program") + .clone(); + smoke.host_programs.push(duplicate); + + let error = manifest + .validate(&repository_root()) + .expect_err("duplicate host programs should fail"); + + assert!(error.contains("more than once")); +} diff --git a/tests/e2e/real_servers.rs b/tests/e2e/real_servers.rs new file mode 100644 index 0000000..b97977b --- /dev/null +++ b/tests/e2e/real_servers.rs @@ -0,0 +1,159 @@ +use std::collections::BTreeSet; +use std::path::{Path, PathBuf}; +use std::time::{Duration, Instant}; + +use serde::Deserialize; + +use crate::harness::E2eContext; +use crate::manifest::{Manifest, ProvisionMethod, QueryKind, RealServerCase}; + +struct RealServerTest<'a> { + case: RealServerCase<'a>, + repository: &'a Path, +} + +#[derive(Deserialize)] +struct ListSymbolsOutput { + detected: BTreeSet, + server: ServerOutput, + matches: Vec, +} + +#[derive(Deserialize)] +struct ServerOutput { + command: Vec, +} + +#[derive(Deserialize)] +struct SymbolOutput { + name: String, +} + +impl<'a> RealServerTest<'a> { + fn new(case: RealServerCase<'a>, repository: &'a Path) -> Self { + Self { case, repository } + } + + fn run(self) -> Result<(), String> { + self.run_inner() + .map_err(|error| format!("E2E case {} failed:\n{error}", self.case.label())) + } + + fn run_inner(&self) -> Result<(), String> { + let started = Instant::now(); + let deadline = Duration::from_secs(self.case.deadline_seconds()); + let context = E2eContext::new() + .map_err(|error| format!("failed to create an isolated E2E context: {error}"))?; + context.copy_project(&self.repository.join(self.case.project()))?; + + for (name, resolver) in self.case.host_programs() { + context.stage_host_program(name, resolver, remaining(started, deadline)?)?; + } + + let server_name = self.case.server_name(self.repository)?; + let args = self.command_args(&server_name); + let arg_refs = args.iter().map(String::as_str).collect::>(); + let output = context.try_run_with_deadline(&arg_refs, remaining(started, deadline)?)?; + output.ensure_success()?; + + match self.case.query_kind() { + QueryKind::ListSymbols => { + let response: ListSymbolsOutput = output.try_json()?; + self.validate_list_symbols(&context, &response) + } + } + } + + fn command_args(&self, server_name: &str) -> Vec { + let mut args = match self.case.query_kind() { + QueryKind::ListSymbols => vec!["list-symbols".to_string(), ".".to_string()], + }; + args.extend(["--lsp".to_string(), server_name.to_string()]); + match self.case.provision_method() { + ProvisionMethod::Download => args.push("--download".to_string()), + } + args.extend([ + "--no-detach".to_string(), + "--json".to_string(), + "--timeout".to_string(), + self.case.lsp_timeout_seconds().to_string(), + ]); + args + } + + fn validate_list_symbols( + &self, + context: &E2eContext, + response: &ListSymbolsOutput, + ) -> Result<(), String> { + if !response.detected.contains(self.case.language()) { + return Err(format!( + "expected detected languages {:?} to contain {:?}", + response.detected, + self.case.language() + )); + } + + let Some(program) = response.server.command.first() else { + return Err("selected server reported an empty command".to_string()); + }; + if !Path::new(program).starts_with(context.home()) { + return Err(format!( + "downloaded server program {} is outside isolated home {}", + program, + context.home().display() + )); + } + + let actual = response + .matches + .iter() + .map(|matched| matched.name.as_str()) + .collect::>(); + let missing = self + .case + .expected_names() + .iter() + .filter(|expected| !actual.contains(expected.as_str())) + .collect::>(); + if missing.is_empty() { + Ok(()) + } else { + Err(format!( + "missing expected symbols {missing:?}; returned symbols: {actual:?}" + )) + } + } +} + +fn remaining(started: Instant, deadline: Duration) -> Result { + let remaining = deadline.saturating_sub(started.elapsed()); + if remaining.is_zero() { + Err(format!( + "case exceeded its overall deadline of {deadline:?}" + )) + } else { + Ok(remaining) + } +} + +fn repository_root() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")) +} + +#[test] +#[ignore = "downloads and runs real LSP servers; executed explicitly in CI"] +fn manifest_real_server_smoke_cases() { + let repository = repository_root(); + let manifest = Manifest::load_validated(&repository).expect("E2E manifest should be valid"); + let failures = manifest + .real_server_smoke_cases() + .filter_map(|case| RealServerTest::new(case, &repository).run().err()) + .collect::>(); + + assert!( + failures.is_empty(), + "real-server E2E failures:\n{}", + failures.join("\n\n") + ); +} From 8817d04f17b5b8a0bb0ff528f7a2c80405f4b98d Mon Sep 17 00:00:00 2001 From: Vasily Kulikov Date: Sat, 5 Sep 2026 19:16:05 +0300 Subject: [PATCH 09/18] cleanup --- .../e2e-real-server-cases-are-manifest-driven.md | 6 ------ .memory/unprocessed/e2e-runtime-root-constraints.md | 11 ----------- .../lsp-config-id-differs-from-cli-server-name.md | 6 ------ .../process-deadline-includes-inherited-pipes.md | 7 ------- .../unprocessed/process-group-cleanup-reaping.md | 10 ---------- .memory/unprocessed/secure-test-temp-location.md | 9 --------- .../test-temp-root-must-allow-short-socket-paths.md | 13 ------------- 7 files changed, 62 deletions(-) delete mode 100644 .memory/unprocessed/e2e-real-server-cases-are-manifest-driven.md delete mode 100644 .memory/unprocessed/e2e-runtime-root-constraints.md delete mode 100644 .memory/unprocessed/lsp-config-id-differs-from-cli-server-name.md delete mode 100644 .memory/unprocessed/process-deadline-includes-inherited-pipes.md delete mode 100644 .memory/unprocessed/process-group-cleanup-reaping.md delete mode 100644 .memory/unprocessed/secure-test-temp-location.md delete mode 100644 .memory/unprocessed/test-temp-root-must-allow-short-socket-paths.md diff --git a/.memory/unprocessed/e2e-real-server-cases-are-manifest-driven.md b/.memory/unprocessed/e2e-real-server-cases-are-manifest-driven.md deleted file mode 100644 index 7025e49..0000000 --- a/.memory/unprocessed/e2e-real-server-cases-are-manifest-driven.md +++ /dev/null @@ -1,6 +0,0 @@ -# Real-server E2E cases are manifest-driven - -The first proposed real-server proof was shaped as a Rust/rust-analyzer-specific test. The user -corrected that direction: the harness must extend to any LSP server and must not hardcode one -server. Keep language IDs, server config IDs, expected symbols, and runtime prerequisites in the -E2E manifest. Runner code may dispatch only on generic operation and provisioning types. diff --git a/.memory/unprocessed/e2e-runtime-root-constraints.md b/.memory/unprocessed/e2e-runtime-root-constraints.md deleted file mode 100644 index 9d9d644..0000000 --- a/.memory/unprocessed/e2e-runtime-root-constraints.md +++ /dev/null @@ -1,11 +0,0 @@ -# E2E runtime-root constraints - -The E2E daemon runtime directory cannot be nested below the ordinary test root. The production -socket suffix (`lsp-cli/`, a server slug of up to 32 characters, a separator, a 24-character hash, -and `.sock`) makes that layout exceed the Unix-domain socket path limit in the current environment. -Create the runtime `TempDir` directly below the selected secure base with a short prefix. - -The configured build-time `XDG_RUNTIME_DIR` may also be mounted read-only by an execution sandbox -even though it is writable in the host environment. Tests using the secure root can therefore need -the test command to run with the sandbox's filesystem restriction lifted; falling back to `/tmp` -would violate the project requirement and conceal the actual path behavior. diff --git a/.memory/unprocessed/lsp-config-id-differs-from-cli-server-name.md b/.memory/unprocessed/lsp-config-id-differs-from-cli-server-name.md deleted file mode 100644 index 6d77bce..0000000 --- a/.memory/unprocessed/lsp-config-id-differs-from-cli-server-name.md +++ /dev/null @@ -1,6 +0,0 @@ -# LSP config ID differs from CLI server name - -The E2E manifest identifies an LSP configuration by the YAML filename stem, such as -`rust_analyzer`. The `--lsp` CLI option does not accept that ID; it selects the configured -user-visible `name`, such as `rust-analyzer`. Generic E2E code must load the name from the selected -LSP YAML instead of passing the manifest ID or duplicating a server-specific name in test code. diff --git a/.memory/unprocessed/process-deadline-includes-inherited-pipes.md b/.memory/unprocessed/process-deadline-includes-inherited-pipes.md deleted file mode 100644 index 6232050..0000000 --- a/.memory/unprocessed/process-deadline-includes-inherited-pipes.md +++ /dev/null @@ -1,7 +0,0 @@ -# Process deadline must include inherited output pipes - -Waiting with a deadline only on the process-group leader is insufficient. The leader can exit while -a descendant remains alive with inherited stdout or stderr descriptors, causing output-reader joins -to block forever after the nominal deadline. Apply the same absolute deadline to both leader exit -and pipe closure; if either pipe remains open, kill the still-addressable process group before -joining readers. diff --git a/.memory/unprocessed/process-group-cleanup-reaping.md b/.memory/unprocessed/process-group-cleanup-reaping.md deleted file mode 100644 index 8becee6..0000000 --- a/.memory/unprocessed/process-group-cleanup-reaping.md +++ /dev/null @@ -1,10 +0,0 @@ -# Process-group cleanup and descendant reaping - -`command-group` successfully sends the kill signal to the E2E command and its descendants, but a -killed grandchild can remain briefly visible in `/proc` as a zombie until its new parent reaps it. -A cleanup regression test must use a bounded wait for the process entry to disappear instead of -interpreting its immediate presence as a live orphan. - -After killing and reaping a process group, the RAII guard must discard its `GroupChild` handle. -Leaving the handle armed makes `Drop` attempt a second group kill; in the unlikely event that the -process-group ID has already been reused, that could signal an unrelated process. diff --git a/.memory/unprocessed/secure-test-temp-location.md b/.memory/unprocessed/secure-test-temp-location.md deleted file mode 100644 index 0ad0471..0000000 --- a/.memory/unprocessed/secure-test-temp-location.md +++ /dev/null @@ -1,9 +0,0 @@ -# Real test sandboxes must not use ambient `/tmp` - -The user clarified that real test state must not be created under `/tmp`, even through the secure -default `tempfile::Builder::tempdir` API. Unit and E2E test helpers should use the existing -`tempfile` crate to create randomized exclusive directories under `XDG_RUNTIME_DIR`, -`XDG_CACHE_HOME`, or `$HOME/.cache`, in that order. - -Synthetic `/tmp` strings used only to test URI or path parsing do not create files and are outside -this policy. diff --git a/.memory/unprocessed/test-temp-root-must-allow-short-socket-paths.md b/.memory/unprocessed/test-temp-root-must-allow-short-socket-paths.md deleted file mode 100644 index 3f7523d..0000000 --- a/.memory/unprocessed/test-temp-root-must-allow-short-socket-paths.md +++ /dev/null @@ -1,13 +0,0 @@ -# Test temp roots must leave room for Unix socket names - -Creating secure test sandboxes below this worktree's `target/test-tmp/` made the absolute paths long -enough that existing Unix listener tests failed with `path must be shorter than SUN_LEN`. Workspace -and daemon test helpers must select a short per-user runtime or cache root, not merely any secure -repository-local directory. - -This should be defended by keeping test directory prefixes compact and by retaining socket tests -that bind the longest production-shaped daemon path. - -The root must also be selected from the build-time environment. Reading `XDG_RUNTIME_DIR` or -`HOME` while tests run races with existing tests that temporarily replace process-wide environment -variables and can place one test's sandbox inside another test's short-lived directory. From 4179cf624fdc2f0e51f17e376e4a9d84ab130929 Mon Sep 17 00:00:00 2001 From: Vasily Kulikov Date: Sat, 5 Sep 2026 19:17:34 +0300 Subject: [PATCH 10/18] Cover every CLI command path in E2E tests --- .github/workflows/ci.yml | 13 +- .memory/unprocessed/ci-reuses-make-targets.md | 6 + .../e2e-fake-lsp-is-a-rust-helper.md | 5 + E2E_TESTS.md | 26 ++- Makefile | 24 ++- src/env_vars.rs | 7 + src/update.rs | 14 +- src/update/tests.rs | 8 +- tests/e2e.rs | 14 ++ tests/e2e/cases.yaml | 28 ++- tests/e2e/catalog.rs | 61 +++--- tests/e2e/filesystem.rs | 30 +++ tests/e2e/fixtures/data/filetypes/fake.yaml | 3 + tests/e2e/fixtures/data/lsp/fake.yaml | 7 + tests/e2e/fixtures/fake-lsp/.gitignore | 1 + tests/e2e/fixtures/fake-lsp/Cargo.lock | 105 +++++++++++ tests/e2e/fixtures/fake-lsp/Cargo.toml | 11 ++ tests/e2e/fixtures/fake-lsp/src/main.rs | 176 ++++++++++++++++++ tests/e2e/fixtures/project/fixture.root | 1 + tests/e2e/fixtures/project/main.fake | 3 + tests/e2e/harness.rs | 28 +++ tests/e2e/lifecycle.rs | 55 ++++++ tests/e2e/local_fixture.rs | 86 +++++++++ tests/e2e/manifest.rs | 56 +++++- tests/e2e/manifest_tests.rs | 19 +- tests/e2e/queries.rs | 80 ++++++++ tests/e2e/update.rs | 175 +++++++++++++++++ 27 files changed, 971 insertions(+), 71 deletions(-) create mode 100644 .memory/unprocessed/ci-reuses-make-targets.md create mode 100644 .memory/unprocessed/e2e-fake-lsp-is-a-rust-helper.md create mode 100644 tests/e2e/filesystem.rs create mode 100644 tests/e2e/fixtures/data/filetypes/fake.yaml create mode 100644 tests/e2e/fixtures/data/lsp/fake.yaml create mode 100644 tests/e2e/fixtures/fake-lsp/.gitignore create mode 100644 tests/e2e/fixtures/fake-lsp/Cargo.lock create mode 100644 tests/e2e/fixtures/fake-lsp/Cargo.toml create mode 100644 tests/e2e/fixtures/fake-lsp/src/main.rs create mode 100644 tests/e2e/fixtures/project/fixture.root create mode 100644 tests/e2e/fixtures/project/main.fake create mode 100644 tests/e2e/lifecycle.rs create mode 100644 tests/e2e/local_fixture.rs create mode 100644 tests/e2e/queries.rs create mode 100644 tests/e2e/update.rs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 73312ca..961a9a7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -35,22 +35,19 @@ jobs: - name: Check formatting if: matrix.rust == 'stable' - run: cargo fmt --check + run: make check-format - name: Check generated command reference if: matrix.rust == 'stable' - run: python3 scripts/update_readme_commands.py --check + run: make check-readme - name: Run tests - run: cargo test --locked + run: make check-tests - name: Run real-server E2E smoke tests if: matrix.rust == 'stable' - run: >- - cargo test --locked --test e2e - real_servers::manifest_real_server_smoke_cases - -- --ignored --exact --nocapture + run: make test-real-server-e2e - name: Run clippy if: matrix.rust == 'stable' - run: cargo clippy --all-targets --all-features -- -D warnings + run: make check-clippy diff --git a/.memory/unprocessed/ci-reuses-make-targets.md b/.memory/unprocessed/ci-reuses-make-targets.md new file mode 100644 index 0000000..346011b --- /dev/null +++ b/.memory/unprocessed/ci-reuses-make-targets.md @@ -0,0 +1,6 @@ +# CI reuses Makefile verification targets + +The initial E2E implementation duplicated formatting, Clippy, and live-server commands between +`Makefile` and `.github/workflows/ci.yml`. The user explicitly rejected that structure. Keep the +commands centralized as granular Make targets; CI should only choose which Make target runs for a +toolchain or lane. diff --git a/.memory/unprocessed/e2e-fake-lsp-is-a-rust-helper.md b/.memory/unprocessed/e2e-fake-lsp-is-a-rust-helper.md new file mode 100644 index 0000000..0591b87 --- /dev/null +++ b/.memory/unprocessed/e2e-fake-lsp-is-a-rust-helper.md @@ -0,0 +1,5 @@ +# The deterministic E2E LSP fixture is a Rust helper + +When offered Python, real rust-analyzer, or a Rust helper for deterministic command-path tests, +the user selected the Rust helper. Keep the helper as an isolated, locked fixture package that +reuses serde_json; do not replace it with a Python script merely to reduce fixture build code. diff --git a/E2E_TESTS.md b/E2E_TESTS.md index b84338e..627d24d 100644 --- a/E2E_TESTS.md +++ b/E2E_TESTS.md @@ -216,7 +216,8 @@ documented exclusions. A validation test should fail when: - two cases select the same user-visible server ambiguously; - a new top-level subcommand has no assigned coverage class. -The version 1 manifest starts with `coverage: partial`, which validates every declared entry +The version 2 manifest assigns every canonical command to a coverage strategy and keeps +`coverage: partial`, which validates every declared language/server entry against the pinned data without requiring unfinished matrix entries. Phase 4 adds the remaining entries and switches it to `coverage: complete`; complete mode enforces every detectable language and compatible pair. @@ -243,9 +244,10 @@ allows the matrix to grow incrementally. ### `run` -`run` replaces the current process with the language server on Unix. Start it with piped stdio, -send a minimal LSP `initialize` / `initialized` / `shutdown` / `exit` exchange, and verify that the -selected real server took over. Test selection and exec errors separately. +`run` replaces the current process with the language server on Unix. The foundation smoke uses a +deterministic server marker to prove replacement. Real-server coverage should additionally use +piped stdio for a minimal `initialize` / `initialized` / `shutdown` / `exit` exchange. Test +selection and exec errors separately. ### `daemon`, `stop`, and `stop-all` @@ -262,16 +264,10 @@ server stderr. Cleanup must run even after an assertion failure. ### `update` -The update repository and release endpoints are currently hardcoded. A deterministic success-path -binary E2E test therefore needs a small production seam that redirects HTTP to a local fixture -server. This change improves testability but is an architectural decision and must be approved -before implementation. - -Without that seam, only argument/error behavior can be deterministic; a live GitHub success test -would be slow, mutable, rate-limited, and capable of replacing test data from the network. - -Recommended decision: introduce a narrowly scoped test-only or configurable repository endpoint, -while keeping the production default unchanged. +The production default uses the lsp-cli-data GitHub release endpoint. The narrowly scoped +`LSP_CLI_DATA_RELEASE_API_URL` override redirects only release metadata lookup, allowing the binary +E2E test to serve metadata and a valid archive locally. This keeps the success path deterministic +without changing normal update behavior. ### Diagnostics and formatting @@ -382,7 +378,7 @@ that class of defect easier to diagnose. - [x] Isolate all environment and runtime state. - [x] Implement deadlines, cleanup guards, JSON helpers, and useful failure diagnostics. - [x] Prove the harness with Rust/rust-analyzer. -- [ ] Cover all 24 subcommand paths with either a real server or a deterministic local fixture. +- [x] Cover all 24 subcommand paths with either a real server or a deterministic local fixture. ### Phase 2: projects diff --git a/Makefile b/Makefile index f99753a..3e1ea25 100644 --- a/Makefile +++ b/Makefile @@ -1,10 +1,26 @@ -test: +.PHONY: test check-format check-tests check-clippy check-readme check-dependencies test-real-server-e2e gen-readme + +test: check-format check-tests check-clippy check-readme check-dependencies + +check-format: cargo fmt --check - cargo test -q - cargo clippy --all-targets --all-features -- -D warnings + cargo fmt --manifest-path tests/e2e/fixtures/fake-lsp/Cargo.toml --check + +check-tests: + cargo test --locked -q + +check-clippy: + cargo clippy --locked --all-targets --all-features -- -D warnings + cargo clippy --locked --manifest-path tests/e2e/fixtures/fake-lsp/Cargo.toml --target-dir target/e2e-fixtures -- -D warnings + +check-readme: python3 scripts/update_readme_commands.py --check + +check-dependencies: cargo deny check +test-real-server-e2e: + cargo test --locked --test e2e real_servers::manifest_real_server_smoke_cases -- --ignored --exact --nocapture + gen-readme: python3 scripts/update_readme_commands.py - diff --git a/src/env_vars.rs b/src/env_vars.rs index b94f5e8..bc8092b 100644 --- a/src/env_vars.rs +++ b/src/env_vars.rs @@ -19,6 +19,9 @@ pub(crate) const PATH: &str = "PATH"; /// Current interactive shell used for shell auto-detection in completion output. pub(crate) const SHELL: &str = "SHELL"; +/// Override the release metadata endpoint, primarily for isolated update testing. +pub(crate) const DATA_RELEASE_API_URL: &str = "LSP_CLI_DATA_RELEASE_API_URL"; + #[cfg(test)] /// Test-only override that tells fake npm installs which executable to materialize. pub(crate) const TEST_FAKE_NPM_PROGRAM: &str = "LSP_CLI_TEST_FAKE_NPM_PROGRAM"; @@ -51,6 +54,10 @@ pub(crate) fn shell() -> Option { std::env::var_os(SHELL) } +pub(crate) fn data_release_api_url() -> Option { + std::env::var(DATA_RELEASE_API_URL).ok() +} + #[cfg(test)] pub(crate) fn fake_npm_program() -> Option { std::env::var_os(TEST_FAKE_NPM_PROGRAM) diff --git a/src/update.rs b/src/update.rs index a6c2abb..2ebc84a 100644 --- a/src/update.rs +++ b/src/update.rs @@ -259,11 +259,7 @@ struct ReleaseDownload { } fn fetch_release(client: &Client, version: &str) -> Result { - let url = if version == "latest" { - format!("https://api.github.com/repos/{DATA_REPOSITORY}/releases/latest") - } else { - format!("https://api.github.com/repos/{DATA_REPOSITORY}/releases/tags/{version}") - }; + let url = crate::env_vars::data_release_api_url().unwrap_or_else(|| release_url(version)); let release: GithubRelease = client .get(url) .send() @@ -292,5 +288,13 @@ fn fetch_release(client: &Client, version: &str) -> Result { }) } +fn release_url(version: &str) -> String { + if version == "latest" { + format!("https://api.github.com/repos/{DATA_REPOSITORY}/releases/latest") + } else { + format!("https://api.github.com/repos/{DATA_REPOSITORY}/releases/tags/{version}") + } +} + #[cfg(test)] mod tests; diff --git a/src/update/tests.rs b/src/update/tests.rs index eb1dfc3..1668ce6 100644 --- a/src/update/tests.rs +++ b/src/update/tests.rs @@ -1,4 +1,4 @@ -use super::{install_downloaded_data, locate_data_root}; +use super::{install_downloaded_data, locate_data_root, release_url}; use crate::runtime_state::RuntimeState; use crate::test_support::TestDir; use flate2::Compression; @@ -29,6 +29,12 @@ fn archive_with_files(files: &[(&str, &[u8])]) -> Vec { encoder.finish().expect("gzip should finish") } +#[test] +fn builds_latest_and_tagged_release_urls() { + assert!(release_url("latest").ends_with("/releases/latest")); + assert!(release_url("v1.2.3").ends_with("/releases/tags/v1.2.3")); +} + #[test] fn installs_valid_downloaded_data_into_runtime_data_dir() { let dir = TestDir::new("update-install"); diff --git a/tests/e2e.rs b/tests/e2e.rs index ec41815..5dad4f6 100644 --- a/tests/e2e.rs +++ b/tests/e2e.rs @@ -2,14 +2,28 @@ clippy::panic, reason = "E2E assertion helpers panic with captured process diagnostics." )] +#![expect( + clippy::expect_used, + reason = "E2E fixtures and assertions fail immediately with contextual expectation messages." +)] #[path = "e2e/catalog.rs"] mod catalog; +#[path = "e2e/filesystem.rs"] +mod filesystem; #[path = "e2e/harness.rs"] mod harness; +#[path = "e2e/lifecycle.rs"] +mod lifecycle; +#[path = "e2e/local_fixture.rs"] +mod local_fixture; #[path = "e2e/manifest.rs"] mod manifest; #[path = "e2e/process.rs"] mod process; +#[path = "e2e/queries.rs"] +mod queries; #[path = "e2e/real_servers.rs"] mod real_servers; +#[path = "e2e/update.rs"] +mod update; diff --git a/tests/e2e/cases.yaml b/tests/e2e/cases.yaml index 0a0749f..0032423 100644 --- a/tests/e2e/cases.yaml +++ b/tests/e2e/cases.yaml @@ -1,6 +1,32 @@ -schema-version: 1 +schema-version: 2 coverage: partial +commands: + - { name: commands, strategy: catalog } + - { name: languages, strategy: catalog } + - { name: servers, strategy: catalog } + - { name: completion, strategy: catalog } + - { name: agent-skill, strategy: catalog } + - { name: detect, strategy: filesystem } + - { name: list-files, strategy: filesystem } + - { name: server-capabilities, strategy: lsp-fixture } + - { name: diagnostics, strategy: lsp-fixture } + - { name: format, strategy: lsp-fixture } + - { name: grep, strategy: lsp-fixture } + - { name: list-symbols, strategy: lsp-fixture } + - { name: list-functions, strategy: lsp-fixture } + - { name: references, strategy: lsp-fixture } + - { name: callers, strategy: lsp-fixture } + - { name: callees, strategy: lsp-fixture } + - { name: definition, strategy: lsp-fixture } + - { name: declaration, strategy: lsp-fixture } + - { name: build-index, strategy: lsp-fixture } + - { name: daemon, strategy: lifecycle } + - { name: stop, strategy: lifecycle } + - { name: stop-all, strategy: lifecycle } + - { name: run, strategy: lifecycle } + - { name: update, strategy: update-fixture } + languages: - id: rust kind: source diff --git a/tests/e2e/catalog.rs b/tests/e2e/catalog.rs index 7a9683b..756f54c 100644 --- a/tests/e2e/catalog.rs +++ b/tests/e2e/catalog.rs @@ -1,40 +1,31 @@ use crate::harness::E2eContext; +use crate::manifest::{CommandStrategy, Manifest}; +use std::collections::BTreeSet; #[test] -fn commands_lists_every_canonical_subcommand() { - let output = E2eContext::new() - .expect("E2E context should initialize") - .run(&["commands"]); +fn catalog_command_paths_are_covered() { + let manifest = Manifest::load_repository().expect("E2E manifest should be valid"); + let context = E2eContext::new().expect("E2E context should initialize"); - output.assert_success(); - assert!(output.stderr_text().is_empty()); - assert_eq!( - output.stdout_text().trim_end(), - concat!( - "commands\n", - "daemon\n", - "stop\n", - "stop-all\n", - "languages\n", - "servers\n", - "server-capabilities\n", - "detect\n", - "diagnostics\n", - "format\n", - "grep\n", - "list-symbols\n", - "list-functions\n", - "list-files\n", - "references\n", - "callers\n", - "callees\n", - "definition\n", - "declaration\n", - "build-index\n", - "update\n", - "completion\n", - "agent-skill\n", - "run" - ) - ); + let commands = context.run(&["commands"]); + commands.assert_success(); + let actual = commands.stdout_text().lines().collect::>(); + assert_eq!(actual, manifest.command_names()); + + for command in manifest.commands_for(CommandStrategy::Catalog) { + let output = match command { + "commands" => continue, + "languages" => context.run(&[command]), + "servers" => context.run(&[command, "--lang", "rust"]), + "completion" => context.run(&[command, "bash"]), + "agent-skill" => context.run(&[command]), + other => panic!("catalog strategy has no scenario for {other:?}"), + }; + output.assert_success(); + assert!( + !output.stdout_text().is_empty(), + "{command} should produce identifying output" + ); + assert!(output.stderr_text().is_empty()); + } } diff --git a/tests/e2e/filesystem.rs b/tests/e2e/filesystem.rs new file mode 100644 index 0000000..d2b6cac --- /dev/null +++ b/tests/e2e/filesystem.rs @@ -0,0 +1,30 @@ +use serde_json::Value; + +use crate::local_fixture::LocalFixture; +use crate::manifest::{CommandStrategy, Manifest}; + +#[test] +fn filesystem_command_paths_are_covered() { + let manifest = Manifest::load_repository().expect("E2E manifest should be valid"); + let fixture = LocalFixture::new().expect("local fixture should initialize"); + + for command in manifest.commands_for(CommandStrategy::Filesystem) { + let output = + fixture + .context() + .run(&[command, ".", "--lsp", fixture.server_name(), "--json"]); + output.assert_success(); + let value: Value = output.json(); + match command { + "detect" => assert_eq!( + value.pointer("/servers/0/server").and_then(Value::as_str), + Some(fixture.server_name()) + ), + "list-files" => assert_eq!( + value.pointer("/files/0").and_then(Value::as_str), + Some("./main.fake") + ), + other => panic!("filesystem strategy has no scenario for {other:?}"), + } + } +} diff --git a/tests/e2e/fixtures/data/filetypes/fake.yaml b/tests/e2e/fixtures/data/filetypes/fake.yaml new file mode 100644 index 0000000..95fd7c3 --- /dev/null +++ b/tests/e2e/fixtures/data/filetypes/fake.yaml @@ -0,0 +1,3 @@ +extensions: + - "fake" +patterns: [] diff --git a/tests/e2e/fixtures/data/lsp/fake.yaml b/tests/e2e/fixtures/data/lsp/fake.yaml new file mode 100644 index 0000000..2cd6466 --- /dev/null +++ b/tests/e2e/fixtures/data/lsp/fake.yaml @@ -0,0 +1,7 @@ +filetypes: + - "fake" +root_markers: + - "fixture.root" +name: "e2e-fake-lsp" +cmdline: "e2e-fake-lsp" +wait-for-index: false diff --git a/tests/e2e/fixtures/fake-lsp/.gitignore b/tests/e2e/fixtures/fake-lsp/.gitignore new file mode 100644 index 0000000..b83d222 --- /dev/null +++ b/tests/e2e/fixtures/fake-lsp/.gitignore @@ -0,0 +1 @@ +/target/ diff --git a/tests/e2e/fixtures/fake-lsp/Cargo.lock b/tests/e2e/fixtures/fake-lsp/Cargo.lock new file mode 100644 index 0000000..a828bcc --- /dev/null +++ b/tests/e2e/fixtures/fake-lsp/Cargo.lock @@ -0,0 +1,105 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "lsp-cli-e2e-fake-lsp" +version = "0.0.0" +dependencies = [ + "serde_json", +] + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.150" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "syn" +version = "3.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12df2e0110f65b775f769bb17ef989067a1d931b2eb822bd4346631eeada89f9" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/tests/e2e/fixtures/fake-lsp/Cargo.toml b/tests/e2e/fixtures/fake-lsp/Cargo.toml new file mode 100644 index 0000000..91f335a --- /dev/null +++ b/tests/e2e/fixtures/fake-lsp/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "lsp-cli-e2e-fake-lsp" +version = "0.0.0" +edition = "2024" +rust-version = "1.91" +publish = false + +[dependencies] +serde_json = "=1.0.150" + +[workspace] diff --git a/tests/e2e/fixtures/fake-lsp/src/main.rs b/tests/e2e/fixtures/fake-lsp/src/main.rs new file mode 100644 index 0000000..85e1865 --- /dev/null +++ b/tests/e2e/fixtures/fake-lsp/src/main.rs @@ -0,0 +1,176 @@ +use std::env; +use std::io::{self, BufRead, BufReader, Write}; + +use serde_json::{Value, json}; + +fn main() -> Result<(), Box> { + if env::var_os("LSP_CLI_E2E_RUN_MARKER").is_some() { + println!("fake LSP server replaced lsp-cli"); + return Ok(()); + } + + let mut input = BufReader::new(io::stdin().lock()); + let mut output = io::stdout().lock(); + let mut root_uri = String::new(); + let mut report_status = false; + + while let Some(message) = read_message(&mut input)? { + let method = message.get("method").and_then(Value::as_str).unwrap_or(""); + if method == "exit" { + break; + } + if method == "initialized" { + if report_status { + write_message( + &mut output, + &json!({ + "jsonrpc": "2.0", + "method": "experimental/serverStatus", + "params": {"health": "ok", "quiescent": true} + }), + )?; + } + continue; + } + let Some(id) = message.get("id").cloned() else { + continue; + }; + + let result = match method { + "initialize" => { + root_uri = message["params"]["rootUri"] + .as_str() + .unwrap_or("file:///") + .to_string(); + report_status = + message["params"]["capabilities"]["experimental"]["serverStatusNotification"] + .as_bool() + .unwrap_or(false); + initialize_result() + } + "workspace/symbol" | "textDocument/documentSymbol" => { + json!([symbol(&root_uri)]) + } + "textDocument/diagnostic" => json!({ + "kind": "full", + "items": [{ + "range": range(1, 2, 13), + "severity": 2, + "code": "fixture", + "source": "e2e-fake-lsp", + "message": "synthetic diagnostic" + }] + }), + "textDocument/formatting" => json!([{ + "range": range(1, 0, 13), + "newText": " formatted" + }]), + "textDocument/references" | "textDocument/definition" | "textDocument/declaration" => { + json!([location(&root_uri)]) + } + "textDocument/prepareCallHierarchy" => json!([call_item(&root_uri)]), + "callHierarchy/incomingCalls" => { + json!([{"from": call_item(&root_uri), "fromRanges": []}]) + } + "callHierarchy/outgoingCalls" => { + json!([{"to": call_item(&root_uri), "fromRanges": []}]) + } + "shutdown" => Value::Null, + _ => { + write_message( + &mut output, + &json!({ + "jsonrpc": "2.0", + "id": id, + "error": {"code": -32601, "message": format!("unsupported method {method}")} + }), + )?; + continue; + } + }; + write_message( + &mut output, + &json!({"jsonrpc": "2.0", "id": id, "result": result}), + )?; + } + Ok(()) +} + +fn initialize_result() -> Value { + json!({ + "capabilities": { + "textDocumentSync": 1, + "workspaceSymbolProvider": true, + "documentSymbolProvider": true, + "definitionProvider": true, + "declarationProvider": true, + "referencesProvider": true, + "callHierarchyProvider": true, + "documentFormattingProvider": true, + "diagnosticProvider": { + "identifier": "fixture", + "interFileDependencies": false, + "workspaceDiagnostics": false + } + }, + "serverInfo": {"name": "e2e-fake-lsp", "version": "1"} + }) +} + +fn symbol(root_uri: &str) -> Value { + json!({ + "name": "Target", + "kind": 12, + "location": location(root_uri) + }) +} + +fn location(root_uri: &str) -> Value { + json!({"uri": format!("{root_uri}/main.fake"), "range": range(0, 3, 9)}) +} + +fn call_item(root_uri: &str) -> Value { + json!({ + "name": "Target", + "kind": 12, + "uri": format!("{root_uri}/main.fake"), + "range": range(0, 0, 13), + "selectionRange": range(0, 3, 9) + }) +} + +fn range(line: u32, start: u32, end: u32) -> Value { + json!({ + "start": {"line": line, "character": start}, + "end": {"line": line, "character": end} + }) +} + +fn read_message(reader: &mut impl BufRead) -> io::Result> { + let mut content_length = None; + loop { + let mut header = String::new(); + if reader.read_line(&mut header)? == 0 { + return Ok(None); + } + if header == "\r\n" { + break; + } + if let Some(value) = header.strip_prefix("Content-Length:") { + content_length = Some(value.trim().parse::().map_err(io::Error::other)?); + } + } + let length = content_length.ok_or_else(|| io::Error::other("missing Content-Length"))?; + let mut body = vec![0; length]; + reader.read_exact(&mut body)?; + serde_json::from_slice(&body) + .map(Some) + .map_err(io::Error::other) +} + +fn write_message(writer: &mut impl Write, message: &Value) -> io::Result<()> { + let body = serde_json::to_vec(message).map_err(io::Error::other)?; + write!(writer, "Content-Length: {}\r\n\r\n", body.len())?; + writer.write_all(&body)?; + writer.flush() +} diff --git a/tests/e2e/fixtures/project/fixture.root b/tests/e2e/fixtures/project/fixture.root new file mode 100644 index 0000000..ee8c1ee --- /dev/null +++ b/tests/e2e/fixtures/project/fixture.root @@ -0,0 +1 @@ +fixture diff --git a/tests/e2e/fixtures/project/main.fake b/tests/e2e/fixtures/project/main.fake new file mode 100644 index 0000000..c92e8e0 --- /dev/null +++ b/tests/e2e/fixtures/project/main.fake @@ -0,0 +1,3 @@ +fn Target() { + unformatted +} diff --git a/tests/e2e/harness.rs b/tests/e2e/harness.rs index 0a71ca9..537ef35 100644 --- a/tests/e2e/harness.rs +++ b/tests/e2e/harness.rs @@ -86,6 +86,19 @@ impl E2eContext { }) } + pub(crate) fn with_data_dir(mut self, data_dir: PathBuf) -> Self { + self.data_dir = data_dir; + self + } + + pub(crate) fn stage_program(&self, name: &str, source: &Path) -> Result<(), String> { + let source = source + .canonicalize() + .map_err(|error| format!("failed to resolve {}: {error}", source.display()))?; + self.link_host_program(&source, &self.bin_dir.join(name)) + .map_err(|error| format!("failed to stage {} as {name:?}: {error}", source.display())) + } + pub(crate) fn stage_host_program( &self, name: &str, @@ -166,6 +179,14 @@ impl E2eContext { &self.home } + pub(crate) fn workspace(&self) -> &Path { + &self.workspace + } + + pub(crate) fn installed_data(&self) -> PathBuf { + self.home.join(".local/share/lsp-cli/data") + } + pub(crate) fn run(&self, args: &[&str]) -> E2eOutput { self.run_with_deadline(args, DEFAULT_COMMAND_DEADLINE) } @@ -185,6 +206,13 @@ impl E2eContext { self.run_command(&mut command, deadline) } + pub(crate) fn run_with_env(&self, args: &[&str], environment: &[(&str, &str)]) -> E2eOutput { + let mut command = self.command(); + command.args(args).envs(environment.iter().copied()); + self.run_command(&mut command, DEFAULT_COMMAND_DEADLINE) + .unwrap_or_else(|diagnostic| panic!("{diagnostic}")) + } + fn command(&self) -> Command { self.command_for(env!("CARGO_BIN_EXE_lsp-cli")) } diff --git a/tests/e2e/lifecycle.rs b/tests/e2e/lifecycle.rs new file mode 100644 index 0000000..9d4b5aa --- /dev/null +++ b/tests/e2e/lifecycle.rs @@ -0,0 +1,55 @@ +use crate::local_fixture::LocalFixture; +use crate::manifest::{CommandStrategy, Manifest}; +use std::collections::BTreeSet; + +#[test] +fn lifecycle_command_paths_are_covered() { + let manifest = Manifest::load_repository().expect("E2E manifest should be valid"); + assert_eq!( + manifest + .commands_for(CommandStrategy::Lifecycle) + .collect::>(), + BTreeSet::from(["daemon", "run", "stop", "stop-all"]) + ); + let fixture = LocalFixture::new().expect("local fixture should initialize"); + let context = fixture.context(); + let server = fixture.server_name(); + + let daemon = context.run(&[ + "daemon", + ".", + "--lsp", + server, + "--no-download", + "--idle-timeout", + "10", + ]); + daemon.assert_success(); + assert!(daemon.stdout_text().trim_end().ends_with(".sock")); + + let stop = context.run(&["stop", ".", "--lsp", server]); + stop.assert_success(); + assert!(stop.stdout_text().contains("stopped")); + + context + .run(&[ + "daemon", + ".", + "--lsp", + server, + "--no-download", + "--idle-timeout", + "10", + ]) + .assert_success(); + let stop_all = context.run(&["stop-all"]); + stop_all.assert_success(); + assert!(stop_all.stdout_text().contains("stopped")); + + let run = context.run_with_env( + &["run", ".", "--lsp", server, "--no-download"], + &[("LSP_CLI_E2E_RUN_MARKER", "1")], + ); + run.assert_success(); + assert_eq!(run.stdout_text(), "fake LSP server replaced lsp-cli\n"); +} diff --git a/tests/e2e/local_fixture.rs b/tests/e2e/local_fixture.rs new file mode 100644 index 0000000..2a62513 --- /dev/null +++ b/tests/e2e/local_fixture.rs @@ -0,0 +1,86 @@ +use std::path::{Path, PathBuf}; +use std::process::Command; +use std::sync::OnceLock; +use std::time::Duration; + +use crate::harness::E2eContext; +use crate::process; + +const SERVER_NAME: &str = "e2e-fake-lsp"; +const BINARY_NAME: &str = "lsp-cli-e2e-fake-lsp"; +const BUILD_DEADLINE: Duration = Duration::from_secs(120); + +pub(crate) struct LocalFixture { + context: E2eContext, +} + +impl LocalFixture { + pub(crate) fn new() -> Result { + let root = repository_root(); + let fixtures = root.join("tests/e2e/fixtures"); + let context = E2eContext::new() + .map_err(|error| format!("failed to create fixture context: {error}"))? + .with_data_dir(fixtures.join("data")); + context.copy_project(&fixtures.join("project"))?; + context.stage_program(SERVER_NAME, fake_server_binary()?)?; + Ok(Self { context }) + } + + pub(crate) fn context(&self) -> &E2eContext { + &self.context + } + + pub(crate) fn server_name(&self) -> &'static str { + SERVER_NAME + } +} + +fn fake_server_binary() -> Result<&'static Path, String> { + static BINARY: OnceLock> = OnceLock::new(); + match BINARY.get_or_init(build_fake_server) { + Ok(path) => Ok(path), + Err(error) => Err(error.clone()), + } +} + +fn build_fake_server() -> Result { + let root = repository_root(); + let manifest = root.join("tests/e2e/fixtures/fake-lsp/Cargo.toml"); + let target = root.join("target/e2e-fixtures"); + let cargo = option_env!("CARGO").unwrap_or("cargo"); + let mut command = Command::new(cargo); + command.args([ + "build", + "--quiet", + "--locked", + "--manifest-path", + manifest + .to_str() + .ok_or_else(|| format!("fixture manifest path is not UTF-8: {}", manifest.display()))?, + "--target-dir", + target + .to_str() + .ok_or_else(|| format!("fixture target path is not UTF-8: {}", target.display()))?, + ]); + let output = process::run(&mut command, BUILD_DEADLINE) + .map_err(|failure| failure.diagnostic("fake LSP build has no daemon runtime"))?; + if !output.status().success() { + return Err(output.diagnostic( + "failed to build the fake LSP fixture", + "fake LSP build has no daemon runtime", + )); + } + let binary = target.join("debug").join(BINARY_NAME); + if binary.is_file() { + Ok(binary) + } else { + Err(format!( + "fake LSP build did not create {}", + binary.display() + )) + } +} + +fn repository_root() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")) +} diff --git a/tests/e2e/manifest.rs b/tests/e2e/manifest.rs index 753f675..d0dd4ed 100644 --- a/tests/e2e/manifest.rs +++ b/tests/e2e/manifest.rs @@ -5,13 +5,14 @@ use std::path::{Component, Path, PathBuf}; use serde::Deserialize; use serde::de::DeserializeOwned; -const MANIFEST_SCHEMA_VERSION: u32 = 1; +const MANIFEST_SCHEMA_VERSION: u32 = 2; #[derive(Clone, Debug, Deserialize)] #[serde(rename_all = "kebab-case", deny_unknown_fields)] pub(crate) struct Manifest { schema_version: u32, coverage: Coverage, + commands: Vec, languages: Vec, pairs: Vec, } @@ -23,6 +24,23 @@ enum Coverage { Complete, } +#[derive(Clone, Debug, Deserialize)] +#[serde(rename_all = "kebab-case", deny_unknown_fields)] +struct CommandCase { + name: String, + strategy: CommandStrategy, +} + +#[derive(Clone, Copy, Debug, Deserialize, Eq, Ord, PartialEq, PartialOrd)] +#[serde(rename_all = "kebab-case")] +pub(crate) enum CommandStrategy { + Catalog, + Filesystem, + LspFixture, + Lifecycle, + UpdateFixture, +} + #[derive(Clone, Debug, Deserialize)] #[serde(rename_all = "kebab-case", deny_unknown_fields)] struct LanguageCase { @@ -147,6 +165,7 @@ impl Manifest { if self.languages.is_empty() { return Err("E2E manifest must declare at least one language".to_string()); } + self.validate_commands()?; if self.pairs.is_empty() { return Err("E2E manifest must declare at least one language/server pair".to_string()); } @@ -167,6 +186,10 @@ impl Manifest { Ok(manifest) } + pub(crate) fn load_repository() -> Result { + Self::load_validated(Path::new(env!("CARGO_MANIFEST_DIR"))) + } + pub(crate) fn real_server_smoke_cases(&self) -> impl Iterator> { self.pairs.iter().filter_map(|pair| { let smoke = pair.smoke.as_ref()?; @@ -182,6 +205,37 @@ impl Manifest { }) } + pub(crate) fn command_names(&self) -> BTreeSet<&str> { + self.commands + .iter() + .map(|command| command.name.as_str()) + .collect() + } + + pub(crate) fn commands_for(&self, strategy: CommandStrategy) -> impl Iterator { + self.commands + .iter() + .filter(move |command| command.strategy == strategy) + .map(|command| command.name.as_str()) + } + + fn validate_commands(&self) -> Result<(), String> { + if self.commands.is_empty() { + return Err("E2E manifest must assign command coverage".to_string()); + } + let mut names = BTreeSet::new(); + for command in &self.commands { + validate_config_id("command", &command.name)?; + if !names.insert(&command.name) { + return Err(format!( + "E2E manifest assigns command {:?} more than once", + command.name + )); + } + } + Ok(()) + } + fn validate_languages( &self, repository: &Path, diff --git a/tests/e2e/manifest_tests.rs b/tests/e2e/manifest_tests.rs index f8f1597..4160b7a 100644 --- a/tests/e2e/manifest_tests.rs +++ b/tests/e2e/manifest_tests.rs @@ -52,13 +52,30 @@ fn complete_mode_rejects_missing_server_pairs() { #[test] fn manifest_rejects_unknown_fields() { let error = serde_yaml::from_str::( - "schema-version: 1\ncoverage: partial\nlanguages: []\npairs: []\nunknown: true\n", + "schema-version: 2\ncoverage: partial\ncommands: []\nlanguages: []\npairs: []\nunknown: true\n", ) .expect_err("unknown manifest fields should fail"); assert!(error.to_string().contains("unknown field `unknown`")); } +#[test] +fn manifest_rejects_duplicate_command_coverage() { + let mut manifest = Manifest::load().expect("E2E manifest should parse"); + let duplicate = manifest + .commands + .first() + .expect("manifest should contain command coverage") + .clone(); + manifest.commands.push(duplicate); + + let error = manifest + .validate(&repository_root()) + .expect_err("duplicate command coverage should fail"); + + assert!(error.contains("more than once")); +} + #[test] fn manifest_rejects_config_path_traversal() { let mut manifest = Manifest::load().expect("E2E manifest should parse"); diff --git a/tests/e2e/queries.rs b/tests/e2e/queries.rs new file mode 100644 index 0000000..d58c1d6 --- /dev/null +++ b/tests/e2e/queries.rs @@ -0,0 +1,80 @@ +use serde_json::Value; + +use crate::local_fixture::LocalFixture; +use crate::manifest::{CommandStrategy, Manifest}; + +#[test] +fn lsp_fixture_command_paths_are_covered() { + let manifest = Manifest::load_repository().expect("E2E manifest should be valid"); + let fixture = LocalFixture::new().expect("local fixture should initialize"); + + for command in manifest.commands_for(CommandStrategy::LspFixture) { + run_query(&fixture, command); + } +} + +fn run_query(fixture: &LocalFixture, command: &str) { + let mut args = command_prefix(command); + args.extend([ + "--lsp".to_string(), + fixture.server_name().to_string(), + "--no-download".to_string(), + "--no-detach".to_string(), + "--timeout".to_string(), + "5".to_string(), + ]); + if !matches!(command, "server-capabilities" | "format" | "build-index") { + args.push("--json".to_string()); + } + if command == "format" { + args.push("--stdout".to_string()); + } + let refs = args.iter().map(String::as_str).collect::>(); + let output = fixture.context().run(&refs); + output.assert_success(); + + match command { + "server-capabilities" => assert!(output.stdout_text().contains("workspace symbols")), + "build-index" => assert!(output.stdout_text().is_empty()), + "format" => { + assert!(output.stdout_text().contains(" formatted")); + assert!( + std::fs::read_to_string(fixture.context().workspace().join("main.fake")) + .expect("fixture source should remain readable") + .contains(" unformatted") + ); + } + "diagnostics" => { + let value: Value = output.json(); + assert_eq!( + value + .pointer("/diagnostics/0/message") + .and_then(Value::as_str), + Some("synthetic diagnostic") + ); + } + other => { + let value: Value = output.json(); + assert_eq!( + value.pointer("/matches/0/name").and_then(Value::as_str), + Some("Target"), + "command {other}" + ); + } + } +} + +fn command_prefix(command: &str) -> Vec { + match command { + "grep" | "references" | "callers" | "callees" | "definition" | "declaration" => { + vec![command.to_string(), "Target".to_string(), ".".to_string()] + } + "format" => vec![command.to_string(), "main.fake".to_string()], + "server-capabilities" + | "diagnostics" + | "list-symbols" + | "list-functions" + | "build-index" => vec![command.to_string(), ".".to_string()], + other => panic!("LSP fixture strategy has no scenario for {other:?}"), + } +} diff --git a/tests/e2e/update.rs b/tests/e2e/update.rs new file mode 100644 index 0000000..c7d2696 --- /dev/null +++ b/tests/e2e/update.rs @@ -0,0 +1,175 @@ +use std::io::{Read, Write}; +use std::net::{TcpListener, TcpStream}; +use std::thread; +use std::time::{Duration, Instant}; + +use flate2::Compression; +use flate2::write::GzEncoder; +use serde_json::json; + +use crate::harness::E2eContext; +use crate::manifest::{CommandStrategy, Manifest}; + +#[test] +fn update_command_path_uses_local_release_fixture() { + let manifest = Manifest::load_repository().expect("E2E manifest should be valid"); + assert_eq!( + manifest + .commands_for(CommandStrategy::UpdateFixture) + .collect::>(), + ["update"] + ); + let archive = data_archive(); + let server = HttpFixture::start(archive); + let context = E2eContext::new().expect("E2E context should initialize"); + + let output = context.run_with_env( + &["update"], + &[("LSP_CLI_DATA_RELEASE_API_URL", server.release_url())], + ); + output.assert_success(); + assert_eq!(output.stdout_text(), "updated lsp-cli data to e2e-v1\n"); + assert!( + context + .installed_data() + .join("filetypes/fake.yaml") + .is_file() + ); + server.finish(); +} + +struct HttpFixture { + release_url: String, + thread: Option>>, +} + +impl HttpFixture { + fn start(archive: Vec) -> Self { + let listener = TcpListener::bind("127.0.0.1:0").expect("HTTP fixture should bind"); + listener + .set_nonblocking(true) + .expect("HTTP fixture should become nonblocking"); + let address = listener + .local_addr() + .expect("HTTP fixture should have an address"); + let base = format!("http://{address}"); + let release_url = format!("{base}/release"); + let thread = thread::spawn(move || serve(listener, &base, &archive)); + Self { + release_url, + thread: Some(thread), + } + } + + fn release_url(&self) -> &str { + &self.release_url + } + + fn finish(mut self) { + self.thread + .take() + .expect("HTTP fixture thread should be present") + .join() + .expect("HTTP fixture thread should not panic") + .expect("HTTP fixture should serve both requests"); + } +} + +impl Drop for HttpFixture { + fn drop(&mut self) { + // Join here because an assertion may unwind before `finish`, leaving a fixture thread alive. + if let Some(thread) = self.thread.take() { + let _result = thread.join(); + } + } +} + +fn serve(listener: TcpListener, base: &str, archive: &[u8]) -> Result<(), String> { + let deadline = Instant::now() + Duration::from_secs(10); + let mut served = 0; + while served < 2 && Instant::now() < deadline { + match listener.accept() { + Ok((mut stream, _address)) => { + serve_request(&mut stream, base, archive)?; + served += 1; + } + Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => { + thread::sleep(Duration::from_millis(10)); + } + Err(error) => return Err(format!("HTTP fixture accept failed: {error}")), + } + } + if served == 2 { + Ok(()) + } else { + Err(format!( + "HTTP fixture served {served} of 2 expected requests" + )) + } +} + +fn serve_request(stream: &mut TcpStream, base: &str, archive: &[u8]) -> Result<(), String> { + stream + .set_read_timeout(Some(Duration::from_secs(2))) + .map_err(|error| error.to_string())?; + let mut request = [0; 4096]; + let length = stream + .read(&mut request) + .map_err(|error| error.to_string())?; + let request = String::from_utf8_lossy( + request + .get(..length) + .expect("read byte count should fit its buffer"), + ); + let (content_type, body) = if request.starts_with("GET /release ") { + let body = json!({ + "tag_name": "e2e-v1", + "tarball_url": format!("{base}/archive"), + "zipball_url": null + }) + .to_string() + .into_bytes(); + ("application/json", body) + } else if request.starts_with("GET /archive ") { + ("application/gzip", archive.to_vec()) + } else { + return Err(format!("unexpected HTTP fixture request: {request}")); + }; + write!( + stream, + "HTTP/1.1 200 OK\r\nContent-Type: {content_type}\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", + body.len() + ) + .and_then(|()| stream.write_all(&body)) + .map_err(|error| error.to_string()) +} + +fn data_archive() -> Vec { + let encoder = GzEncoder::new(Vec::new(), Compression::default()); + let mut archive = tar::Builder::new(encoder); + append( + &mut archive, + "data/filetypes/fake.yaml", + b"extensions: [fake]\n", + ); + append( + &mut archive, + "data/lsp/fake.yaml", + b"filetypes: [fake]\nroot_markers: []\nname: fake\ncmdline: fake\n", + ); + archive + .into_inner() + .expect("archive should finish") + .finish() + .expect("gzip stream should finish") +} + +fn append(archive: &mut tar::Builder>>, path: &str, contents: &[u8]) { + let mut header = tar::Header::new_gnu(); + header.set_size(u64::try_from(contents.len()).expect("fixture size should fit u64")); + header.set_mode(0o644); + header.set_cksum(); + archive + .append_data(&mut header, path, contents) + .expect("fixture archive entry should append"); +} From b11e4c87e5e66115990fe7f945f32be3e3ab0f87 Mon Sep 17 00:00:00 2001 From: Vasily Kulikov Date: Sat, 5 Sep 2026 19:25:34 +0300 Subject: [PATCH 11/18] Audit existing E2E playground projects --- .../unprocessed/playground-validity-audit.md | 10 ++++++ E2E_TESTS.md | 35 ++++++++++++++++++- 2 files changed, 44 insertions(+), 1 deletion(-) create mode 100644 .memory/unprocessed/playground-validity-audit.md diff --git a/.memory/unprocessed/playground-validity-audit.md b/.memory/unprocessed/playground-validity-audit.md new file mode 100644 index 0000000..6c7c573 --- /dev/null +++ b/.memory/unprocessed/playground-validity-audit.md @@ -0,0 +1,10 @@ +# Existing playgrounds are not all valid standalone projects + +`E2E_TESTS.md` requires every source-language playground to be valid, but the existing C++ fixture +does not link because `main.cpp` calls undefined `f()` and `g()`. The Rust fixture also cannot be +checked through its own manifest because Cargo treats the nested package as an undeclared member +of the repository's root package workspace. + +The project audit must distinguish source presence from validated buildability. Repair these +fixtures in the dedicated cleanup step rather than silently treating their files as a valid E2E +baseline. diff --git a/E2E_TESTS.md b/E2E_TESTS.md index 627d24d..c62d61c 100644 --- a/E2E_TESTS.md +++ b/E2E_TESTS.md @@ -102,6 +102,39 @@ it should contain: Prefer equivalent domain concepts and symbol names across projects when natural. Do not force a language into constructs it does not support merely to make fixtures textually identical. +### Existing playground audit + +The ten existing playgrounds were audited against these requirements on 2026-09-05. `Present` +means the committed source provides the semantic shape; `missing` identifies follow-up work; +`unverified` means the required compiler or runtime was not installed for this audit. Under the +strict declaration rule, an interface, trait, protocol, or declaration file is required when the +language can express one without relying on comments or third-party syntax. + +| Project | Valid, small, multi-file | Stable workspace symbol | Functions and methods | Separate declaration | Cross-file references | Caller/callee chain | Types and fields | Formatting mutation | Diagnostic mutation | +|---|---|---|---|---|---|---|---|---|---| +| C | Present, but compilation database is not portable | `Order` | Functions present; methods not applicable | Present in `order.h` | Present | Present | Present | Missing recipe; baseline is not formatter-clean | Missing recipe | +| C++ | **Invalid:** undefined `f()` and `g()` prevent linking; compilation database is not portable | `playground::Order` | Present | Present in `order.hpp` | Present | Present | Present | Missing recipe; baseline is not formatter-clean | Missing recipe | +| C# | Unverified; `dotnet` unavailable | `Order` | Present | **Missing:** an interface can provide it | Present | Present | Present | Missing recipe | Missing recipe | +| Go | Unverified; `go` unavailable | `Order` | Present | **Missing:** an interface can provide it | Present | Present | Present | Missing recipe; baseline is visibly not `gofmt`-clean | Missing recipe | +| Java | Unverified; JDK and Maven unavailable | `Order` | Present | **Missing:** an interface can provide it | Present | Present | Present | Missing recipe | Missing recipe | +| JavaScript | Present; exercised with Node.js | `Order` | Present | **Missing:** a declaration file can provide it | Present | Present | Present | Missing recipe | Missing recipe | +| Lua | Unverified; Lua unavailable | **Missing:** only local functions are declared | Functions present; methods missing | Not applicable: Lua has no native declaration construct | Present for `format_timestamp` | Present | Partial: a module table field exists, but no structured domain type | Missing recipe | Missing recipe | +| Python | Present; exercised with Python | `Order` | Present | **Missing:** a protocol or abstract base can provide it | Present | Present | Present | Missing recipe; baseline is visibly not formatter-clean | Missing recipe | +| Rust | **Invalid as a standalone project:** Cargo treats it as an undeclared root-workspace member | `Order` | Present | **Missing:** a trait can provide it | Present | Present | Present | Missing recipe | Missing recipe | +| TypeScript | Unverified; local TypeScript compiler unavailable | `Order` | Present | **Missing:** an interface can provide it | Present | Present | Present | Missing recipe | Missing recipe | + +The C sources compile and link, while the C++ sources compile but fail at link time because the +calls added in `main.cpp` have no definitions. JavaScript and Python execute successfully. The Rust +check fails before compilation because the nested package is neither a root-workspace member nor +excluded from that workspace. C and C++ also embed an old absolute checkout path in +`compile_commands.json`; language servers may therefore ignore their intended include paths after +the repository is moved or copied into an isolated E2E sandbox. + +No playground currently defines the exact source edit and expected diagnostic needed for a stable +mutation test. Those recipes should live in manifest data rather than language-specific Rust test +code. The next project-phase item will repair and normalize the fixtures; this audit intentionally +does not mix those changes with the inventory. + Tests securely create randomized sandboxes with the `tempfile` crate under the user's `XDG_RUNTIME_DIR`, `XDG_CACHE_HOME`, or `$HOME/.cache`, in that order, then copy a project there before formatting it or introducing diagnostics. Real test state must not use the ambient system @@ -382,7 +415,7 @@ that class of defect easier to diagnose. ### Phase 2: projects -- [ ] Audit the ten existing playgrounds against the common semantic requirements. +- [x] Audit the ten existing playgrounds against the common semantic requirements. - [ ] Remove duplicated setup patterns within each class of fixture. - [ ] Add CUDA, Kotlin, Objective-C, and Objective-C++ projects. - [ ] Add `gomod` and `gowork` detection fixtures. From a3ed999375adf63d32f02a5d2e5a547370aefcd5 Mon Sep 17 00:00:00 2001 From: Vasily Kulikov Date: Sat, 5 Sep 2026 19:31:47 +0300 Subject: [PATCH 12/18] Deduplicate E2E fixture setup --- .../unprocessed/e2e-fixture-dedup-scope.md | 5 +++ E2E_TESTS.md | 2 +- tests/e2e.rs | 8 ++++ tests/e2e/catalog.rs | 12 +++--- tests/e2e/filesystem.rs | 5 +-- tests/e2e/fixture.rs | 41 +++++++++++++++++++ tests/e2e/lifecycle.rs | 7 ++-- tests/e2e/local_fixture.rs | 27 ++++++------ tests/e2e/manifest_tests.rs | 17 ++++---- tests/e2e/queries.rs | 5 +-- tests/e2e/real_servers.rs | 11 ++--- tests/e2e/update.rs | 10 ++--- 12 files changed, 99 insertions(+), 51 deletions(-) create mode 100644 .memory/unprocessed/e2e-fixture-dedup-scope.md create mode 100644 tests/e2e/fixture.rs diff --git a/.memory/unprocessed/e2e-fixture-dedup-scope.md b/.memory/unprocessed/e2e-fixture-dedup-scope.md new file mode 100644 index 0000000..bcf98ef --- /dev/null +++ b/.memory/unprocessed/e2e-fixture-dedup-scope.md @@ -0,0 +1,5 @@ +# E2E fixture deduplication applies to the Rust harness + +The project-phase deduplication item applies to repeated E2E harness setup, including validated +manifest loading, isolated context construction, and command-strategy selection. It does not mean +removing the intentionally equivalent domain concepts from the per-language playground sources. diff --git a/E2E_TESTS.md b/E2E_TESTS.md index c62d61c..ff9cea0 100644 --- a/E2E_TESTS.md +++ b/E2E_TESTS.md @@ -416,7 +416,7 @@ that class of defect easier to diagnose. ### Phase 2: projects - [x] Audit the ten existing playgrounds against the common semantic requirements. -- [ ] Remove duplicated setup patterns within each class of fixture. +- [x] Remove duplicated setup patterns within each class of fixture. - [ ] Add CUDA, Kotlin, Objective-C, and Objective-C++ projects. - [ ] Add `gomod` and `gowork` detection fixtures. - [ ] Update `playground/README.md` with manual reproduction commands. diff --git a/tests/e2e.rs b/tests/e2e.rs index 5dad4f6..8e0efa0 100644 --- a/tests/e2e.rs +++ b/tests/e2e.rs @@ -11,6 +11,8 @@ mod catalog; #[path = "e2e/filesystem.rs"] mod filesystem; +#[path = "e2e/fixture.rs"] +mod fixture; #[path = "e2e/harness.rs"] mod harness; #[path = "e2e/lifecycle.rs"] @@ -27,3 +29,9 @@ mod queries; mod real_servers; #[path = "e2e/update.rs"] mod update; + +use std::path::Path; + +pub(crate) fn repository_root() -> &'static Path { + Path::new(env!("CARGO_MANIFEST_DIR")) +} diff --git a/tests/e2e/catalog.rs b/tests/e2e/catalog.rs index 756f54c..d66b32c 100644 --- a/tests/e2e/catalog.rs +++ b/tests/e2e/catalog.rs @@ -1,18 +1,18 @@ -use crate::harness::E2eContext; -use crate::manifest::{CommandStrategy, Manifest}; +use crate::fixture::E2eFixture; +use crate::manifest::CommandStrategy; use std::collections::BTreeSet; #[test] fn catalog_command_paths_are_covered() { - let manifest = Manifest::load_repository().expect("E2E manifest should be valid"); - let context = E2eContext::new().expect("E2E context should initialize"); + let fixture = E2eFixture::new().expect("E2E fixture should initialize"); + let context = fixture.context(); let commands = context.run(&["commands"]); commands.assert_success(); let actual = commands.stdout_text().lines().collect::>(); - assert_eq!(actual, manifest.command_names()); + assert_eq!(actual, fixture.manifest().command_names()); - for command in manifest.commands_for(CommandStrategy::Catalog) { + for command in fixture.commands_for(CommandStrategy::Catalog) { let output = match command { "commands" => continue, "languages" => context.run(&[command]), diff --git a/tests/e2e/filesystem.rs b/tests/e2e/filesystem.rs index d2b6cac..3919fa4 100644 --- a/tests/e2e/filesystem.rs +++ b/tests/e2e/filesystem.rs @@ -1,14 +1,13 @@ use serde_json::Value; use crate::local_fixture::LocalFixture; -use crate::manifest::{CommandStrategy, Manifest}; +use crate::manifest::CommandStrategy; #[test] fn filesystem_command_paths_are_covered() { - let manifest = Manifest::load_repository().expect("E2E manifest should be valid"); let fixture = LocalFixture::new().expect("local fixture should initialize"); - for command in manifest.commands_for(CommandStrategy::Filesystem) { + for command in fixture.commands_for(CommandStrategy::Filesystem) { let output = fixture .context() diff --git a/tests/e2e/fixture.rs b/tests/e2e/fixture.rs new file mode 100644 index 0000000..d7977f3 --- /dev/null +++ b/tests/e2e/fixture.rs @@ -0,0 +1,41 @@ +use std::path::PathBuf; + +use crate::harness::E2eContext; +use crate::manifest::{CommandStrategy, Manifest}; + +pub(crate) struct E2eFixture { + context: E2eContext, + manifest: Manifest, +} + +impl E2eFixture { + pub(crate) fn new() -> Result { + Self::initialize(None) + } + + pub(crate) fn new_with_data_dir(data_dir: PathBuf) -> Result { + Self::initialize(Some(data_dir)) + } + + pub(crate) fn context(&self) -> &E2eContext { + &self.context + } + + pub(crate) fn manifest(&self) -> &Manifest { + &self.manifest + } + + pub(crate) fn commands_for(&self, strategy: CommandStrategy) -> impl Iterator { + self.manifest.commands_for(strategy) + } + + fn initialize(data_dir: Option) -> Result { + let manifest = Manifest::load_repository()?; + let mut context = + E2eContext::new().map_err(|error| format!("failed to create E2E context: {error}"))?; + if let Some(data_dir) = data_dir { + context = context.with_data_dir(data_dir); + } + Ok(Self { context, manifest }) + } +} diff --git a/tests/e2e/lifecycle.rs b/tests/e2e/lifecycle.rs index 9d4b5aa..bba0378 100644 --- a/tests/e2e/lifecycle.rs +++ b/tests/e2e/lifecycle.rs @@ -1,17 +1,16 @@ use crate::local_fixture::LocalFixture; -use crate::manifest::{CommandStrategy, Manifest}; +use crate::manifest::CommandStrategy; use std::collections::BTreeSet; #[test] fn lifecycle_command_paths_are_covered() { - let manifest = Manifest::load_repository().expect("E2E manifest should be valid"); + let fixture = LocalFixture::new().expect("local fixture should initialize"); assert_eq!( - manifest + fixture .commands_for(CommandStrategy::Lifecycle) .collect::>(), BTreeSet::from(["daemon", "run", "stop", "stop-all"]) ); - let fixture = LocalFixture::new().expect("local fixture should initialize"); let context = fixture.context(); let server = fixture.server_name(); diff --git a/tests/e2e/local_fixture.rs b/tests/e2e/local_fixture.rs index 2a62513..f2c25ac 100644 --- a/tests/e2e/local_fixture.rs +++ b/tests/e2e/local_fixture.rs @@ -3,31 +3,38 @@ use std::process::Command; use std::sync::OnceLock; use std::time::Duration; +use crate::fixture::E2eFixture; use crate::harness::E2eContext; +use crate::manifest::CommandStrategy; use crate::process; +use crate::repository_root; const SERVER_NAME: &str = "e2e-fake-lsp"; const BINARY_NAME: &str = "lsp-cli-e2e-fake-lsp"; const BUILD_DEADLINE: Duration = Duration::from_secs(120); pub(crate) struct LocalFixture { - context: E2eContext, + fixture: E2eFixture, } impl LocalFixture { pub(crate) fn new() -> Result { let root = repository_root(); let fixtures = root.join("tests/e2e/fixtures"); - let context = E2eContext::new() - .map_err(|error| format!("failed to create fixture context: {error}"))? - .with_data_dir(fixtures.join("data")); - context.copy_project(&fixtures.join("project"))?; - context.stage_program(SERVER_NAME, fake_server_binary()?)?; - Ok(Self { context }) + let fixture = E2eFixture::new_with_data_dir(fixtures.join("data"))?; + fixture.context().copy_project(&fixtures.join("project"))?; + fixture + .context() + .stage_program(SERVER_NAME, fake_server_binary()?)?; + Ok(Self { fixture }) } pub(crate) fn context(&self) -> &E2eContext { - &self.context + self.fixture.context() + } + + pub(crate) fn commands_for(&self, strategy: CommandStrategy) -> impl Iterator { + self.fixture.commands_for(strategy) } pub(crate) fn server_name(&self) -> &'static str { @@ -80,7 +87,3 @@ fn build_fake_server() -> Result { )) } } - -fn repository_root() -> PathBuf { - PathBuf::from(env!("CARGO_MANIFEST_DIR")) -} diff --git a/tests/e2e/manifest_tests.rs b/tests/e2e/manifest_tests.rs index 4160b7a..20045a3 100644 --- a/tests/e2e/manifest_tests.rs +++ b/tests/e2e/manifest_tests.rs @@ -1,8 +1,5 @@ use super::*; - -fn repository_root() -> PathBuf { - PathBuf::from(env!("CARGO_MANIFEST_DIR")) -} +use crate::repository_root; fn first_smoke(manifest: &mut Manifest) -> &mut SmokeCase { manifest @@ -18,7 +15,7 @@ fn first_smoke(manifest: &mut Manifest) -> &mut SmokeCase { fn partial_manifest_matches_pinned_data() { Manifest::load() .expect("E2E manifest should parse") - .validate(&repository_root()) + .validate(repository_root()) .expect("E2E manifest should be valid"); } @@ -28,7 +25,7 @@ fn complete_mode_rejects_the_partial_matrix() { manifest.coverage = Coverage::Complete; let error = manifest - .validate(&repository_root()) + .validate(repository_root()) .expect_err("partial matrix should not satisfy complete coverage"); assert!(error.contains("complete E2E manifest is missing languages")); @@ -70,7 +67,7 @@ fn manifest_rejects_duplicate_command_coverage() { manifest.commands.push(duplicate); let error = manifest - .validate(&repository_root()) + .validate(repository_root()) .expect_err("duplicate command coverage should fail"); assert!(error.contains("more than once")); @@ -91,7 +88,7 @@ fn manifest_rejects_config_path_traversal() { .language = "../rust".to_string(); let error = manifest - .validate(&repository_root()) + .validate(repository_root()) .expect_err("config path traversal should fail"); assert!(error.contains("must be one normalized path component")); @@ -104,7 +101,7 @@ fn manifest_rejects_invalid_smoke_deadlines() { smoke.deadline_seconds = smoke.lsp_timeout_seconds.saturating_sub(1); let error = manifest - .validate(&repository_root()) + .validate(repository_root()) .expect_err("short overall deadline should fail"); assert!(error.contains("deadline must not be shorter")); @@ -122,7 +119,7 @@ fn manifest_rejects_duplicate_host_programs() { smoke.host_programs.push(duplicate); let error = manifest - .validate(&repository_root()) + .validate(repository_root()) .expect_err("duplicate host programs should fail"); assert!(error.contains("more than once")); diff --git a/tests/e2e/queries.rs b/tests/e2e/queries.rs index d58c1d6..82808d4 100644 --- a/tests/e2e/queries.rs +++ b/tests/e2e/queries.rs @@ -1,14 +1,13 @@ use serde_json::Value; use crate::local_fixture::LocalFixture; -use crate::manifest::{CommandStrategy, Manifest}; +use crate::manifest::CommandStrategy; #[test] fn lsp_fixture_command_paths_are_covered() { - let manifest = Manifest::load_repository().expect("E2E manifest should be valid"); let fixture = LocalFixture::new().expect("local fixture should initialize"); - for command in manifest.commands_for(CommandStrategy::LspFixture) { + for command in fixture.commands_for(CommandStrategy::LspFixture) { run_query(&fixture, command); } } diff --git a/tests/e2e/real_servers.rs b/tests/e2e/real_servers.rs index b97977b..c9f65ea 100644 --- a/tests/e2e/real_servers.rs +++ b/tests/e2e/real_servers.rs @@ -1,11 +1,12 @@ use std::collections::BTreeSet; -use std::path::{Path, PathBuf}; +use std::path::Path; use std::time::{Duration, Instant}; use serde::Deserialize; use crate::harness::E2eContext; use crate::manifest::{Manifest, ProvisionMethod, QueryKind, RealServerCase}; +use crate::repository_root; struct RealServerTest<'a> { case: RealServerCase<'a>, @@ -137,18 +138,14 @@ fn remaining(started: Instant, deadline: Duration) -> Result { } } -fn repository_root() -> PathBuf { - PathBuf::from(env!("CARGO_MANIFEST_DIR")) -} - #[test] #[ignore = "downloads and runs real LSP servers; executed explicitly in CI"] fn manifest_real_server_smoke_cases() { let repository = repository_root(); - let manifest = Manifest::load_validated(&repository).expect("E2E manifest should be valid"); + let manifest = Manifest::load_validated(repository).expect("E2E manifest should be valid"); let failures = manifest .real_server_smoke_cases() - .filter_map(|case| RealServerTest::new(case, &repository).run().err()) + .filter_map(|case| RealServerTest::new(case, repository).run().err()) .collect::>(); assert!( diff --git a/tests/e2e/update.rs b/tests/e2e/update.rs index c7d2696..1add672 100644 --- a/tests/e2e/update.rs +++ b/tests/e2e/update.rs @@ -7,21 +7,21 @@ use flate2::Compression; use flate2::write::GzEncoder; use serde_json::json; -use crate::harness::E2eContext; -use crate::manifest::{CommandStrategy, Manifest}; +use crate::fixture::E2eFixture; +use crate::manifest::CommandStrategy; #[test] fn update_command_path_uses_local_release_fixture() { - let manifest = Manifest::load_repository().expect("E2E manifest should be valid"); + let fixture = E2eFixture::new().expect("E2E fixture should initialize"); assert_eq!( - manifest + fixture .commands_for(CommandStrategy::UpdateFixture) .collect::>(), ["update"] ); let archive = data_archive(); let server = HttpFixture::start(archive); - let context = E2eContext::new().expect("E2E context should initialize"); + let context = fixture.context(); let output = context.run_with_env( &["update"], From 26f5045692e4375a624fb1ca6956e65604bfaec0 Mon Sep 17 00:00:00 2001 From: Vasily Kulikov Date: Sat, 5 Sep 2026 19:42:50 +0300 Subject: [PATCH 13/18] Add CUDA Kotlin and Objective-C playgrounds --- .../new-playground-build-metadata.md | 6 +++ E2E_TESTS.md | 6 +-- playground/cuda/compile_commands.json | 50 +++++++++++++++++++ playground/cuda/include/order.cuh | 24 +++++++++ playground/cuda/include/report.cuh | 10 ++++ playground/cuda/src/main.cu | 14 ++++++ playground/cuda/src/order.cu | 21 ++++++++ playground/cuda/src/report/formatter.cu | 10 ++++ playground/kotlin/settings.gradle.kts | 1 + .../kotlin/src/main/kotlin/playground/App.kt | 9 ++++ .../src/main/kotlin/playground/order/Order.kt | 20 ++++++++ .../kotlin/playground/order/OrderService.kt | 9 ++++ .../playground/report/OrderFormatter.kt | 6 +++ playground/objc/compile_commands.json | 17 +++++++ playground/objc/include/Order.h | 32 ++++++++++++ playground/objc/include/OrderFormatter.h | 10 ++++ playground/objc/src/Order.m | 41 +++++++++++++++ playground/objc/src/OrderFormatter.m | 12 +++++ playground/objc/src/main.m | 12 +++++ playground/objcpp/compile_commands.json | 17 +++++++ playground/objcpp/include/Order.hpp | 36 +++++++++++++ playground/objcpp/include/OrderFormatter.hpp | 10 ++++ playground/objcpp/src/Order.mm | 43 ++++++++++++++++ playground/objcpp/src/OrderFormatter.mm | 13 +++++ playground/objcpp/src/main.mm | 10 ++++ tests/e2e/cases.yaml | 12 +++++ 26 files changed, 448 insertions(+), 3 deletions(-) create mode 100644 .memory/unprocessed/new-playground-build-metadata.md create mode 100644 playground/cuda/compile_commands.json create mode 100644 playground/cuda/include/order.cuh create mode 100644 playground/cuda/include/report.cuh create mode 100644 playground/cuda/src/main.cu create mode 100644 playground/cuda/src/order.cu create mode 100644 playground/cuda/src/report/formatter.cu create mode 100644 playground/kotlin/settings.gradle.kts create mode 100644 playground/kotlin/src/main/kotlin/playground/App.kt create mode 100644 playground/kotlin/src/main/kotlin/playground/order/Order.kt create mode 100644 playground/kotlin/src/main/kotlin/playground/order/OrderService.kt create mode 100644 playground/kotlin/src/main/kotlin/playground/report/OrderFormatter.kt create mode 100644 playground/objc/compile_commands.json create mode 100644 playground/objc/include/Order.h create mode 100644 playground/objc/include/OrderFormatter.h create mode 100644 playground/objc/src/Order.m create mode 100644 playground/objc/src/OrderFormatter.m create mode 100644 playground/objc/src/main.m create mode 100644 playground/objcpp/compile_commands.json create mode 100644 playground/objcpp/include/Order.hpp create mode 100644 playground/objcpp/include/OrderFormatter.hpp create mode 100644 playground/objcpp/src/Order.mm create mode 100644 playground/objcpp/src/OrderFormatter.mm create mode 100644 playground/objcpp/src/main.mm diff --git a/.memory/unprocessed/new-playground-build-metadata.md b/.memory/unprocessed/new-playground-build-metadata.md new file mode 100644 index 0000000..93f1d2e --- /dev/null +++ b/.memory/unprocessed/new-playground-build-metadata.md @@ -0,0 +1,6 @@ +# New playgrounds use dependency-free and relocatable metadata + +The Kotlin playground uses compiler-valid sources plus `settings.gradle.kts`, without adding a +Gradle or Maven Kotlin plugin dependency. CUDA, Objective-C, and Objective-C++ use relative +`compile_commands.json` working directories so their compilation metadata remains valid after the +E2E harness copies a project into a randomized sandbox. diff --git a/E2E_TESTS.md b/E2E_TESTS.md index ff9cea0..55122ba 100644 --- a/E2E_TESTS.md +++ b/E2E_TESTS.md @@ -74,9 +74,9 @@ Reuse the projects under `playground/` for: | `rust` | `playground/rust` | | `typescript` | `playground/typescript` | -### Projects to add +### Additional projects -Add source projects for: +Source projects are committed for: - `playground/cuda` - `playground/kotlin` @@ -417,7 +417,7 @@ that class of defect easier to diagnose. - [x] Audit the ten existing playgrounds against the common semantic requirements. - [x] Remove duplicated setup patterns within each class of fixture. -- [ ] Add CUDA, Kotlin, Objective-C, and Objective-C++ projects. +- [x] Add CUDA, Kotlin, Objective-C, and Objective-C++ projects. - [ ] Add `gomod` and `gowork` detection fixtures. - [ ] Update `playground/README.md` with manual reproduction commands. - [ ] Run each relevant command manually against every new project. diff --git a/playground/cuda/compile_commands.json b/playground/cuda/compile_commands.json new file mode 100644 index 0000000..57cb3ea --- /dev/null +++ b/playground/cuda/compile_commands.json @@ -0,0 +1,50 @@ +[ + { + "directory": ".", + "arguments": [ + "clang++", + "-x", + "cuda", + "--cuda-host-only", + "-nocudainc", + "-nocudalib", + "-std=c++17", + "-Iinclude", + "-fsyntax-only", + "src/main.cu" + ], + "file": "src/main.cu" + }, + { + "directory": ".", + "arguments": [ + "clang++", + "-x", + "cuda", + "--cuda-host-only", + "-nocudainc", + "-nocudalib", + "-std=c++17", + "-Iinclude", + "-fsyntax-only", + "src/order.cu" + ], + "file": "src/order.cu" + }, + { + "directory": ".", + "arguments": [ + "clang++", + "-x", + "cuda", + "--cuda-host-only", + "-nocudainc", + "-nocudalib", + "-std=c++17", + "-Iinclude", + "-fsyntax-only", + "src/report/formatter.cu" + ], + "file": "src/report/formatter.cu" + } +] diff --git a/playground/cuda/include/order.cuh b/playground/cuda/include/order.cuh new file mode 100644 index 0000000..f8c71e5 --- /dev/null +++ b/playground/cuda/include/order.cuh @@ -0,0 +1,24 @@ +#ifndef PLAYGROUND_CUDA_ORDER_CUH +#define PLAYGROUND_CUDA_ORDER_CUH + +#include + +struct OrderItem { + const char *name; + int quantity; + double price; + + __host__ __device__ double total() const; +}; + +struct Order { + const char *customer; + const OrderItem *items; + std::size_t item_count; + + __host__ __device__ double total() const; +}; + +Order sample_order(); + +#endif diff --git a/playground/cuda/include/report.cuh b/playground/cuda/include/report.cuh new file mode 100644 index 0000000..24ace10 --- /dev/null +++ b/playground/cuda/include/report.cuh @@ -0,0 +1,10 @@ +#ifndef PLAYGROUND_CUDA_REPORT_CUH +#define PLAYGROUND_CUDA_REPORT_CUH + +#include + +#include "order.cuh" + +std::string format_order(const Order &order); + +#endif diff --git a/playground/cuda/src/main.cu b/playground/cuda/src/main.cu new file mode 100644 index 0000000..eb2c34b --- /dev/null +++ b/playground/cuda/src/main.cu @@ -0,0 +1,14 @@ +#include "order.cuh" +#include "report.cuh" + +#include + +__global__ void calculate_order_total(Order order, double *result) { + *result = order.total(); +} + +int main() { + const Order order = sample_order(); + std::cout << format_order(order) << '\n'; + return 0; +} diff --git a/playground/cuda/src/order.cu b/playground/cuda/src/order.cu new file mode 100644 index 0000000..42b9856 --- /dev/null +++ b/playground/cuda/src/order.cu @@ -0,0 +1,21 @@ +#include "order.cuh" + +__host__ __device__ double OrderItem::total() const { + return static_cast(quantity) * price; +} + +__host__ __device__ double Order::total() const { + double value = 0.0; + for (std::size_t index = 0; index < item_count; ++index) { + value += items[index].total(); + } + return value; +} + +Order sample_order() { + static const OrderItem items[] = { + {"GPU", 1, 499.0}, + {"Power Cable", 2, 12.5}, + }; + return {"Jensen", items, 2}; +} diff --git a/playground/cuda/src/report/formatter.cu b/playground/cuda/src/report/formatter.cu new file mode 100644 index 0000000..d6b8514 --- /dev/null +++ b/playground/cuda/src/report/formatter.cu @@ -0,0 +1,10 @@ +#include "report.cuh" + +#include + +std::string format_order(const Order &order) { + std::ostringstream output; + output << order.customer << " has " << order.item_count << " items worth " + << order.total(); + return output.str(); +} diff --git a/playground/kotlin/settings.gradle.kts b/playground/kotlin/settings.gradle.kts new file mode 100644 index 0000000..27118e9 --- /dev/null +++ b/playground/kotlin/settings.gradle.kts @@ -0,0 +1 @@ +rootProject.name = "lsp-cli-playground-kotlin" diff --git a/playground/kotlin/src/main/kotlin/playground/App.kt b/playground/kotlin/src/main/kotlin/playground/App.kt new file mode 100644 index 0000000..5a77eb2 --- /dev/null +++ b/playground/kotlin/src/main/kotlin/playground/App.kt @@ -0,0 +1,9 @@ +package playground + +import playground.order.sampleOrder +import playground.report.formatOrder + +fun main() { + val order = sampleOrder() + println(formatOrder(order)) +} diff --git a/playground/kotlin/src/main/kotlin/playground/order/Order.kt b/playground/kotlin/src/main/kotlin/playground/order/Order.kt new file mode 100644 index 0000000..afa49eb --- /dev/null +++ b/playground/kotlin/src/main/kotlin/playground/order/Order.kt @@ -0,0 +1,20 @@ +package playground.order + +interface OrderTotal { + fun total(): Double +} + +data class OrderItem( + val name: String, + val quantity: Int, + val price: Double +) : OrderTotal { + override fun total(): Double = quantity * price +} + +data class Order( + val customer: String, + val items: List +) : OrderTotal { + override fun total(): Double = items.map { item -> item.total() }.sum() +} diff --git a/playground/kotlin/src/main/kotlin/playground/order/OrderService.kt b/playground/kotlin/src/main/kotlin/playground/order/OrderService.kt new file mode 100644 index 0000000..f92875c --- /dev/null +++ b/playground/kotlin/src/main/kotlin/playground/order/OrderService.kt @@ -0,0 +1,9 @@ +package playground.order + +fun sampleOrder(): Order = Order( + customer = "JetBrains", + items = listOf( + OrderItem(name = "Keyboard", quantity = 1, price = 120.0), + OrderItem(name = "Keycap", quantity = 4, price = 3.5) + ) +) diff --git a/playground/kotlin/src/main/kotlin/playground/report/OrderFormatter.kt b/playground/kotlin/src/main/kotlin/playground/report/OrderFormatter.kt new file mode 100644 index 0000000..f65e3ef --- /dev/null +++ b/playground/kotlin/src/main/kotlin/playground/report/OrderFormatter.kt @@ -0,0 +1,6 @@ +package playground.report + +import playground.order.Order + +fun formatOrder(order: Order): String = + "${order.customer} has ${order.items.size} items worth ${"%.2f".format(order.total())}" diff --git a/playground/objc/compile_commands.json b/playground/objc/compile_commands.json new file mode 100644 index 0000000..7c6e5d2 --- /dev/null +++ b/playground/objc/compile_commands.json @@ -0,0 +1,17 @@ +[ + { + "directory": ".", + "arguments": ["clang", "-x", "objective-c", "-Iinclude", "-fsyntax-only", "src/main.m"], + "file": "src/main.m" + }, + { + "directory": ".", + "arguments": ["clang", "-x", "objective-c", "-Iinclude", "-fsyntax-only", "src/Order.m"], + "file": "src/Order.m" + }, + { + "directory": ".", + "arguments": ["clang", "-x", "objective-c", "-Iinclude", "-fsyntax-only", "src/OrderFormatter.m"], + "file": "src/OrderFormatter.m" + } +] diff --git a/playground/objc/include/Order.h b/playground/objc/include/Order.h new file mode 100644 index 0000000..c383d5c --- /dev/null +++ b/playground/objc/include/Order.h @@ -0,0 +1,32 @@ +#ifndef PLAYGROUND_OBJC_ORDER_H +#define PLAYGROUND_OBJC_ORDER_H + +#include + +typedef struct { + const char *name; + int quantity; + double price; +} OrderItem; + +__attribute__((objc_root_class)) +@interface Order { +@private + const char *_customer; + const OrderItem *_items; + size_t _itemCount; +} + +- (instancetype)initWithCustomer:(const char *)customer + items:(const OrderItem *)items + count:(size_t)itemCount; +- (const char *)customer; +- (size_t)itemCount; +- (double)total; + +@end + +double item_total(OrderItem item); +Order *sample_order(void); + +#endif diff --git a/playground/objc/include/OrderFormatter.h b/playground/objc/include/OrderFormatter.h new file mode 100644 index 0000000..33258ae --- /dev/null +++ b/playground/objc/include/OrderFormatter.h @@ -0,0 +1,10 @@ +#ifndef PLAYGROUND_OBJC_ORDER_FORMATTER_H +#define PLAYGROUND_OBJC_ORDER_FORMATTER_H + +#include + +#include "Order.h" + +void format_order(Order *order, char *buffer, size_t buffer_size); + +#endif diff --git a/playground/objc/src/Order.m b/playground/objc/src/Order.m new file mode 100644 index 0000000..83af5f8 --- /dev/null +++ b/playground/objc/src/Order.m @@ -0,0 +1,41 @@ +#include "Order.h" + +@implementation Order + +- (instancetype)initWithCustomer:(const char *)customer + items:(const OrderItem *)items + count:(size_t)itemCount { + _customer = customer; + _items = items; + _itemCount = itemCount; + return self; +} + +- (const char *)customer { + return _customer; +} + +- (size_t)itemCount { + return _itemCount; +} + +- (double)total { + double value = 0.0; + for (size_t index = 0; index < _itemCount; ++index) { + value += item_total(_items[index]); + } + return value; +} + +@end + +double item_total(OrderItem item) { return item.quantity * item.price; } + +Order *sample_order(void) { + static const OrderItem items[] = { + {"Display", 1, 600.0}, + {"Stand", 1, 80.0}, + }; + Order *order = (Order *)0; + return [order initWithCustomer:"Brad" items:items count:2]; +} diff --git a/playground/objc/src/OrderFormatter.m b/playground/objc/src/OrderFormatter.m new file mode 100644 index 0000000..16c96b6 --- /dev/null +++ b/playground/objc/src/OrderFormatter.m @@ -0,0 +1,12 @@ +#include "OrderFormatter.h" + +#include + +void format_order(Order *order, char *buffer, size_t buffer_size) { + if (order == (Order *)0) { + snprintf(buffer, buffer_size, "empty order"); + return; + } + snprintf(buffer, buffer_size, "%s has %zu items worth %.2f", [order customer], + [order itemCount], [order total]); +} diff --git a/playground/objc/src/main.m b/playground/objc/src/main.m new file mode 100644 index 0000000..5057ece --- /dev/null +++ b/playground/objc/src/main.m @@ -0,0 +1,12 @@ +#include "Order.h" +#include "OrderFormatter.h" + +#include + +int main(void) { + Order *order = sample_order(); + char summary[128]; + format_order(order, summary, sizeof(summary)); + puts(summary); + return 0; +} diff --git a/playground/objcpp/compile_commands.json b/playground/objcpp/compile_commands.json new file mode 100644 index 0000000..9fbd37b --- /dev/null +++ b/playground/objcpp/compile_commands.json @@ -0,0 +1,17 @@ +[ + { + "directory": ".", + "arguments": ["clang++", "-x", "objective-c++", "-std=c++17", "-Iinclude", "-fsyntax-only", "src/main.mm"], + "file": "src/main.mm" + }, + { + "directory": ".", + "arguments": ["clang++", "-x", "objective-c++", "-std=c++17", "-Iinclude", "-fsyntax-only", "src/Order.mm"], + "file": "src/Order.mm" + }, + { + "directory": ".", + "arguments": ["clang++", "-x", "objective-c++", "-std=c++17", "-Iinclude", "-fsyntax-only", "src/OrderFormatter.mm"], + "file": "src/OrderFormatter.mm" + } +] diff --git a/playground/objcpp/include/Order.hpp b/playground/objcpp/include/Order.hpp new file mode 100644 index 0000000..9c2778d --- /dev/null +++ b/playground/objcpp/include/Order.hpp @@ -0,0 +1,36 @@ +#ifndef PLAYGROUND_OBJCPP_ORDER_HPP +#define PLAYGROUND_OBJCPP_ORDER_HPP + +#include +#include + +struct OrderItem { + std::string name; + int quantity; + double price; + + double total() const; +}; + +@protocol OrderTotaling +- (double)total; +@end + +__attribute__((objc_root_class)) +@interface Order { +@private + std::string _customer; + std::vector _items; +} + +- (instancetype)initWithCustomer:(std::string)customer + items:(std::vector)items; +- (const std::string &)customer; +- (const std::vector &)items; +- (double)total; + +@end + +Order *sample_order(); + +#endif diff --git a/playground/objcpp/include/OrderFormatter.hpp b/playground/objcpp/include/OrderFormatter.hpp new file mode 100644 index 0000000..3c5e0dc --- /dev/null +++ b/playground/objcpp/include/OrderFormatter.hpp @@ -0,0 +1,10 @@ +#ifndef PLAYGROUND_OBJCPP_ORDER_FORMATTER_HPP +#define PLAYGROUND_OBJCPP_ORDER_FORMATTER_HPP + +#include + +#include "Order.hpp" + +std::string format_order(Order *order); + +#endif diff --git a/playground/objcpp/src/Order.mm b/playground/objcpp/src/Order.mm new file mode 100644 index 0000000..de2ed17 --- /dev/null +++ b/playground/objcpp/src/Order.mm @@ -0,0 +1,43 @@ +#include "Order.hpp" + +#include + +double OrderItem::total() const { + return static_cast(quantity) * price; +} + +@implementation Order + +- (instancetype)initWithCustomer:(std::string)customer + items:(std::vector)items { + _customer = std::move(customer); + _items = std::move(items); + return self; +} + +- (const std::string &)customer { + return _customer; +} + +- (const std::vector &)items { + return _items; +} + +- (double)total { + double value = 0.0; + for (const OrderItem &item : _items) { + value += item.total(); + } + return value; +} + +@end + +Order *sample_order() { + Order *order = (Order *)nullptr; + std::vector items = { + {"Compiler", 1, 75.0}, + {"Book", 2, 42.0}, + }; + return [order initWithCustomer:std::string("Bjarne") items:std::move(items)]; +} diff --git a/playground/objcpp/src/OrderFormatter.mm b/playground/objcpp/src/OrderFormatter.mm new file mode 100644 index 0000000..9e650e0 --- /dev/null +++ b/playground/objcpp/src/OrderFormatter.mm @@ -0,0 +1,13 @@ +#include "OrderFormatter.hpp" + +#include + +std::string format_order(Order *order) { + if (order == nullptr) { + return "empty order"; + } + std::ostringstream output; + output << [order customer] << " has " << [order items].size() + << " items worth " << [order total]; + return output.str(); +} diff --git a/playground/objcpp/src/main.mm b/playground/objcpp/src/main.mm new file mode 100644 index 0000000..15dc38c --- /dev/null +++ b/playground/objcpp/src/main.mm @@ -0,0 +1,10 @@ +#include "Order.hpp" +#include "OrderFormatter.hpp" + +#include + +int main() { + Order *order = sample_order(); + std::cout << format_order(order) << '\n'; + return 0; +} diff --git a/tests/e2e/cases.yaml b/tests/e2e/cases.yaml index 0032423..4e0a62d 100644 --- a/tests/e2e/cases.yaml +++ b/tests/e2e/cases.yaml @@ -28,6 +28,18 @@ commands: - { name: update, strategy: update-fixture } languages: + - id: cuda + kind: source + project: playground/cuda + - id: kotlin + kind: source + project: playground/kotlin + - id: objc + kind: source + project: playground/objc + - id: objcpp + kind: source + project: playground/objcpp - id: rust kind: source project: playground/rust From 043c1b4f98a38be991ef40173adf7f0aea1357d7 Mon Sep 17 00:00:00 2001 From: Vasily Kulikov Date: Sat, 5 Sep 2026 21:03:31 +0300 Subject: [PATCH 14/18] Split E2E cases and add Go metadata fixtures --- .../e2e-cases-are-split-by-language.md | 6 + E2E_TESTS.md | 27 ++-- playground/gomod/go.mod | 3 + playground/gowork/go.work | 1 + tests/e2e.rs | 2 + tests/e2e/case_files.rs | 33 +++++ tests/e2e/cases/cuda.yaml | 4 + tests/e2e/cases/gomod.yaml | 4 + tests/e2e/cases/gowork.yaml | 4 + tests/e2e/cases/kotlin.yaml | 4 + tests/e2e/cases/objc.yaml | 4 + tests/e2e/cases/objcpp.yaml | 4 + tests/e2e/cases/rust.yaml | 25 ++++ tests/e2e/{cases.yaml => cases/suite.yaml} | 38 ------ tests/e2e/manifest.rs | 116 ++++++++++++------ tests/e2e/manifest_tests.rs | 32 ++++- 16 files changed, 219 insertions(+), 88 deletions(-) create mode 100644 .memory/unprocessed/e2e-cases-are-split-by-language.md create mode 100644 playground/gomod/go.mod create mode 100644 playground/gowork/go.work create mode 100644 tests/e2e/case_files.rs create mode 100644 tests/e2e/cases/cuda.yaml create mode 100644 tests/e2e/cases/gomod.yaml create mode 100644 tests/e2e/cases/gowork.yaml create mode 100644 tests/e2e/cases/kotlin.yaml create mode 100644 tests/e2e/cases/objc.yaml create mode 100644 tests/e2e/cases/objcpp.yaml create mode 100644 tests/e2e/cases/rust.yaml rename tests/e2e/{cases.yaml => cases/suite.yaml} (59%) diff --git a/.memory/unprocessed/e2e-cases-are-split-by-language.md b/.memory/unprocessed/e2e-cases-are-split-by-language.md new file mode 100644 index 0000000..7205191 --- /dev/null +++ b/.memory/unprocessed/e2e-cases-are-split-by-language.md @@ -0,0 +1,6 @@ +# E2E case manifests are split by language + +The E2E manifest is a directory rather than one `cases.yaml` file. Global command coverage lives +in `cases/suite.yaml`; each `cases/.yaml` owns exactly one project and all server pairs +for that language. The `gowork` case intentionally contains only an isolated `go.work` file so its +detection does not overlap `gomod` or `go`. diff --git a/E2E_TESTS.md b/E2E_TESTS.md index 55122ba..fdf1531 100644 --- a/E2E_TESTS.md +++ b/E2E_TESTS.md @@ -83,9 +83,10 @@ Source projects are committed for: - `playground/objc` - `playground/objcpp` -Add minimal detection fixtures for `gomod` and `gowork`. These IDs describe Go workspace metadata, -not source languages, so they can cover detection, file listing, server selection, initialization, -and lifecycle, but cannot independently provide meaningful symbol or call-hierarchy assertions. +Minimal detection fixtures are committed under `playground/gomod` and `playground/gowork`. These +IDs describe Go workspace metadata, not source languages, so they can cover detection, file +listing, server selection, initialization, and lifecycle, but cannot independently provide +meaningful symbol or call-hierarchy assertions. Every source-language project should be small, valid, and multi-file. Where the language permits, it should contain: @@ -212,7 +213,9 @@ tests/ queries.rs lifecycle.rs update.rs - cases.yaml + cases/ + suite.yaml + .yaml ``` Keep every Rust file under 600 lines. Move repeated process setup and assertions into helpers as @@ -239,8 +242,10 @@ Each test process should set at least: Do not rely on a developer's user configuration, downloaded server cache, daemon sockets, current shell, or ambient server versions. -The manifest should include stable case data, provisioning metadata, expected capabilities, and -documented exclusions. A validation test should fail when: +The manifest directory should include stable case data, provisioning metadata, expected +capabilities, and documented exclusions. `tests/e2e/cases/suite.yaml` owns global command coverage, +while each `tests/e2e/cases/.yaml` owns one project and all of its server pairs. A +validation test should fail when: - a detectable filetype lacks a project; - a compatible pair lacks a manifest entry; @@ -257,10 +262,10 @@ and compatible pair. ### Extending the manifest -To cover an existing filetype, add its small project under `playground/`, declare it once under -`languages`, then add a `pairs` entry for each compatible server. To introduce a genuinely new -filetype or server, first add its YAML config and commit it in the `data` submodule, then update the -submodule revision and the E2E manifest in this repository. +To cover an existing filetype, add its small project under `playground/`, add one case file named +after the filetype ID, then add a `pairs` entry there for each compatible server. To introduce a +genuinely new filetype or server, first add its YAML config and commit it in the `data` submodule, +then update the submodule revision and the E2E cases in this repository. Pair entries use the LSP YAML filename stem as their stable config ID. The test runner loads the configured user-visible server name for `--lsp`; do not duplicate it in the manifest. Each optional @@ -418,7 +423,7 @@ that class of defect easier to diagnose. - [x] Audit the ten existing playgrounds against the common semantic requirements. - [x] Remove duplicated setup patterns within each class of fixture. - [x] Add CUDA, Kotlin, Objective-C, and Objective-C++ projects. -- [ ] Add `gomod` and `gowork` detection fixtures. +- [x] Add `gomod` and `gowork` detection fixtures. - [ ] Update `playground/README.md` with manual reproduction commands. - [ ] Run each relevant command manually against every new project. diff --git a/playground/gomod/go.mod b/playground/gomod/go.mod new file mode 100644 index 0000000..a37eedc --- /dev/null +++ b/playground/gomod/go.mod @@ -0,0 +1,3 @@ +module example.com/lsp-cli-playground-gomod + +go 1.22 diff --git a/playground/gowork/go.work b/playground/gowork/go.work new file mode 100644 index 0000000..233b100 --- /dev/null +++ b/playground/gowork/go.work @@ -0,0 +1 @@ +go 1.22 diff --git a/tests/e2e.rs b/tests/e2e.rs index 8e0efa0..68c9b61 100644 --- a/tests/e2e.rs +++ b/tests/e2e.rs @@ -7,6 +7,8 @@ reason = "E2E fixtures and assertions fail immediately with contextual expectation messages." )] +#[path = "e2e/case_files.rs"] +mod case_files; #[path = "e2e/catalog.rs"] mod catalog; #[path = "e2e/filesystem.rs"] diff --git a/tests/e2e/case_files.rs b/tests/e2e/case_files.rs new file mode 100644 index 0000000..56c47da --- /dev/null +++ b/tests/e2e/case_files.rs @@ -0,0 +1,33 @@ +use std::fs; +use std::path::{Path, PathBuf}; + +use serde::de::DeserializeOwned; + +pub(crate) fn yaml_paths(directory: &Path) -> Result, String> { + let entries = fs::read_dir(directory) + .map_err(|error| format!("failed to read {}: {error}", directory.display()))?; + let mut paths = entries + .map(|entry| { + entry + .map(|entry| entry.path()) + .map_err(|error| format!("failed to read {}: {error}", directory.display())) + }) + .collect::, _>>()?; + paths.retain(|path| path.extension().and_then(|extension| extension.to_str()) == Some("yaml")); + paths.sort(); + Ok(paths) +} + +pub(crate) fn read_yaml(path: &Path) -> Result { + let contents = fs::read_to_string(path) + .map_err(|error| format!("failed to read {}: {error}", path.display()))?; + serde_yaml::from_str(&contents) + .map_err(|error| format!("failed to parse {}: {error}", path.display())) +} + +pub(crate) fn file_stem(path: &Path) -> Result { + path.file_stem() + .and_then(|stem| stem.to_str()) + .map(str::to_owned) + .ok_or_else(|| format!("{} has no UTF-8 file stem", path.display())) +} diff --git a/tests/e2e/cases/cuda.yaml b/tests/e2e/cases/cuda.yaml new file mode 100644 index 0000000..6f234f3 --- /dev/null +++ b/tests/e2e/cases/cuda.yaml @@ -0,0 +1,4 @@ +language: + id: cuda + kind: source + project: playground/cuda diff --git a/tests/e2e/cases/gomod.yaml b/tests/e2e/cases/gomod.yaml new file mode 100644 index 0000000..83deac0 --- /dev/null +++ b/tests/e2e/cases/gomod.yaml @@ -0,0 +1,4 @@ +language: + id: gomod + kind: metadata + project: playground/gomod diff --git a/tests/e2e/cases/gowork.yaml b/tests/e2e/cases/gowork.yaml new file mode 100644 index 0000000..b6696f0 --- /dev/null +++ b/tests/e2e/cases/gowork.yaml @@ -0,0 +1,4 @@ +language: + id: gowork + kind: metadata + project: playground/gowork diff --git a/tests/e2e/cases/kotlin.yaml b/tests/e2e/cases/kotlin.yaml new file mode 100644 index 0000000..21bd1b4 --- /dev/null +++ b/tests/e2e/cases/kotlin.yaml @@ -0,0 +1,4 @@ +language: + id: kotlin + kind: source + project: playground/kotlin diff --git a/tests/e2e/cases/objc.yaml b/tests/e2e/cases/objc.yaml new file mode 100644 index 0000000..5e0ccdc --- /dev/null +++ b/tests/e2e/cases/objc.yaml @@ -0,0 +1,4 @@ +language: + id: objc + kind: source + project: playground/objc diff --git a/tests/e2e/cases/objcpp.yaml b/tests/e2e/cases/objcpp.yaml new file mode 100644 index 0000000..4e99a5c --- /dev/null +++ b/tests/e2e/cases/objcpp.yaml @@ -0,0 +1,4 @@ +language: + id: objcpp + kind: source + project: playground/objcpp diff --git a/tests/e2e/cases/rust.yaml b/tests/e2e/cases/rust.yaml new file mode 100644 index 0000000..95e624c --- /dev/null +++ b/tests/e2e/cases/rust.yaml @@ -0,0 +1,25 @@ +language: + id: rust + kind: source + project: playground/rust + +pairs: + - language: rust + server: rust_analyzer + smoke: + provision: + method: download + query: + kind: list-symbols + expected-names: + - Order + - OrderItem + - sample_order + - format_order + host-programs: + - name: cargo + resolve: [rustup, which, cargo] + - name: rustc + resolve: [rustup, which, rustc] + lsp-timeout-seconds: 30 + deadline-seconds: 180 diff --git a/tests/e2e/cases.yaml b/tests/e2e/cases/suite.yaml similarity index 59% rename from tests/e2e/cases.yaml rename to tests/e2e/cases/suite.yaml index 4e0a62d..91c2944 100644 --- a/tests/e2e/cases.yaml +++ b/tests/e2e/cases/suite.yaml @@ -26,41 +26,3 @@ commands: - { name: stop-all, strategy: lifecycle } - { name: run, strategy: lifecycle } - { name: update, strategy: update-fixture } - -languages: - - id: cuda - kind: source - project: playground/cuda - - id: kotlin - kind: source - project: playground/kotlin - - id: objc - kind: source - project: playground/objc - - id: objcpp - kind: source - project: playground/objcpp - - id: rust - kind: source - project: playground/rust - -pairs: - - language: rust - server: rust_analyzer - smoke: - provision: - method: download - query: - kind: list-symbols - expected-names: - - Order - - OrderItem - - sample_order - - format_order - host-programs: - - name: cargo - resolve: [rustup, which, cargo] - - name: rustc - resolve: [rustup, which, rustc] - lsp-timeout-seconds: 30 - deadline-seconds: 180 diff --git a/tests/e2e/manifest.rs b/tests/e2e/manifest.rs index d0dd4ed..3d6f723 100644 --- a/tests/e2e/manifest.rs +++ b/tests/e2e/manifest.rs @@ -1,14 +1,14 @@ use std::collections::BTreeSet; -use std::fs; use std::path::{Component, Path, PathBuf}; use serde::Deserialize; -use serde::de::DeserializeOwned; + +use crate::case_files::{file_stem, read_yaml, yaml_paths}; +use crate::repository_root; const MANIFEST_SCHEMA_VERSION: u32 = 2; -#[derive(Clone, Debug, Deserialize)] -#[serde(rename_all = "kebab-case", deny_unknown_fields)] +#[derive(Clone, Debug)] pub(crate) struct Manifest { schema_version: u32, coverage: Coverage, @@ -17,6 +17,22 @@ pub(crate) struct Manifest { pairs: Vec, } +#[derive(Debug, Deserialize)] +#[serde(rename_all = "kebab-case", deny_unknown_fields)] +struct SuiteFile { + schema_version: u32, + coverage: Coverage, + commands: Vec, +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "kebab-case", deny_unknown_fields)] +struct LanguageFile { + language: LanguageCase, + #[serde(default)] + pairs: Vec, +} + #[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq)] #[serde(rename_all = "kebab-case")] enum Coverage { @@ -151,8 +167,34 @@ struct LspConfig { impl Manifest { fn load() -> Result { - serde_yaml::from_str(include_str!("cases.yaml")) - .map_err(|error| format!("failed to parse E2E manifest: {error}")) + Self::load_cases(repository_root()) + } + + fn load_cases(repository: &Path) -> Result { + let directory = repository.join("tests/e2e/cases"); + let suite_path = directory.join("suite.yaml"); + let suite: SuiteFile = read_yaml(&suite_path)?; + let mut languages = Vec::new(); + let mut pairs = Vec::new(); + + for path in yaml_paths(&directory)? { + if path == suite_path { + continue; + } + let case_id = file_stem(&path)?; + let case: LanguageFile = read_yaml(&path)?; + let (language, mut language_pairs) = case.into_parts(&case_id, &path)?; + languages.push(language); + pairs.append(&mut language_pairs); + } + + Ok(Self { + schema_version: suite.schema_version, + coverage: suite.coverage, + commands: suite.commands, + languages, + pairs, + }) } fn validate(&self, repository: &Path) -> Result<(), String> { @@ -181,13 +223,13 @@ impl Manifest { } pub(crate) fn load_validated(repository: &Path) -> Result { - let manifest = Self::load()?; + let manifest = Self::load_cases(repository)?; manifest.validate(repository)?; Ok(manifest) } pub(crate) fn load_repository() -> Result { - Self::load_validated(Path::new(env!("CARGO_MANIFEST_DIR"))) + Self::load_validated(repository_root()) } pub(crate) fn real_server_smoke_cases(&self) -> impl Iterator> { @@ -332,6 +374,35 @@ impl Manifest { } } +impl LanguageFile { + fn into_parts( + self, + case_id: &str, + path: &Path, + ) -> Result<(LanguageCase, Vec), String> { + if self.language.id != case_id { + return Err(format!( + "E2E case filename {case_id:?} does not match language ID {:?} in {}", + self.language.id, + path.display() + )); + } + if let Some(pair) = self + .pairs + .iter() + .find(|pair| pair.language != self.language.id) + { + return Err(format!( + "E2E case {} for language {:?} contains pair for language {:?}", + path.display(), + self.language.id, + pair.language + )); + } + Ok((self.language, self.pairs)) + } +} + impl PairCase { fn key(&self) -> PairKey { PairKey { @@ -497,35 +568,6 @@ fn compatible_pairs( Ok(pairs) } -fn yaml_paths(directory: &Path) -> Result, String> { - let entries = fs::read_dir(directory) - .map_err(|error| format!("failed to read {}: {error}", directory.display()))?; - let mut paths = entries - .map(|entry| { - entry - .map(|entry| entry.path()) - .map_err(|error| format!("failed to read {}: {error}", directory.display())) - }) - .collect::, _>>()?; - paths.retain(|path| path.extension().and_then(|extension| extension.to_str()) == Some("yaml")); - paths.sort(); - Ok(paths) -} - -fn read_yaml(path: &Path) -> Result { - let contents = fs::read_to_string(path) - .map_err(|error| format!("failed to read {}: {error}", path.display()))?; - serde_yaml::from_str(&contents) - .map_err(|error| format!("failed to parse {}: {error}", path.display())) -} - -fn file_stem(path: &Path) -> Result { - path.file_stem() - .and_then(|stem| stem.to_str()) - .map(str::to_owned) - .ok_or_else(|| format!("{} has no UTF-8 file stem", path.display())) -} - fn validate_config_id(kind: &str, value: &str) -> Result<(), String> { let mut components = Path::new(value).components(); if value.is_empty() diff --git a/tests/e2e/manifest_tests.rs b/tests/e2e/manifest_tests.rs index 20045a3..7a50d6d 100644 --- a/tests/e2e/manifest_tests.rs +++ b/tests/e2e/manifest_tests.rs @@ -48,14 +48,42 @@ fn complete_mode_rejects_missing_server_pairs() { #[test] fn manifest_rejects_unknown_fields() { - let error = serde_yaml::from_str::( - "schema-version: 2\ncoverage: partial\ncommands: []\nlanguages: []\npairs: []\nunknown: true\n", + let error = serde_yaml::from_str::( + "schema-version: 2\ncoverage: partial\ncommands: []\nunknown: true\n", ) .expect_err("unknown manifest fields should fail"); assert!(error.to_string().contains("unknown field `unknown`")); } +#[test] +fn language_case_filename_must_match_its_id() { + let case = serde_yaml::from_str::( + "language:\n id: gomod\n kind: metadata\n project: playground/gomod\n", + ) + .expect("language case should parse"); + + let error = case + .into_parts("wrong", Path::new("cases/wrong.yaml")) + .expect_err("case filename should match language ID"); + + assert!(error.contains("does not match language ID")); +} + +#[test] +fn language_case_must_own_its_pairs() { + let case = serde_yaml::from_str::( + "language:\n id: gomod\n kind: metadata\n project: playground/gomod\npairs:\n - language: gowork\n server: gopls\n", + ) + .expect("language case should parse"); + + let error = case + .into_parts("gomod", Path::new("cases/gomod.yaml")) + .expect_err("case should contain only its own language pairs"); + + assert!(error.contains("contains pair for language")); +} + #[test] fn manifest_rejects_duplicate_command_coverage() { let mut manifest = Manifest::load().expect("E2E manifest should parse"); From 5856b10967c18fa9ee742510259cf6a3cd13d319 Mon Sep 17 00:00:00 2001 From: Vasily Kulikov Date: Sat, 5 Sep 2026 21:12:43 +0300 Subject: [PATCH 15/18] Document playground reproduction commands --- E2E_TESTS.md | 2 +- playground/README.md | 161 ++++++++++++++++++++++++++++++++----------- 2 files changed, 120 insertions(+), 43 deletions(-) diff --git a/E2E_TESTS.md b/E2E_TESTS.md index fdf1531..4ea85f1 100644 --- a/E2E_TESTS.md +++ b/E2E_TESTS.md @@ -424,7 +424,7 @@ that class of defect easier to diagnose. - [x] Remove duplicated setup patterns within each class of fixture. - [x] Add CUDA, Kotlin, Objective-C, and Objective-C++ projects. - [x] Add `gomod` and `gowork` detection fixtures. -- [ ] Update `playground/README.md` with manual reproduction commands. +- [x] Update `playground/README.md` with manual reproduction commands. - [ ] Run each relevant command manually against every new project. ### Phase 3: preferred-server smoke matrix diff --git a/playground/README.md b/playground/README.md index df5662a..c98d5ee 100644 --- a/playground/README.md +++ b/playground/README.md @@ -1,51 +1,128 @@ -This directory contains small multi-file projects for manual `lsp-cli` experiments. - -Each language playground is intentionally small but has enough structure to exercise: - -- `detect` -- `list-files` -- `list-symbols` -- `list-functions` -- `grep` -- `definition` -- `declaration` -- `references` -- `callers` -- `callees` - -Suggested commands: - -```sh -cargo run -- detect playground/python -cargo run -- detect playground/python --lang python -cargo run -- detect playground/python --lsp pyright-langserver -cargo run -- detect playground/c --lang c -cargo run -- detect playground/cpp --lang cpp -cargo run -- grep Order playground/rust -cargo run -- list-symbols playground/c -cargo run -- list-symbols playground/java/src/main/java/playground/order/Order.java -cargo run -- definition format_order playground/c -cargo run -- references OrderFormatter playground/csharp -cargo run -- server-capabilities playground/rust --lsp rust-analyzer -cargo run -- daemon playground/python -cargo run -- stop playground/python +# Playground projects + +This directory contains small projects for reproducing `lsp-cli` behavior manually. Run commands +from the repository root. Build the current binary and list the configured servers for a language +before choosing one: + +```sh +cargo build +cargo run -- servers --lang cuda ``` -Check automatic server selection with -`cargo run -- definition format_order playground/c --lang c` or -`cargo run -- definition format_order playground/cpp --lang cpp`. -Server selection follows the configured preferences and server availability. +The examples below use one server name for orientation, not as the only supported choice. Replace +it with any compatible server printed by `servers --lang`. The selected executable must already be +available in `PATH`; remove `--no-download` only when its configuration supports downloading. + +## Source-language workflow + +The newest source projects share stable order-domain symbols: + +| Language | Project | Example server | File | Type | Function | +|---|---|---|---|---|---| +| CUDA | `playground/cuda` | `clangd` | `src/main.cu` | `Order` | `format_order` | +| Kotlin | `playground/kotlin` | `kotlin-language-server` | `src/main/kotlin/playground/App.kt` | `Order` | `formatOrder` | +| Objective-C | `playground/objc` | `clangd` | `src/main.m` | `Order` | `format_order` | +| Objective-C++ | `playground/objcpp` | `clangd` | `src/main.mm` | `Order` | `format_order` | + +Set the values from one row. This CUDA example can be changed to any other row without changing +the commands below: + +```sh +language=cuda +project=playground/cuda +server=clangd +file="$project/src/main.cu" +symbol=Order +function=format_order +``` + +Start with detection and file selection. These commands do not start a daemon: + +```sh +cargo run -- detect "$project" --lang "$language" --lsp "$server" --no-download +cargo run -- list-files "$project" --lang "$language" --lsp "$server" +``` + +Use a small shell helper to run each request directly against the chosen server: + +```sh +lsp_cli() { + cargo run -- "$@" --lang "$language" --lsp "$server" --no-download --no-detach +} + +lsp_cli server-capabilities "$project" +lsp_cli diagnostics "$project" +lsp_cli build-index "$project" +lsp_cli list-symbols "$project" +lsp_cli list-functions "$project" +lsp_cli grep "$symbol" "$project" +lsp_cli definition "$function" "$project" +lsp_cli declaration "$function" "$project" +lsp_cli references "$function" "$project" +lsp_cli callers "$function" "$project" +lsp_cli callees "$function" "$project" +lsp_cli format "$file" --stdout +``` -The projects reuse a similar domain across languages so symbol names are easy to remember -while trying different LSP servers. +`format --stdout` leaves the tracked project unchanged. Formatting, declarations, diagnostics, +workspace symbols, and call hierarchy are optional LSP capabilities. A clear unsupported-capability +error is therefore a useful result when the selected server does not advertise an operation. + +Exercise daemon reuse separately: + +```sh +cargo run -- daemon "$project" --lang "$language" --lsp "$server" --no-download +cargo run -- server-capabilities "$project" --lang "$language" --lsp "$server" --no-download --detach +cargo run -- stop "$project" --lang "$language" --lsp "$server" +``` + +## Go metadata workflow + +The metadata-only fixtures deliberately isolate filename detection: + +| Language | Project | Example server | +|---|---|---| +| Go module metadata | `playground/gomod` | `gopls` | +| Go workspace metadata | `playground/gowork` | `gopls` | + +Set `language` to `gomod` or `gowork`, then use the corresponding project: + +```sh +language=gomod +project=playground/gomod +server=gopls + +cargo run -- servers --lang "$language" +cargo run -- detect "$project" --lang "$language" --lsp "$server" --no-download +cargo run -- list-files "$project" --lang "$language" --lsp "$server" +cargo run -- server-capabilities "$project" --lang "$language" --lsp "$server" --no-download --no-detach +cargo run -- daemon "$project" --lang "$language" --lsp "$server" --no-download +cargo run -- stop "$project" --lang "$language" --lsp "$server" +``` + +These fixtures can exercise detection, file listing, server initialization, and lifecycle. They do +not contain source symbols, so symbol, reference, formatting, diagnostic, and call-hierarchy +commands have no meaningful metadata-only expectation. + +## Existing project examples + +The older playgrounds use the same order domain where it is natural. A few useful starting points +are: + +```sh +cargo run -- grep Order playground/rust --lsp rust-analyzer --no-detach +cargo run -- list-symbols playground/java/src/main/java/playground/order/Order.java +cargo run -- definition format_order playground/c --lang c +cargo run -- references OrderFormatter playground/csharp +``` -The Lua playground exercises discovery of the local `normalize_timestamp` function, -which may be absent from LuaLS workspace-symbol results: +The Lua playground instead exercises discovery of the local `normalize_timestamp` function, which +may be absent from LuaLS workspace-symbol results: ```sh cargo run -- references normalize_timestamp playground/lua --lsp lua-language-server --detach ``` -Compare runs with `max-requests-in-flight: 1` and `max-requests-in-flight: 20` in -`lsp-cli.yaml`. Both should report the call in `timestamp.lua`; the setting changes -how many file-symbol requests can be outstanding, not which files are searched. +Compare runs with `max-requests-in-flight: 1` and `max-requests-in-flight: 20` in `lsp-cli.yaml`. +Both should report the call in `timestamp.lua`; the setting changes how many file-symbol requests +can be outstanding, not which files are searched. From 8ff22e51fd1dd1e084dffe81a65ffc33b459fe6b Mon Sep 17 00:00:00 2001 From: Vasily Kulikov Date: Sat, 5 Sep 2026 22:04:46 +0300 Subject: [PATCH 16/18] Run manual E2E checks after provisioning --- .../manual-e2e-verification-follows-provisioning.md | 6 ++++++ E2E_TESTS.md | 5 ++++- 2 files changed, 10 insertions(+), 1 deletion(-) create mode 100644 .memory/unprocessed/manual-e2e-verification-follows-provisioning.md diff --git a/.memory/unprocessed/manual-e2e-verification-follows-provisioning.md b/.memory/unprocessed/manual-e2e-verification-follows-provisioning.md new file mode 100644 index 0000000..bf60b5a --- /dev/null +++ b/.memory/unprocessed/manual-e2e-verification-follows-provisioning.md @@ -0,0 +1,6 @@ +# Manual E2E verification follows server provisioning + +The project plan originally required running every relevant LSP command against new playgrounds +before selecting or provisioning their servers. Most new languages had no runnable compatible +server, so the manual verification item was moved immediately after preferred-server provisioning +and remains unchecked until those prerequisites exist. diff --git a/E2E_TESTS.md b/E2E_TESTS.md index 4ea85f1..5652934 100644 --- a/E2E_TESTS.md +++ b/E2E_TESTS.md @@ -425,12 +425,15 @@ that class of defect easier to diagnose. - [x] Add CUDA, Kotlin, Objective-C, and Objective-C++ projects. - [x] Add `gomod` and `gowork` detection fixtures. - [x] Update `playground/README.md` with manual reproduction commands. -- [ ] Run each relevant command manually against every new project. ### Phase 3: preferred-server smoke matrix +Manual LSP verification follows server selection and provisioning so it runs against reproducible, +runnable servers rather than ambient or unpinned installations. + - [ ] Select and pin one preferred server for each source language. - [ ] Add provisioning scripts without new Rust dependencies. +- [ ] Run each relevant command manually against every new project. - [ ] Implement capability-aware query assertions. - [ ] Implement direct/detached lifecycle scenarios. - [ ] Add the pull-request E2E job. From b100fa3a233f9388198f7b7308896e536935a25a Mon Sep 17 00:00:00 2001 From: Vasily Kulikov Date: Sat, 5 Sep 2026 22:21:52 +0300 Subject: [PATCH 17/18] Pin preferred E2E server matrix --- .../official-first-e2e-server-pins.md | 10 ++ E2E_TESTS.md | 31 ++++- GOTCHAS.md | 15 +++ tests/e2e.rs | 2 + tests/e2e/cases/c.yaml | 10 ++ tests/e2e/cases/cpp.yaml | 10 ++ tests/e2e/cases/cs.yaml | 10 ++ tests/e2e/cases/cuda.yaml | 6 + tests/e2e/cases/go.yaml | 10 ++ tests/e2e/cases/java.yaml | 10 ++ tests/e2e/cases/javascript.yaml | 10 ++ tests/e2e/cases/kotlin.yaml | 6 + tests/e2e/cases/lua.yaml | 10 ++ tests/e2e/cases/objc.yaml | 6 + tests/e2e/cases/objcpp.yaml | 6 + tests/e2e/cases/python.yaml | 10 ++ tests/e2e/cases/rust.yaml | 2 + tests/e2e/cases/suite.yaml | 2 +- tests/e2e/cases/typescript.yaml | 10 ++ tests/e2e/manifest.rs | 117 ++++++++-------- tests/e2e/manifest_data.rs | 64 +++++++++ tests/e2e/manifest_tests.rs | 126 +++++++++++++++++- 22 files changed, 416 insertions(+), 67 deletions(-) create mode 100644 .memory/unprocessed/official-first-e2e-server-pins.md create mode 100644 tests/e2e/cases/c.yaml create mode 100644 tests/e2e/cases/cpp.yaml create mode 100644 tests/e2e/cases/cs.yaml create mode 100644 tests/e2e/cases/go.yaml create mode 100644 tests/e2e/cases/java.yaml create mode 100644 tests/e2e/cases/javascript.yaml create mode 100644 tests/e2e/cases/lua.yaml create mode 100644 tests/e2e/cases/python.yaml create mode 100644 tests/e2e/cases/typescript.yaml create mode 100644 tests/e2e/manifest_data.rs diff --git a/.memory/unprocessed/official-first-e2e-server-pins.md b/.memory/unprocessed/official-first-e2e-server-pins.md new file mode 100644 index 0000000..574b294 --- /dev/null +++ b/.memory/unprocessed/official-first-e2e-server-pins.md @@ -0,0 +1,10 @@ +# Official-first E2E server pins expose launcher mismatches + +The user selected an official-first preferred E2E server matrix and explicitly accepted bespoke +provisioning and current command/config gaps. In particular, use `kotlin_lsp` rather than the +community `kotlin_language_server`, and `roslyn_ls` rather than OmniSharp. + +The pinned Mason registry exposes Kotlin LSP as `intellij-server`, although the data config starts +`kotlin-lsp`. It exposes Roslyn as `roslyn-language-server`, although the data config starts `dotnet` +with a literal `` DLL path. The preferred-server selection can be committed independently, +but provisioning must resolve these mismatches before claiming either smoke case is runnable. diff --git a/E2E_TESTS.md b/E2E_TESTS.md index 5652934..1444b63 100644 --- a/E2E_TESTS.md +++ b/E2E_TESTS.md @@ -254,7 +254,8 @@ validation test should fail when: - two cases select the same user-visible server ambiguously; - a new top-level subcommand has no assigned coverage class. -The version 2 manifest assigns every canonical command to a coverage strategy and keeps +The version 3 manifest assigns every canonical command to a coverage strategy, selects one pinned +preferred server for every source-language project, and keeps `coverage: partial`, which validates every declared language/server entry against the pinned data without requiring unfinished matrix entries. Phase 4 adds the remaining entries and switches it to `coverage: complete`; complete mode enforces every detectable language @@ -268,12 +269,36 @@ genuinely new filetype or server, first add its YAML config and commit it in the then update the submodule revision and the E2E cases in this repository. Pair entries use the LSP YAML filename stem as their stable config ID. The test runner loads the -configured user-visible server name for `--lsp`; do not duplicate it in the manifest. Each optional +configured user-visible server name for `--lsp`; do not duplicate it in the manifest. A +`preferred` block marks a source language's merge-gate server and records its exact version. Every +source language must have exactly one; metadata-only filetypes must not have one. Each optional `smoke` block declares a generic provisioning method, query kind, semantic expectations, runtime host programs, and deadlines. Language-specific prerequisites and expected symbols belong in YAML, not in the Rust runner. The first provisioning method is `download`; add other mechanisms as typed methods when needed instead of branching on server names. +### Preferred server pins + +The initial pins come from Mason registry release `2026-09-05-weary-okapi`, whose registry archive +digest is `sha256:ecbd69b9f967754250413ac92dc9b0301d6b95699882554b234e2f832ca2b7f0`. +Versions are recorded per pair because multiple languages may deliberately reuse one server. + +| Languages | LSP config ID | Pinned version | +| --- | --- | --- | +| C, C++, CUDA, Objective-C, Objective-C++ | `clangd` | `22.1.6` | +| C# | `roslyn_ls` | `5.11.0-1.26380.4` | +| Go | `gopls` | `v0.23.0` | +| Java | `jdtls` | `v1.60.0` | +| JavaScript, TypeScript | `ts_ls` | `6.0.0` | +| Kotlin | `kotlin_lsp` | `kotlin-lsp/v262.9593.0` | +| Lua | `lua_ls` | `3.19.1` | +| Python | `pyright` | `1.1.413` | +| Rust | `rust_analyzer` | `2026-08-31` | + +These selections are manifest data, not language-specific runner branches. The provisioning phase +must consume the exact versions rather than treating them as documentation or requesting the +registry's current version. + In `coverage: complete` mode, manifest validation makes a new detectable filetype or compatible filetype/server relationship fail until its project and pair are declared. Partial mode intentionally allows the matrix to grow incrementally. @@ -431,7 +456,7 @@ that class of defect easier to diagnose. Manual LSP verification follows server selection and provisioning so it runs against reproducible, runnable servers rather than ambient or unpinned installations. -- [ ] Select and pin one preferred server for each source language. +- [x] Select and pin one preferred server for each source language. - [ ] Add provisioning scripts without new Rust dependencies. - [ ] Run each relevant command manually against every new project. - [ ] Implement capability-aware query assertions. diff --git a/GOTCHAS.md b/GOTCHAS.md index 651c1c4..9fd8f3d 100644 --- a/GOTCHAS.md +++ b/GOTCHAS.md @@ -67,6 +67,21 @@ # LSP server implementations +## kotlin-lsp + +- The `kotlin_lsp` data config starts `kotlin-lsp --stdio`, while the Mason package currently + exposes its launcher as `intellij-server`. Selecting the official server is therefore possible, + but `lsp-cli --download` cannot launch it without a data-config correction or a provisioning + alias. Do not treat the preferred-server pin as a runnable smoke case until that mismatch is + resolved. + +## roslyn-language-server + +- The `roslyn_ls` data config starts `dotnet` with a literal `` DLL placeholder, while + the Mason package exposes a `roslyn-language-server` launcher. The official C# server needs a + concrete installed DLL path or corrected launcher command before it can participate in automated + smoke tests. + ## lua-language-server - Opening every source file for symbol discovery also triggers diagnostics. The installed diff --git a/tests/e2e.rs b/tests/e2e.rs index 68c9b61..3c1ed29 100644 --- a/tests/e2e.rs +++ b/tests/e2e.rs @@ -23,6 +23,8 @@ mod lifecycle; mod local_fixture; #[path = "e2e/manifest.rs"] mod manifest; +#[path = "e2e/manifest_data.rs"] +mod manifest_data; #[path = "e2e/process.rs"] mod process; #[path = "e2e/queries.rs"] diff --git a/tests/e2e/cases/c.yaml b/tests/e2e/cases/c.yaml new file mode 100644 index 0000000..bf8d59d --- /dev/null +++ b/tests/e2e/cases/c.yaml @@ -0,0 +1,10 @@ +language: + id: c + kind: source + project: playground/c + +pairs: + - language: c + server: clangd + preferred: + version: "22.1.6" diff --git a/tests/e2e/cases/cpp.yaml b/tests/e2e/cases/cpp.yaml new file mode 100644 index 0000000..291d383 --- /dev/null +++ b/tests/e2e/cases/cpp.yaml @@ -0,0 +1,10 @@ +language: + id: cpp + kind: source + project: playground/cpp + +pairs: + - language: cpp + server: clangd + preferred: + version: "22.1.6" diff --git a/tests/e2e/cases/cs.yaml b/tests/e2e/cases/cs.yaml new file mode 100644 index 0000000..5e5c8ca --- /dev/null +++ b/tests/e2e/cases/cs.yaml @@ -0,0 +1,10 @@ +language: + id: cs + kind: source + project: playground/csharp + +pairs: + - language: cs + server: roslyn_ls + preferred: + version: "5.11.0-1.26380.4" diff --git a/tests/e2e/cases/cuda.yaml b/tests/e2e/cases/cuda.yaml index 6f234f3..8c6297a 100644 --- a/tests/e2e/cases/cuda.yaml +++ b/tests/e2e/cases/cuda.yaml @@ -2,3 +2,9 @@ language: id: cuda kind: source project: playground/cuda + +pairs: + - language: cuda + server: clangd + preferred: + version: "22.1.6" diff --git a/tests/e2e/cases/go.yaml b/tests/e2e/cases/go.yaml new file mode 100644 index 0000000..2058954 --- /dev/null +++ b/tests/e2e/cases/go.yaml @@ -0,0 +1,10 @@ +language: + id: go + kind: source + project: playground/go + +pairs: + - language: go + server: gopls + preferred: + version: "v0.23.0" diff --git a/tests/e2e/cases/java.yaml b/tests/e2e/cases/java.yaml new file mode 100644 index 0000000..1ee5b80 --- /dev/null +++ b/tests/e2e/cases/java.yaml @@ -0,0 +1,10 @@ +language: + id: java + kind: source + project: playground/java + +pairs: + - language: java + server: jdtls + preferred: + version: "v1.60.0" diff --git a/tests/e2e/cases/javascript.yaml b/tests/e2e/cases/javascript.yaml new file mode 100644 index 0000000..ab281b2 --- /dev/null +++ b/tests/e2e/cases/javascript.yaml @@ -0,0 +1,10 @@ +language: + id: javascript + kind: source + project: playground/js + +pairs: + - language: javascript + server: ts_ls + preferred: + version: "6.0.0" diff --git a/tests/e2e/cases/kotlin.yaml b/tests/e2e/cases/kotlin.yaml index 21bd1b4..a036066 100644 --- a/tests/e2e/cases/kotlin.yaml +++ b/tests/e2e/cases/kotlin.yaml @@ -2,3 +2,9 @@ language: id: kotlin kind: source project: playground/kotlin + +pairs: + - language: kotlin + server: kotlin_lsp + preferred: + version: "kotlin-lsp/v262.9593.0" diff --git a/tests/e2e/cases/lua.yaml b/tests/e2e/cases/lua.yaml new file mode 100644 index 0000000..83b0a47 --- /dev/null +++ b/tests/e2e/cases/lua.yaml @@ -0,0 +1,10 @@ +language: + id: lua + kind: source + project: playground/lua + +pairs: + - language: lua + server: lua_ls + preferred: + version: "3.19.1" diff --git a/tests/e2e/cases/objc.yaml b/tests/e2e/cases/objc.yaml index 5e0ccdc..2c4a244 100644 --- a/tests/e2e/cases/objc.yaml +++ b/tests/e2e/cases/objc.yaml @@ -2,3 +2,9 @@ language: id: objc kind: source project: playground/objc + +pairs: + - language: objc + server: clangd + preferred: + version: "22.1.6" diff --git a/tests/e2e/cases/objcpp.yaml b/tests/e2e/cases/objcpp.yaml index 4e99a5c..25d8585 100644 --- a/tests/e2e/cases/objcpp.yaml +++ b/tests/e2e/cases/objcpp.yaml @@ -2,3 +2,9 @@ language: id: objcpp kind: source project: playground/objcpp + +pairs: + - language: objcpp + server: clangd + preferred: + version: "22.1.6" diff --git a/tests/e2e/cases/python.yaml b/tests/e2e/cases/python.yaml new file mode 100644 index 0000000..edcbd0f --- /dev/null +++ b/tests/e2e/cases/python.yaml @@ -0,0 +1,10 @@ +language: + id: python + kind: source + project: playground/python + +pairs: + - language: python + server: pyright + preferred: + version: "1.1.413" diff --git a/tests/e2e/cases/rust.yaml b/tests/e2e/cases/rust.yaml index 95e624c..abb8979 100644 --- a/tests/e2e/cases/rust.yaml +++ b/tests/e2e/cases/rust.yaml @@ -6,6 +6,8 @@ language: pairs: - language: rust server: rust_analyzer + preferred: + version: "2026-08-31" smoke: provision: method: download diff --git a/tests/e2e/cases/suite.yaml b/tests/e2e/cases/suite.yaml index 91c2944..d3917be 100644 --- a/tests/e2e/cases/suite.yaml +++ b/tests/e2e/cases/suite.yaml @@ -1,4 +1,4 @@ -schema-version: 2 +schema-version: 3 coverage: partial commands: diff --git a/tests/e2e/cases/typescript.yaml b/tests/e2e/cases/typescript.yaml new file mode 100644 index 0000000..a42b59f --- /dev/null +++ b/tests/e2e/cases/typescript.yaml @@ -0,0 +1,10 @@ +language: + id: typescript + kind: source + project: playground/typescript + +pairs: + - language: typescript + server: ts_ls + preferred: + version: "6.0.0" diff --git a/tests/e2e/manifest.rs b/tests/e2e/manifest.rs index 3d6f723..b4d885f 100644 --- a/tests/e2e/manifest.rs +++ b/tests/e2e/manifest.rs @@ -4,9 +4,12 @@ use std::path::{Component, Path, PathBuf}; use serde::Deserialize; use crate::case_files::{file_stem, read_yaml, yaml_paths}; +use crate::manifest_data::{ + FiletypeConfig, LspConfig, PairKey, compatible_pairs, detectable_languages, +}; use crate::repository_root; -const MANIFEST_SCHEMA_VERSION: u32 = 2; +const MANIFEST_SCHEMA_VERSION: u32 = 3; #[derive(Clone, Debug)] pub(crate) struct Manifest { @@ -65,7 +68,7 @@ struct LanguageCase { project: PathBuf, } -#[derive(Clone, Copy, Debug, Deserialize)] +#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq)] #[serde(rename_all = "kebab-case")] enum ProjectKind { Source, @@ -86,13 +89,14 @@ impl ProjectKind { struct PairCase { language: String, server: String, + preferred: Option, smoke: Option, } -#[derive(Clone, Debug, Eq, Ord, PartialEq, PartialOrd)] -struct PairKey { - language: String, - server: String, +#[derive(Clone, Debug, Deserialize)] +#[serde(rename_all = "kebab-case", deny_unknown_fields)] +struct PreferredServer { + version: String, } #[derive(Clone, Debug, Deserialize)] @@ -144,27 +148,6 @@ pub(crate) struct RealServerCase<'a> { smoke: &'a SmokeCase, } -#[derive(Deserialize)] -struct FiletypeConfig { - #[serde(default)] - extensions: Vec, - #[serde(default)] - patterns: Vec, -} - -impl FiletypeConfig { - fn is_detectable(&self) -> bool { - !self.extensions.is_empty() || !self.patterns.is_empty() - } -} - -#[derive(Deserialize)] -struct LspConfig { - #[serde(default)] - filetypes: Vec, - name: String, -} - impl Manifest { fn load() -> Result { Self::load_cases(repository_root()) @@ -215,6 +198,7 @@ impl Manifest { let data = repository.join("data"); let declared_languages = self.validate_languages(repository, &data)?; let declared_pairs = self.validate_pairs(&data, &declared_languages)?; + self.validate_preferred_servers()?; if self.coverage == Coverage::Complete { Self::validate_complete_coverage(&data, &declared_languages, &declared_pairs)?; @@ -338,10 +322,39 @@ impl Manifest { if let Some(smoke) = &pair.smoke { smoke.validate(pair)?; } + if let Some(preferred) = &pair.preferred { + preferred.validate(pair)?; + } } Ok(declared) } + fn validate_preferred_servers(&self) -> Result<(), String> { + for language in &self.languages { + let count = self + .pairs + .iter() + .filter(|pair| pair.language == language.id && pair.preferred.is_some()) + .count(); + match (language.kind, count) { + (ProjectKind::Source, 1) | (ProjectKind::Metadata, 0) => {} + (ProjectKind::Source, _) => { + return Err(format!( + "E2E source language {:?} must select exactly one preferred server; found {count}", + language.id + )); + } + (ProjectKind::Metadata, _) => { + return Err(format!( + "E2E metadata language {:?} must not select a preferred server", + language.id + )); + } + } + } + Ok(()) + } + fn validate_complete_coverage( data: &Path, declared_languages: &BTreeSet, @@ -412,6 +425,25 @@ impl PairCase { } } +impl PreferredServer { + fn validate(&self, pair: &PairCase) -> Result<(), String> { + let version = self.version.trim(); + if version.is_empty() || version != self.version { + return Err(format!( + "preferred E2E server {}/{} must have a non-empty, trimmed version", + pair.language, pair.server + )); + } + if version.eq_ignore_ascii_case("latest") || version.eq_ignore_ascii_case("stable") { + return Err(format!( + "preferred E2E server {}/{} must use an exact version instead of {:?}", + pair.language, pair.server, self.version + )); + } + Ok(()) + } +} + impl SmokeCase { fn validate(&self, pair: &PairCase) -> Result<(), String> { let label = format!("{}/{}", pair.language, pair.server); @@ -537,37 +569,6 @@ impl LanguageCase { } } -fn detectable_languages(data: &Path) -> Result, String> { - let mut languages = BTreeSet::new(); - for path in yaml_paths(&data.join("filetypes"))? { - let config: FiletypeConfig = read_yaml(&path)?; - if config.is_detectable() { - languages.insert(file_stem(&path)?); - } - } - Ok(languages) -} - -fn compatible_pairs( - data: &Path, - detectable: &BTreeSet, -) -> Result, String> { - let mut pairs = BTreeSet::new(); - for path in yaml_paths(&data.join("lsp"))? { - let config: LspConfig = read_yaml(&path)?; - let server = file_stem(&path)?; - for language in config.filetypes { - if detectable.contains(&language) { - pairs.insert(PairKey { - language, - server: server.clone(), - }); - } - } - } - Ok(pairs) -} - fn validate_config_id(kind: &str, value: &str) -> Result<(), String> { let mut components = Path::new(value).components(); if value.is_empty() diff --git a/tests/e2e/manifest_data.rs b/tests/e2e/manifest_data.rs new file mode 100644 index 0000000..102deac --- /dev/null +++ b/tests/e2e/manifest_data.rs @@ -0,0 +1,64 @@ +use std::collections::BTreeSet; +use std::path::Path; + +use serde::Deserialize; + +use crate::case_files::{file_stem, read_yaml, yaml_paths}; + +#[derive(Clone, Debug, Eq, Ord, PartialEq, PartialOrd)] +pub(crate) struct PairKey { + pub(crate) language: String, + pub(crate) server: String, +} + +#[derive(Deserialize)] +pub(crate) struct FiletypeConfig { + #[serde(default)] + extensions: Vec, + #[serde(default)] + patterns: Vec, +} + +impl FiletypeConfig { + pub(crate) fn is_detectable(&self) -> bool { + !self.extensions.is_empty() || !self.patterns.is_empty() + } +} + +#[derive(Deserialize)] +pub(crate) struct LspConfig { + #[serde(default)] + pub(crate) filetypes: Vec, + pub(crate) name: String, +} + +pub(crate) fn detectable_languages(data: &Path) -> Result, String> { + let mut languages = BTreeSet::new(); + for path in yaml_paths(&data.join("filetypes"))? { + let config: FiletypeConfig = read_yaml(&path)?; + if config.is_detectable() { + languages.insert(file_stem(&path)?); + } + } + Ok(languages) +} + +pub(crate) fn compatible_pairs( + data: &Path, + detectable: &BTreeSet, +) -> Result, String> { + let mut pairs = BTreeSet::new(); + for path in yaml_paths(&data.join("lsp"))? { + let config: LspConfig = read_yaml(&path)?; + let server = file_stem(&path)?; + for language in config.filetypes { + if detectable.contains(&language) { + pairs.insert(PairKey { + language, + server: server.clone(), + }); + } + } + } + Ok(pairs) +} diff --git a/tests/e2e/manifest_tests.rs b/tests/e2e/manifest_tests.rs index 7a50d6d..9f11bdf 100644 --- a/tests/e2e/manifest_tests.rs +++ b/tests/e2e/manifest_tests.rs @@ -4,11 +4,20 @@ use crate::repository_root; fn first_smoke(manifest: &mut Manifest) -> &mut SmokeCase { manifest .pairs - .first_mut() - .expect("manifest should contain a pair") + .iter_mut() + .find(|pair| pair.smoke.is_some()) + .expect("manifest should contain a smoke pair") .smoke .as_mut() - .expect("first pair should have a smoke case") + .expect("selected pair should have a smoke case") +} + +fn preferred_pair_mut<'a>(manifest: &'a mut Manifest, language: &str) -> &'a mut PairCase { + manifest + .pairs + .iter_mut() + .find(|pair| pair.language == language && pair.preferred.is_some()) + .expect("language should have a preferred pair") } #[test] @@ -19,6 +28,50 @@ fn partial_manifest_matches_pinned_data() { .expect("E2E manifest should be valid"); } +#[test] +fn source_languages_select_the_approved_preferred_servers() { + let manifest = Manifest::load().expect("E2E manifest should parse"); + let expected = [ + ("c", "clangd", "22.1.6"), + ("cpp", "clangd", "22.1.6"), + ("cs", "roslyn_ls", "5.11.0-1.26380.4"), + ("cuda", "clangd", "22.1.6"), + ("go", "gopls", "v0.23.0"), + ("java", "jdtls", "v1.60.0"), + ("javascript", "ts_ls", "6.0.0"), + ("kotlin", "kotlin_lsp", "kotlin-lsp/v262.9593.0"), + ("lua", "lua_ls", "3.19.1"), + ("objc", "clangd", "22.1.6"), + ("objcpp", "clangd", "22.1.6"), + ("python", "pyright", "1.1.413"), + ("rust", "rust_analyzer", "2026-08-31"), + ("typescript", "ts_ls", "6.0.0"), + ]; + + for (language, server, version) in expected { + let pair = manifest + .pairs + .iter() + .find(|pair| pair.language == language && pair.preferred.is_some()) + .expect("source language should have a preferred pair"); + assert_eq!(pair.server, server); + assert_eq!( + pair.preferred + .as_ref() + .expect("pair should be preferred") + .version, + version + ); + } + + let preferred_count = manifest + .pairs + .iter() + .filter(|pair| pair.preferred.is_some()) + .count(); + assert_eq!(preferred_count, expected.len()); +} + #[test] fn complete_mode_rejects_the_partial_matrix() { let mut manifest = Manifest::load().expect("E2E manifest should parse"); @@ -28,7 +81,7 @@ fn complete_mode_rejects_the_partial_matrix() { .validate(repository_root()) .expect_err("partial matrix should not satisfy complete coverage"); - assert!(error.contains("complete E2E manifest is missing languages")); + assert!(error.contains("complete E2E manifest is missing pairs")); } #[test] @@ -49,13 +102,76 @@ fn complete_mode_rejects_missing_server_pairs() { #[test] fn manifest_rejects_unknown_fields() { let error = serde_yaml::from_str::( - "schema-version: 2\ncoverage: partial\ncommands: []\nunknown: true\n", + "schema-version: 3\ncoverage: partial\ncommands: []\nunknown: true\n", ) .expect_err("unknown manifest fields should fail"); assert!(error.to_string().contains("unknown field `unknown`")); } +#[test] +fn manifest_rejects_a_source_language_without_a_preferred_server() { + let mut manifest = Manifest::load().expect("E2E manifest should parse"); + preferred_pair_mut(&mut manifest, "c").preferred = None; + + let error = manifest + .validate(repository_root()) + .expect_err("source language should require one preferred server"); + + assert!(error.contains("must select exactly one preferred server; found 0")); +} + +#[test] +fn manifest_rejects_multiple_preferred_servers_for_one_language() { + let mut manifest = Manifest::load().expect("E2E manifest should parse"); + let mut duplicate = preferred_pair_mut(&mut manifest, "c").clone(); + duplicate.server = "ccls".to_string(); + manifest.pairs.push(duplicate); + + let error = manifest + .validate(repository_root()) + .expect_err("source language should have only one preferred server"); + + assert!(error.contains("must select exactly one preferred server; found 2")); +} + +#[test] +fn manifest_rejects_a_preferred_server_for_metadata() { + let mut manifest = Manifest::load().expect("E2E manifest should parse"); + manifest.pairs.push(PairCase { + language: "gomod".to_string(), + server: "gopls".to_string(), + preferred: Some(PreferredServer { + version: "v0.23.0".to_string(), + }), + smoke: None, + }); + + let error = manifest + .validate(repository_root()) + .expect_err("metadata language should not select a preferred server"); + + assert!(error.contains("must not select a preferred server")); +} + +#[test] +fn manifest_rejects_non_exact_preferred_versions() { + for version in ["", " latest", "latest", "stable"] { + let mut manifest = Manifest::load().expect("E2E manifest should parse"); + preferred_pair_mut(&mut manifest, "c") + .preferred + .as_mut() + .expect("pair should be preferred") + .version = version.to_string(); + + let error = manifest + .validate(repository_root()) + .expect_err("preferred version should be exact"); + + assert!(error.contains("preferred E2E server c/clangd")); + } +} + #[test] fn language_case_filename_must_match_its_id() { let case = serde_yaml::from_str::( From 364e0dd19fff75671d678efd0e5d49b8f1a467b5 Mon Sep 17 00:00:00 2001 From: Vasily Kulikov Date: Sat, 5 Sep 2026 22:49:21 +0300 Subject: [PATCH 18/18] Provision E2E servers through Mason downloads --- .../e2e-servers-use-mason-latest.md | 9 ++ E2E_TESTS.md | 109 +++++++-------- GOTCHAS.md | 15 +- data | 2 +- src/mason/install.rs | 56 +++++++- src/mason/install/tests.rs | 69 +++++++++- src/mason/link.rs | 10 ++ src/mason/link/tests.rs | 27 +++- src/mason/source.rs | 16 +++ src/test_support.rs | 22 +++ tests/e2e/cases/c.yaml | 2 - tests/e2e/cases/cpp.yaml | 2 - tests/e2e/cases/cs.yaml | 2 - tests/e2e/cases/cuda.yaml | 2 - tests/e2e/cases/go.yaml | 2 - tests/e2e/cases/java.yaml | 2 - tests/e2e/cases/javascript.yaml | 2 - tests/e2e/cases/kotlin.yaml | 2 - tests/e2e/cases/lua.yaml | 2 - tests/e2e/cases/objc.yaml | 2 - tests/e2e/cases/objcpp.yaml | 2 - tests/e2e/cases/python.yaml | 2 - tests/e2e/cases/rust.yaml | 2 - tests/e2e/cases/suite.yaml | 2 +- tests/e2e/cases/typescript.yaml | 2 - tests/e2e/manifest.rs | 73 +++------- tests/e2e/manifest_data.rs | 50 ++++++- tests/e2e/manifest_tests.rs | 130 +++++------------- 28 files changed, 367 insertions(+), 251 deletions(-) create mode 100644 .memory/unprocessed/e2e-servers-use-mason-latest.md diff --git a/.memory/unprocessed/e2e-servers-use-mason-latest.md b/.memory/unprocessed/e2e-servers-use-mason-latest.md new file mode 100644 index 0000000..852526f --- /dev/null +++ b/.memory/unprocessed/e2e-servers-use-mason-latest.md @@ -0,0 +1,9 @@ +# Real-server E2E provisioning uses Mason latest + +The user corrected the preferred-server E2E design: do not install LSP servers separately and do +not pin a Mason registry release or package versions. Each real-server test must use `--download` +against Mason latest. The preferred smoke pair must be derived from the first production preference +in `data/lsp-cli.yaml`; do not duplicate that selection in the E2E case YAML. + +This deliberately trades reproducibility for immediate upstream compatibility coverage. Preserve +the resolved Mason source ID in diagnostics so failures can still identify the installed version. diff --git a/E2E_TESTS.md b/E2E_TESTS.md index 1444b63..8bf88b9 100644 --- a/E2E_TESTS.md +++ b/E2E_TESTS.md @@ -11,7 +11,7 @@ server lifecycle, and semantically relevant LSP results. They must not depend on ## Working definition of supported -At pinned `lsp-cli-data` revision `013a75f6412917b710aa4683b9c5761c0c679975`, the data tree has: +At pinned `lsp-cli-data` revision `59ea88365855ca6a5ab35715c931d30c734e1b6e`, the data tree has: - 362 filetype configurations; - 362 LSP configurations; @@ -254,8 +254,8 @@ validation test should fail when: - two cases select the same user-visible server ambiguously; - a new top-level subcommand has no assigned coverage class. -The version 3 manifest assigns every canonical command to a coverage strategy, selects one pinned -preferred server for every source-language project, and keeps +The version 4 manifest assigns every canonical command to a coverage strategy, derives one +preferred smoke-matrix server for every source-language project from `data/lsp-cli.yaml`, and keeps `coverage: partial`, which validates every declared language/server entry against the pinned data without requiring unfinished matrix entries. Phase 4 adds the remaining entries and switches it to `coverage: complete`; complete mode enforces every detectable language @@ -269,35 +269,34 @@ genuinely new filetype or server, first add its YAML config and commit it in the then update the submodule revision and the E2E cases in this repository. Pair entries use the LSP YAML filename stem as their stable config ID. The test runner loads the -configured user-visible server name for `--lsp`; do not duplicate it in the manifest. A -`preferred` block marks a source language's merge-gate server and records its exact version. Every -source language must have exactly one; metadata-only filetypes must not have one. Each optional +configured user-visible server name for `--lsp`; do not duplicate it in the manifest. The first +server in each source language's production preference list is also its merge-gate smoke server. +Manifest validation resolves that user-visible name to one compatible LSP config and requires the +corresponding pair to exist. Each optional `smoke` block declares a generic provisioning method, query kind, semantic expectations, runtime host programs, and deadlines. Language-specific prerequisites and expected symbols belong in YAML, not in the Rust runner. The first provisioning method is `download`; add other mechanisms as typed methods when needed instead of branching on server names. -### Preferred server pins - -The initial pins come from Mason registry release `2026-09-05-weary-okapi`, whose registry archive -digest is `sha256:ecbd69b9f967754250413ac92dc9b0301d6b95699882554b234e2f832ca2b7f0`. -Versions are recorded per pair because multiple languages may deliberately reuse one server. - -| Languages | LSP config ID | Pinned version | -| --- | --- | --- | -| C, C++, CUDA, Objective-C, Objective-C++ | `clangd` | `22.1.6` | -| C# | `roslyn_ls` | `5.11.0-1.26380.4` | -| Go | `gopls` | `v0.23.0` | -| Java | `jdtls` | `v1.60.0` | -| JavaScript, TypeScript | `ts_ls` | `6.0.0` | -| Kotlin | `kotlin_lsp` | `kotlin-lsp/v262.9593.0` | -| Lua | `lua_ls` | `3.19.1` | -| Python | `pyright` | `1.1.413` | -| Rust | `rust_analyzer` | `2026-08-31` | - -These selections are manifest data, not language-specific runner branches. The provisioning phase -must consume the exact versions rather than treating them as documentation or requesting the -registry's current version. +### Preferred server matrix + +| Languages | LSP config ID | +| --- | --- | +| C, C++, CUDA, Objective-C, Objective-C++ | `clangd` | +| C# | `roslyn_ls` | +| Go | `gopls` | +| Java | `jdtls` | +| JavaScript, TypeScript | `ts_ls` | +| Kotlin | `kotlin_lsp` | +| Lua | `lua_ls` | +| Python | `pyright` | +| Rust | `rust_analyzer` | + +These selections come from the shipped production preferences, not duplicated manifest flags or +language-specific runner branches. Real-server tests use `--download` with a clean isolated home, +so Mason's current registry release selects and installs the server version on every run. Failure +diagnostics must retain the resolved package source ID so an upstream version change can be +identified after the fact. In `coverage: complete` mode, manifest validation makes a new detectable filetype or compatible filetype/server relationship fail until its project and pair are declared. Partial mode intentionally @@ -346,27 +345,26 @@ completion from a fixed sleep. ## Real-server provisioning -Pin every server and required toolchain version. The manifest should distinguish: +Do not install LSP servers separately. Every real-server case must pass `--download`, allowing the +production Mason integration to select the current registry package, install it inside the case's +isolated home, and return the resolved executable. This applies uniformly to direct archives and +npm, PyPI, Cargo, Go, NuGet, GitHub, or generic package sources supported by the downloader. -- directly installed executables; -- npm, PyPI, Cargo, Go, or other package-manager installations; -- archive-based installations; -- servers requiring a language SDK or compiler; -- servers not installable through the current downloader. +Language SDKs and package-manager runtimes remain explicit host prerequisites. Keep their resolver +commands in the manifest so a missing prerequisite produces a case-specific error rather than a +silent skip. Do not silently skip a required pair because its executable is absent. A CI lane either provisions the server or reports the pair as an explicit, reviewed exclusion. -Adding and pinning these external test tools requires product-owner approval under the repository's +Downloading these external test tools requires product-owner approval under the repository's dependency policy. They need not become Rust package dependencies, but they are still operational dependencies with maintenance, security, licensing, storage, and network consequences. -Pros of pinned versions: reproducible failures and controlled upgrades. Cons: compatibility with -new upstream releases is detected only when pins are deliberately refreshed. - -An unpinned “latest” lane can complement the pinned suite on a schedule. Its advantage is early -warning of upstream breakage; its disadvantage is nondeterminism, so it must not be the only merge -gate. +Always using Mason latest detects upstream compatibility changes immediately and avoids maintaining +a second installation path. The tradeoff is a nondeterministic merge gate: a registry or server +release can break an unchanged commit, and reproducing the failure depends on the recorded source +ID remaining available upstream. ## CI plan @@ -376,7 +374,7 @@ Run: - all existing unit tests and checks through `make test`; - global and detection E2E tests; -- one preferred, pinned server per source language; +- one preferred current-Mason server per source language; - every relevant subcommand across that smoke matrix; - manifest/data consistency checks. @@ -385,13 +383,9 @@ Run: Run all 141 compatible pairs, sharded by language and server installation family. Use fail-fast disabled so one broken server does not hide the rest of the compatibility report. -Cache downloaded toolchains and server packages using keys that include the pinned version. Do not -share homes, daemon runtime directories, or mutable workspaces between parallel jobs. - -### Scheduled latest-version compatibility - -Optionally run supported servers at current upstream versions. Report failures separately from the -pinned merge gate until a human confirms whether the server or `lsp-cli` needs adaptation. +Do not share homes, daemon runtime directories, or mutable workspaces between parallel jobs. CI may +cache immutable download transport data, but each case must retain isolated runtime state and must +not substitute a separately installed server for `--download`. ### Manual workflow @@ -430,7 +424,7 @@ that class of defect easier to diagnose. - [x] Confirm detectable support versus configured support: use the 16 detectable IDs. - [x] Confirm that unsupported optional capabilities count as a passing, asserted outcome. -- [x] Approve pinned external server/toolchain provisioning. +- [x] Approve latest-Mason external server provisioning through `--download`. - [x] Approve a narrow HTTP endpoint seam for deterministic `update` E2E coverage. - [x] Confirm PR smoke plus nightly exhaustive CI cadence. @@ -453,11 +447,11 @@ that class of defect easier to diagnose. ### Phase 3: preferred-server smoke matrix -Manual LSP verification follows server selection and provisioning so it runs against reproducible, -runnable servers rather than ambient or unpinned installations. +Manual LSP verification follows server selection and downloader support so it runs against servers +resolved by the same current Mason registry used in CI rather than ambient installations. -- [x] Select and pin one preferred server for each source language. -- [ ] Add provisioning scripts without new Rust dependencies. +- [x] Configure one production preference per source language and derive the smoke matrix from it. +- [x] Provision servers through `--download`; add no separate installers or Rust dependencies. - [ ] Run each relevant command manually against every new project. - [ ] Implement capability-aware query assertions. - [ ] Implement direct/detached lifecycle scenarios. @@ -475,12 +469,12 @@ runnable servers rather than ambient or unpinned installations. ### Phase 5: hardening - [ ] Run `make test`. -- [ ] Run the full pinned E2E matrix from a clean environment. +- [ ] Run the full latest-Mason E2E matrix from a clean environment. - [ ] Check every new or edited test file for boilerplate and duplication. - [ ] Check every source file remains below 600 lines. - [ ] Add regression tests for every bug uncovered during rollout. - [ ] Add LSP/server-specific discoveries to `GOTCHAS.md`. -- [ ] Document how to refresh server pins and triage nightly failures. +- [ ] Document how to identify upstream server versions and triage nightly failures. ## Definition of done @@ -493,7 +487,7 @@ The work is complete when: behavior; - direct, detached, stop, and stop-all lifecycle paths are covered; - formatting and diagnostics cannot dirty tracked files; -- required servers and toolchains are pinned and reproducibly provisioned; +- required servers are provisioned through `--download`, and resolved source IDs are retained; - PR smoke, nightly exhaustive, and manual targeted workflows are documented and passing; - `make test` passes; - known protocol/server deviations are recorded in `GOTCHAS.md`; @@ -508,7 +502,8 @@ The work is complete when: - Capability-aware results mean “all commands tested” does not mean “all commands succeed on every server.” It means every applicable success path and every inapplicable user-facing response is verified. -- Third-party server pinning creates a recurring upgrade and security-review obligation. +- Latest-server testing creates recurring upstream-flake, security-review, and reproducibility + risks even when this repository is unchanged. - Some servers require proprietary, platform-specific, or unusually heavy SDKs. Their treatment must be an explicit product decision rather than an automatic skip. - The current difference between the 362 configured filetypes and 16 detectable filetypes may need diff --git a/GOTCHAS.md b/GOTCHAS.md index 9fd8f3d..23a9029 100644 --- a/GOTCHAS.md +++ b/GOTCHAS.md @@ -69,18 +69,15 @@ ## kotlin-lsp -- The `kotlin_lsp` data config starts `kotlin-lsp --stdio`, while the Mason package currently - exposes its launcher as `intellij-server`. Selecting the official server is therefore possible, - but `lsp-cli --download` cannot launch it without a data-config correction or a provisioning - alias. Do not treat the preferred-server pin as a runnable smoke case until that mismatch is - resolved. +- The Mason package exposes Kotlin LSP as `intellij-server`, not `kotlin-lsp`. The data config must + use the packaged launcher name so `--download` can resolve it; a display name or upstream product + name is not necessarily an executable name. ## roslyn-language-server -- The `roslyn_ls` data config starts `dotnet` with a literal `` DLL placeholder, while - the Mason package exposes a `roslyn-language-server` launcher. The official C# server needs a - concrete installed DLL path or corrected launcher command before it can participate in automated - smoke tests. +- The Mason package exposes Roslyn through the `roslyn-language-server` .NET tool launcher. A data + config containing a literal installation placeholder cannot work with generic `--download`; + launch the Mason-exposed command and let the NuGet backend manage its concrete installation path. ## lua-language-server diff --git a/data b/data index 013a75f..59ea883 160000 --- a/data +++ b/data @@ -1 +1 @@ -Subproject commit 013a75f6412917b710aa4683b9c5761c0c679975 +Subproject commit 59ea88365855ca6a5ab35715c931d30c734e1b6e diff --git a/src/mason/install.rs b/src/mason/install.rs index 8b5bafe..a1116ff 100644 --- a/src/mason/install.rs +++ b/src/mason/install.rs @@ -36,7 +36,8 @@ pub(crate) fn resolve_cached_program( SourceId::Npm { .. } | SourceId::Pypi { .. } | SourceId::Cargo { .. } - | SourceId::Golang { .. } => { + | SourceId::Golang { .. } + | SourceId::Nuget { .. } => { let resolved_program = resolve_program(package, program, state, &TemplateContext::empty())?; Ok(if is_resolved_program_runnable(&resolved_program) { @@ -78,6 +79,10 @@ pub(crate) fn resolve_or_install_program( module_path, version, } => install_golang_package(state, package, &module_path, &version, program), + SourceId::Nuget { + package_name, + version, + } => install_nuget_package(state, package, &package_name, &version, program), SourceId::Github { repository, version, @@ -319,6 +324,55 @@ fn install_golang_package( ) } +fn install_nuget_package( + state: &RuntimeState, + package: &MasonPackage, + package_name: &str, + version: &str, + program: &str, +) -> Result { + use_cached_program_or( + package, + program, + state, + &TemplateContext::empty(), + |resolved_program| { + require_command("dotnet", package, program)?; + let install_dir = prepare_install_dir(state, package)?.join("bin"); + crate::fs::create_dir_all(&install_dir)?; + + let mut cmd = nuget_install_command(package_name, version, &install_dir); + run_install_command(&mut cmd, package, "dotnet tool install")?; + + finalize_install( + state, + package, + program, + &resolved_program, + &TemplateContext::empty(), + "dotnet did not produce a runnable", + ) + }, + ) +} + +fn nuget_install_command( + package_name: &str, + version: &str, + install_dir: &std::path::Path, +) -> Command { + let mut command = Command::new("dotnet"); + command + .arg("tool") + .arg("install") + .arg(package_name) + .arg("--tool-path") + .arg(install_dir) + .arg("--version") + .arg(version); + command +} + fn install_github_package( state: &RuntimeState, package: &MasonPackage, diff --git a/src/mason/install/tests.rs b/src/mason/install/tests.rs index 25a40f6..da8b9e3 100644 --- a/src/mason/install/tests.rs +++ b/src/mason/install/tests.rs @@ -1,4 +1,15 @@ -use super::artifacts::parse_archive_file_spec; +use std::path::Path; + +#[cfg(unix)] +use std::fs; + +use super::{ + artifacts::parse_archive_file_spec, nuget_install_command, resolve_or_install_program, +}; +#[cfg(unix)] +use crate::runtime_state::RuntimeState; +#[cfg(unix)] +use crate::test_support::{TestDir, env_var, make_executable, roslyn_package, with_env_vars}; #[test] fn parses_archive_file_spec() { @@ -14,3 +25,59 @@ fn parses_archive_file_spec() { ("clangd-linux-22.1.0.zip", None) ); } + +#[test] +fn builds_exact_nuget_tool_install_command() { + let command = nuget_install_command( + "roslyn-language-server", + "5.11.0-1.26380.4", + Path::new("managed/bin"), + ); + + assert_eq!(command.get_program(), "dotnet"); + assert_eq!( + command.get_args().collect::>(), + [ + "tool", + "install", + "roslyn-language-server", + "--tool-path", + "managed/bin", + "--version", + "5.11.0-1.26380.4", + ] + ); +} + +#[cfg(unix)] +#[test] +fn installs_and_caches_nuget_tool_with_a_receipt() { + let dir = TestDir::new("mason-nuget-install"); + let state = RuntimeState::new(dir.path().join("state")); + let bin_dir = dir.path().join("bin"); + fs::create_dir_all(&bin_dir).expect("fake program directory should exist"); + let dotnet = dir.write_file( + "bin/dotnet", + "#!/bin/sh\nwhile [ \"$#\" -gt 0 ]; do\n if [ \"$1\" = \"--tool-path\" ]; then\n shift\n tool_path=$1\n fi\n shift\ndone\n/bin/mkdir -p \"$tool_path\"\n: > \"$tool_path/roslyn-language-server\"\n/bin/chmod 755 \"$tool_path/roslyn-language-server\"\n", + ); + make_executable(&dotnet); + + let installed = with_env_vars(&[env_var("PATH", &bin_dir)], || { + resolve_or_install_program(&state, &roslyn_package(), "roslyn-language-server") + .expect("NuGet tool should install") + }); + + assert_eq!( + installed, + state + .package_dir("roslyn-language-server") + .join("bin/roslyn-language-server") + ); + let receipt = fs::read_to_string(state.receipt_path("roslyn-language-server")) + .expect("install receipt should exist"); + assert!(receipt.contains("pkg:nuget/roslyn-language-server@5.11.0-1.26380.4")); + + let cached = resolve_or_install_program(&state, &roslyn_package(), "roslyn-language-server") + .expect("installed NuGet tool should be reusable from cache"); + assert_eq!(cached, installed); +} diff --git a/src/mason/link.rs b/src/mason/link.rs index 6f2eeb5..effe801 100644 --- a/src/mason/link.rs +++ b/src/mason/link.rs @@ -115,6 +115,16 @@ pub(crate) fn resolve_program( )); } + if let Some(relative) = rendered.strip_prefix("nuget:") { + let executable = format!("{relative}{}", std::env::consts::EXE_SUFFIX); + return Ok(ResolvedProgram::Direct( + state + .package_dir(&package.name) + .join("bin") + .join(executable), + )); + } + if let Some(relative) = rendered.strip_prefix("python:") { return Ok(ResolvedProgram::Wrapper(WrapperProgram { launcher_path: state.bin_dir().join(program), diff --git a/src/mason/link/tests.rs b/src/mason/link/tests.rs index 1347e5b..3d8eb80 100644 --- a/src/mason/link/tests.rs +++ b/src/mason/link/tests.rs @@ -9,8 +9,8 @@ use crate::mason::registry::{ use crate::mason::template::TemplateContext; use crate::runtime_state::RuntimeState; use crate::test_support::{ - TestDir, env_var, jdtls_package, make_executable, pyright_package, suggested_language, - with_env_vars, + TestDir, env_var, jdtls_package, make_executable, pyright_package, roslyn_package, + suggested_language, with_env_vars, }; use std::collections::BTreeMap; use std::fs; @@ -196,6 +196,29 @@ fn resolves_npm_program_path() { ); } +#[test] +fn resolves_nuget_program_path() { + let dir = TestDir::new("mason-link"); + let state = RuntimeState::new(dir.path().join("state")); + + assert_eq!( + resolve_program_path( + &roslyn_package(), + "roslyn-language-server", + &state, + &TemplateContext::empty(), + ) + .expect("NuGet path should resolve"), + state + .package_dir("roslyn-language-server") + .join("bin") + .join(format!( + "roslyn-language-server{}", + std::env::consts::EXE_SUFFIX + )) + ); +} + #[test] fn resolves_github_template_program_path() { let dir = TestDir::new("mason-link"); diff --git a/src/mason/source.rs b/src/mason/source.rs index dab4a05..dba556a 100644 --- a/src/mason/source.rs +++ b/src/mason/source.rs @@ -19,6 +19,10 @@ pub(crate) enum SourceId { module_path: String, version: String, }, + Nuget { + package_name: String, + version: String, + }, Github { repository: String, version: String, @@ -77,6 +81,10 @@ pub(crate) fn parse_source_id(source_id: &str) -> Result { module_path: decoded_name, version, }, + "nuget" => SourceId::Nuget { + package_name: decoded_name, + version, + }, "github" => SourceId::Github { repository: decoded_name, version, @@ -186,6 +194,14 @@ mod tests { version: "v0.21.1".to_string(), } ); + assert_eq!( + parse_source_id("pkg:nuget/roslyn-language-server@5.11.0-1.26380.4") + .expect("NuGet source should parse"), + SourceId::Nuget { + package_name: "roslyn-language-server".to_string(), + version: "5.11.0-1.26380.4".to_string(), + } + ); } #[test] diff --git a/src/test_support.rs b/src/test_support.rs index c9530a4..64a8d61 100644 --- a/src/test_support.rs +++ b/src/test_support.rs @@ -178,6 +178,28 @@ pub(crate) fn pyright_package() -> MasonPackage { } } +pub(crate) fn roslyn_package() -> MasonPackage { + MasonPackage { + name: "roslyn-language-server".to_string(), + categories: vec!["LSP".to_string()], + source: MasonSource { + id: "pkg:nuget/roslyn-language-server@5.11.0-1.26380.4".to_string(), + extra_packages: Vec::new(), + asset: None, + download: None, + version_overrides: Vec::new(), + }, + bin: BTreeMap::from([( + "roslyn-language-server".to_string(), + "nuget:roslyn-language-server".to_string(), + )]), + share: BTreeMap::new(), + neovim: MasonNeovim { + lspconfig: Some("roslyn_ls".to_string()), + }, + } +} + pub(crate) fn jdtls_package() -> MasonPackage { MasonPackage { name: "jdtls".to_string(), diff --git a/tests/e2e/cases/c.yaml b/tests/e2e/cases/c.yaml index bf8d59d..88cdb47 100644 --- a/tests/e2e/cases/c.yaml +++ b/tests/e2e/cases/c.yaml @@ -6,5 +6,3 @@ language: pairs: - language: c server: clangd - preferred: - version: "22.1.6" diff --git a/tests/e2e/cases/cpp.yaml b/tests/e2e/cases/cpp.yaml index 291d383..1c27cf5 100644 --- a/tests/e2e/cases/cpp.yaml +++ b/tests/e2e/cases/cpp.yaml @@ -6,5 +6,3 @@ language: pairs: - language: cpp server: clangd - preferred: - version: "22.1.6" diff --git a/tests/e2e/cases/cs.yaml b/tests/e2e/cases/cs.yaml index 5e5c8ca..55d061f 100644 --- a/tests/e2e/cases/cs.yaml +++ b/tests/e2e/cases/cs.yaml @@ -6,5 +6,3 @@ language: pairs: - language: cs server: roslyn_ls - preferred: - version: "5.11.0-1.26380.4" diff --git a/tests/e2e/cases/cuda.yaml b/tests/e2e/cases/cuda.yaml index 8c6297a..2b8741d 100644 --- a/tests/e2e/cases/cuda.yaml +++ b/tests/e2e/cases/cuda.yaml @@ -6,5 +6,3 @@ language: pairs: - language: cuda server: clangd - preferred: - version: "22.1.6" diff --git a/tests/e2e/cases/go.yaml b/tests/e2e/cases/go.yaml index 2058954..8200a4e 100644 --- a/tests/e2e/cases/go.yaml +++ b/tests/e2e/cases/go.yaml @@ -6,5 +6,3 @@ language: pairs: - language: go server: gopls - preferred: - version: "v0.23.0" diff --git a/tests/e2e/cases/java.yaml b/tests/e2e/cases/java.yaml index 1ee5b80..1e8bae9 100644 --- a/tests/e2e/cases/java.yaml +++ b/tests/e2e/cases/java.yaml @@ -6,5 +6,3 @@ language: pairs: - language: java server: jdtls - preferred: - version: "v1.60.0" diff --git a/tests/e2e/cases/javascript.yaml b/tests/e2e/cases/javascript.yaml index ab281b2..9e585fd 100644 --- a/tests/e2e/cases/javascript.yaml +++ b/tests/e2e/cases/javascript.yaml @@ -6,5 +6,3 @@ language: pairs: - language: javascript server: ts_ls - preferred: - version: "6.0.0" diff --git a/tests/e2e/cases/kotlin.yaml b/tests/e2e/cases/kotlin.yaml index a036066..518587f 100644 --- a/tests/e2e/cases/kotlin.yaml +++ b/tests/e2e/cases/kotlin.yaml @@ -6,5 +6,3 @@ language: pairs: - language: kotlin server: kotlin_lsp - preferred: - version: "kotlin-lsp/v262.9593.0" diff --git a/tests/e2e/cases/lua.yaml b/tests/e2e/cases/lua.yaml index 83b0a47..e4b6e93 100644 --- a/tests/e2e/cases/lua.yaml +++ b/tests/e2e/cases/lua.yaml @@ -6,5 +6,3 @@ language: pairs: - language: lua server: lua_ls - preferred: - version: "3.19.1" diff --git a/tests/e2e/cases/objc.yaml b/tests/e2e/cases/objc.yaml index 2c4a244..32e11af 100644 --- a/tests/e2e/cases/objc.yaml +++ b/tests/e2e/cases/objc.yaml @@ -6,5 +6,3 @@ language: pairs: - language: objc server: clangd - preferred: - version: "22.1.6" diff --git a/tests/e2e/cases/objcpp.yaml b/tests/e2e/cases/objcpp.yaml index 25d8585..5f4af3c 100644 --- a/tests/e2e/cases/objcpp.yaml +++ b/tests/e2e/cases/objcpp.yaml @@ -6,5 +6,3 @@ language: pairs: - language: objcpp server: clangd - preferred: - version: "22.1.6" diff --git a/tests/e2e/cases/python.yaml b/tests/e2e/cases/python.yaml index edcbd0f..729fde3 100644 --- a/tests/e2e/cases/python.yaml +++ b/tests/e2e/cases/python.yaml @@ -6,5 +6,3 @@ language: pairs: - language: python server: pyright - preferred: - version: "1.1.413" diff --git a/tests/e2e/cases/rust.yaml b/tests/e2e/cases/rust.yaml index abb8979..95e624c 100644 --- a/tests/e2e/cases/rust.yaml +++ b/tests/e2e/cases/rust.yaml @@ -6,8 +6,6 @@ language: pairs: - language: rust server: rust_analyzer - preferred: - version: "2026-08-31" smoke: provision: method: download diff --git a/tests/e2e/cases/suite.yaml b/tests/e2e/cases/suite.yaml index d3917be..f4352f5 100644 --- a/tests/e2e/cases/suite.yaml +++ b/tests/e2e/cases/suite.yaml @@ -1,4 +1,4 @@ -schema-version: 3 +schema-version: 4 coverage: partial commands: diff --git a/tests/e2e/cases/typescript.yaml b/tests/e2e/cases/typescript.yaml index a42b59f..46fe591 100644 --- a/tests/e2e/cases/typescript.yaml +++ b/tests/e2e/cases/typescript.yaml @@ -6,5 +6,3 @@ language: pairs: - language: typescript server: ts_ls - preferred: - version: "6.0.0" diff --git a/tests/e2e/manifest.rs b/tests/e2e/manifest.rs index b4d885f..81dc928 100644 --- a/tests/e2e/manifest.rs +++ b/tests/e2e/manifest.rs @@ -5,11 +5,11 @@ use serde::Deserialize; use crate::case_files::{file_stem, read_yaml, yaml_paths}; use crate::manifest_data::{ - FiletypeConfig, LspConfig, PairKey, compatible_pairs, detectable_languages, + FiletypeConfig, LspConfig, PairKey, compatible_pairs, detectable_languages, preferred_pairs, }; use crate::repository_root; -const MANIFEST_SCHEMA_VERSION: u32 = 3; +const MANIFEST_SCHEMA_VERSION: u32 = 4; #[derive(Clone, Debug)] pub(crate) struct Manifest { @@ -89,16 +89,9 @@ impl ProjectKind { struct PairCase { language: String, server: String, - preferred: Option, smoke: Option, } -#[derive(Clone, Debug, Deserialize)] -#[serde(rename_all = "kebab-case", deny_unknown_fields)] -struct PreferredServer { - version: String, -} - #[derive(Clone, Debug, Deserialize)] #[serde(rename_all = "kebab-case", deny_unknown_fields)] struct SmokeCase { @@ -198,7 +191,7 @@ impl Manifest { let data = repository.join("data"); let declared_languages = self.validate_languages(repository, &data)?; let declared_pairs = self.validate_pairs(&data, &declared_languages)?; - self.validate_preferred_servers()?; + self.validate_preferred_servers(&data, &declared_pairs)?; if self.coverage == Coverage::Complete { Self::validate_complete_coverage(&data, &declared_languages, &declared_pairs)?; @@ -322,34 +315,27 @@ impl Manifest { if let Some(smoke) = &pair.smoke { smoke.validate(pair)?; } - if let Some(preferred) = &pair.preferred { - preferred.validate(pair)?; - } } Ok(declared) } - fn validate_preferred_servers(&self) -> Result<(), String> { - for language in &self.languages { - let count = self - .pairs - .iter() - .filter(|pair| pair.language == language.id && pair.preferred.is_some()) - .count(); - match (language.kind, count) { - (ProjectKind::Source, 1) | (ProjectKind::Metadata, 0) => {} - (ProjectKind::Source, _) => { - return Err(format!( - "E2E source language {:?} must select exactly one preferred server; found {count}", - language.id - )); - } - (ProjectKind::Metadata, _) => { - return Err(format!( - "E2E metadata language {:?} must not select a preferred server", - language.id - )); - } + fn validate_preferred_servers( + &self, + data: &Path, + declared_pairs: &BTreeSet, + ) -> Result<(), String> { + let source_languages = self + .languages + .iter() + .filter(|language| language.kind == ProjectKind::Source) + .map(|language| language.id.clone()) + .collect::>(); + for pair in preferred_pairs(data, &source_languages)? { + if !declared_pairs.contains(&pair) { + return Err(format!( + "E2E source language {:?} is missing its data-preferred server pair {:?}", + pair.language, pair.server + )); } } Ok(()) @@ -425,25 +411,6 @@ impl PairCase { } } -impl PreferredServer { - fn validate(&self, pair: &PairCase) -> Result<(), String> { - let version = self.version.trim(); - if version.is_empty() || version != self.version { - return Err(format!( - "preferred E2E server {}/{} must have a non-empty, trimmed version", - pair.language, pair.server - )); - } - if version.eq_ignore_ascii_case("latest") || version.eq_ignore_ascii_case("stable") { - return Err(format!( - "preferred E2E server {}/{} must use an exact version instead of {:?}", - pair.language, pair.server, self.version - )); - } - Ok(()) - } -} - impl SmokeCase { fn validate(&self, pair: &PairCase) -> Result<(), String> { let label = format!("{}/{}", pair.language, pair.server); diff --git a/tests/e2e/manifest_data.rs b/tests/e2e/manifest_data.rs index 102deac..6cf6b05 100644 --- a/tests/e2e/manifest_data.rs +++ b/tests/e2e/manifest_data.rs @@ -1,4 +1,4 @@ -use std::collections::BTreeSet; +use std::collections::{BTreeMap, BTreeSet}; use std::path::Path; use serde::Deserialize; @@ -32,6 +32,12 @@ pub(crate) struct LspConfig { pub(crate) name: String, } +#[derive(Deserialize)] +struct CliConfig { + #[serde(default)] + lsp: BTreeMap>, +} + pub(crate) fn detectable_languages(data: &Path) -> Result, String> { let mut languages = BTreeSet::new(); for path in yaml_paths(&data.join("filetypes"))? { @@ -62,3 +68,45 @@ pub(crate) fn compatible_pairs( } Ok(pairs) } + +pub(crate) fn preferred_pairs( + data: &Path, + languages: &BTreeSet, +) -> Result, String> { + let config: CliConfig = read_yaml(&data.join("lsp-cli.yaml"))?; + let mut lsps = Vec::new(); + for path in yaml_paths(&data.join("lsp"))? { + lsps.push((file_stem(&path)?, read_yaml::(&path)?)); + } + + languages + .iter() + .map(|language| { + let preferred_name = config + .lsp + .get(language) + .and_then(|servers| servers.first()) + .ok_or_else(|| { + format!("E2E source language {language:?} has no preferred server in data") + })?; + let matching = lsps + .iter() + .filter(|(_id, lsp)| { + lsp.name == *preferred_name && lsp.filetypes.contains(language) + }) + .collect::>(); + match matching.as_slice() { + [(server, _lsp)] => Ok(PairKey { + language: language.clone(), + server: server.clone(), + }), + [] => Err(format!( + "preferred server {preferred_name:?} for E2E source language {language:?} has no compatible LSP config" + )), + _ => Err(format!( + "preferred server {preferred_name:?} for E2E source language {language:?} is ambiguous" + )), + } + }) + .collect() +} diff --git a/tests/e2e/manifest_tests.rs b/tests/e2e/manifest_tests.rs index 9f11bdf..2edf996 100644 --- a/tests/e2e/manifest_tests.rs +++ b/tests/e2e/manifest_tests.rs @@ -12,14 +12,6 @@ fn first_smoke(manifest: &mut Manifest) -> &mut SmokeCase { .expect("selected pair should have a smoke case") } -fn preferred_pair_mut<'a>(manifest: &'a mut Manifest, language: &str) -> &'a mut PairCase { - manifest - .pairs - .iter_mut() - .find(|pair| pair.language == language && pair.preferred.is_some()) - .expect("language should have a preferred pair") -} - #[test] fn partial_manifest_matches_pinned_data() { Manifest::load() @@ -32,44 +24,37 @@ fn partial_manifest_matches_pinned_data() { fn source_languages_select_the_approved_preferred_servers() { let manifest = Manifest::load().expect("E2E manifest should parse"); let expected = [ - ("c", "clangd", "22.1.6"), - ("cpp", "clangd", "22.1.6"), - ("cs", "roslyn_ls", "5.11.0-1.26380.4"), - ("cuda", "clangd", "22.1.6"), - ("go", "gopls", "v0.23.0"), - ("java", "jdtls", "v1.60.0"), - ("javascript", "ts_ls", "6.0.0"), - ("kotlin", "kotlin_lsp", "kotlin-lsp/v262.9593.0"), - ("lua", "lua_ls", "3.19.1"), - ("objc", "clangd", "22.1.6"), - ("objcpp", "clangd", "22.1.6"), - ("python", "pyright", "1.1.413"), - ("rust", "rust_analyzer", "2026-08-31"), - ("typescript", "ts_ls", "6.0.0"), + ("c", "clangd"), + ("cpp", "clangd"), + ("cs", "roslyn_ls"), + ("cuda", "clangd"), + ("go", "gopls"), + ("java", "jdtls"), + ("javascript", "ts_ls"), + ("kotlin", "kotlin_lsp"), + ("lua", "lua_ls"), + ("objc", "clangd"), + ("objcpp", "clangd"), + ("python", "pyright"), + ("rust", "rust_analyzer"), + ("typescript", "ts_ls"), ]; - for (language, server, version) in expected { - let pair = manifest - .pairs - .iter() - .find(|pair| pair.language == language && pair.preferred.is_some()) - .expect("source language should have a preferred pair"); - assert_eq!(pair.server, server); - assert_eq!( - pair.preferred - .as_ref() - .expect("pair should be preferred") - .version, - version - ); - } - - let preferred_count = manifest - .pairs + let languages = expected .iter() - .filter(|pair| pair.preferred.is_some()) - .count(); - assert_eq!(preferred_count, expected.len()); + .map(|(language, _server)| (*language).to_string()) + .collect(); + let preferred = preferred_pairs(&repository_root().join("data"), &languages) + .expect("data preferences should resolve"); + let actual = preferred + .iter() + .map(|pair| (pair.language.as_str(), pair.server.as_str())) + .collect::>(); + assert_eq!(actual, expected.as_slice()); + + manifest + .validate(repository_root()) + .expect("preferred data pairs should be declared"); } #[test] @@ -102,7 +87,7 @@ fn complete_mode_rejects_missing_server_pairs() { #[test] fn manifest_rejects_unknown_fields() { let error = serde_yaml::from_str::( - "schema-version: 3\ncoverage: partial\ncommands: []\nunknown: true\n", + "schema-version: 4\ncoverage: partial\ncommands: []\nunknown: true\n", ) .expect_err("unknown manifest fields should fail"); @@ -112,64 +97,15 @@ fn manifest_rejects_unknown_fields() { #[test] fn manifest_rejects_a_source_language_without_a_preferred_server() { let mut manifest = Manifest::load().expect("E2E manifest should parse"); - preferred_pair_mut(&mut manifest, "c").preferred = None; + manifest + .pairs + .retain(|pair| !(pair.language == "c" && pair.server == "clangd")); let error = manifest .validate(repository_root()) .expect_err("source language should require one preferred server"); - assert!(error.contains("must select exactly one preferred server; found 0")); -} - -#[test] -fn manifest_rejects_multiple_preferred_servers_for_one_language() { - let mut manifest = Manifest::load().expect("E2E manifest should parse"); - let mut duplicate = preferred_pair_mut(&mut manifest, "c").clone(); - duplicate.server = "ccls".to_string(); - manifest.pairs.push(duplicate); - - let error = manifest - .validate(repository_root()) - .expect_err("source language should have only one preferred server"); - - assert!(error.contains("must select exactly one preferred server; found 2")); -} - -#[test] -fn manifest_rejects_a_preferred_server_for_metadata() { - let mut manifest = Manifest::load().expect("E2E manifest should parse"); - manifest.pairs.push(PairCase { - language: "gomod".to_string(), - server: "gopls".to_string(), - preferred: Some(PreferredServer { - version: "v0.23.0".to_string(), - }), - smoke: None, - }); - - let error = manifest - .validate(repository_root()) - .expect_err("metadata language should not select a preferred server"); - - assert!(error.contains("must not select a preferred server")); -} - -#[test] -fn manifest_rejects_non_exact_preferred_versions() { - for version in ["", " latest", "latest", "stable"] { - let mut manifest = Manifest::load().expect("E2E manifest should parse"); - preferred_pair_mut(&mut manifest, "c") - .preferred - .as_mut() - .expect("pair should be preferred") - .version = version.to_string(); - - let error = manifest - .validate(repository_root()) - .expect_err("preferred version should be exact"); - - assert!(error.contains("preferred E2E server c/clangd")); - } + assert!(error.contains("missing its data-preferred server pair")); } #[test]