diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fb1abf0..961a9a7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -35,15 +35,19 @@ jobs: - name: Check formatting if: matrix.rust == 'stable' - run: cargo fmt --check + run: make check-format - name: Check generated command reference if: matrix.rust == 'stable' - run: python3 scripts/update_readme_commands.py --check + run: make check-readme - name: Run tests - run: cargo test --locked + run: make check-tests + + - name: Run real-server E2E smoke tests + if: matrix.rust == 'stable' + run: make test-real-server-e2e - name: Run clippy if: matrix.rust == 'stable' - run: cargo clippy --all-targets --all-features -- -D warnings + run: make check-clippy diff --git a/.memory/unprocessed/ci-reuses-make-targets.md b/.memory/unprocessed/ci-reuses-make-targets.md new file mode 100644 index 0000000..346011b --- /dev/null +++ b/.memory/unprocessed/ci-reuses-make-targets.md @@ -0,0 +1,6 @@ +# CI reuses Makefile verification targets + +The initial E2E implementation duplicated formatting, Clippy, and live-server commands between +`Makefile` and `.github/workflows/ci.yml`. The user explicitly rejected that structure. Keep the +commands centralized as granular Make targets; CI should only choose which Make target runs for a +toolchain or lane. diff --git a/.memory/unprocessed/e2e-cases-are-split-by-language.md b/.memory/unprocessed/e2e-cases-are-split-by-language.md new file mode 100644 index 0000000..7205191 --- /dev/null +++ b/.memory/unprocessed/e2e-cases-are-split-by-language.md @@ -0,0 +1,6 @@ +# E2E case manifests are split by language + +The E2E manifest is a directory rather than one `cases.yaml` file. Global command coverage lives +in `cases/suite.yaml`; each `cases/.yaml` owns exactly one project and all server pairs +for that language. The `gowork` case intentionally contains only an isolated `go.work` file so its +detection does not overlap `gomod` or `go`. diff --git a/.memory/unprocessed/e2e-fake-lsp-is-a-rust-helper.md b/.memory/unprocessed/e2e-fake-lsp-is-a-rust-helper.md new file mode 100644 index 0000000..0591b87 --- /dev/null +++ b/.memory/unprocessed/e2e-fake-lsp-is-a-rust-helper.md @@ -0,0 +1,5 @@ +# The deterministic E2E LSP fixture is a Rust helper + +When offered Python, real rust-analyzer, or a Rust helper for deterministic command-path tests, +the user selected the Rust helper. Keep the helper as an isolated, locked fixture package that +reuses serde_json; do not replace it with a Python script merely to reduce fixture build code. diff --git a/.memory/unprocessed/e2e-fixture-dedup-scope.md b/.memory/unprocessed/e2e-fixture-dedup-scope.md new file mode 100644 index 0000000..bcf98ef --- /dev/null +++ b/.memory/unprocessed/e2e-fixture-dedup-scope.md @@ -0,0 +1,5 @@ +# E2E fixture deduplication applies to the Rust harness + +The project-phase deduplication item applies to repeated E2E harness setup, including validated +manifest loading, isolated context construction, and command-strategy selection. It does not mean +removing the intentionally equivalent domain concepts from the per-language playground sources. diff --git a/.memory/unprocessed/e2e-servers-use-mason-latest.md b/.memory/unprocessed/e2e-servers-use-mason-latest.md new file mode 100644 index 0000000..852526f --- /dev/null +++ b/.memory/unprocessed/e2e-servers-use-mason-latest.md @@ -0,0 +1,9 @@ +# Real-server E2E provisioning uses Mason latest + +The user corrected the preferred-server E2E design: do not install LSP servers separately and do +not pin a Mason registry release or package versions. Each real-server test must use `--download` +against Mason latest. The preferred smoke pair must be derived from the first production preference +in `data/lsp-cli.yaml`; do not duplicate that selection in the E2E case YAML. + +This deliberately trades reproducibility for immediate upstream compatibility coverage. Preserve +the resolved Mason source ID in diagnostics so failures can still identify the installed version. diff --git a/.memory/unprocessed/manual-e2e-verification-follows-provisioning.md b/.memory/unprocessed/manual-e2e-verification-follows-provisioning.md new file mode 100644 index 0000000..bf60b5a --- /dev/null +++ b/.memory/unprocessed/manual-e2e-verification-follows-provisioning.md @@ -0,0 +1,6 @@ +# Manual E2E verification follows server provisioning + +The project plan originally required running every relevant LSP command against new playgrounds +before selecting or provisioning their servers. Most new languages had no runnable compatible +server, so the manual verification item was moved immediately after preferred-server provisioning +and remains unchecked until those prerequisites exist. diff --git a/.memory/unprocessed/new-playground-build-metadata.md b/.memory/unprocessed/new-playground-build-metadata.md new file mode 100644 index 0000000..93f1d2e --- /dev/null +++ b/.memory/unprocessed/new-playground-build-metadata.md @@ -0,0 +1,6 @@ +# New playgrounds use dependency-free and relocatable metadata + +The Kotlin playground uses compiler-valid sources plus `settings.gradle.kts`, without adding a +Gradle or Maven Kotlin plugin dependency. CUDA, Objective-C, and Objective-C++ use relative +`compile_commands.json` working directories so their compilation metadata remains valid after the +E2E harness copies a project into a randomized sandbox. diff --git a/.memory/unprocessed/official-first-e2e-server-pins.md b/.memory/unprocessed/official-first-e2e-server-pins.md new file mode 100644 index 0000000..574b294 --- /dev/null +++ b/.memory/unprocessed/official-first-e2e-server-pins.md @@ -0,0 +1,10 @@ +# Official-first E2E server pins expose launcher mismatches + +The user selected an official-first preferred E2E server matrix and explicitly accepted bespoke +provisioning and current command/config gaps. In particular, use `kotlin_lsp` rather than the +community `kotlin_language_server`, and `roslyn_ls` rather than OmniSharp. + +The pinned Mason registry exposes Kotlin LSP as `intellij-server`, although the data config starts +`kotlin-lsp`. It exposes Roslyn as `roslyn-language-server`, although the data config starts `dotnet` +with a literal `` DLL path. The preferred-server selection can be committed independently, +but provisioning must resolve these mismatches before claiming either smoke case is runnable. diff --git a/.memory/unprocessed/playground-validity-audit.md b/.memory/unprocessed/playground-validity-audit.md new file mode 100644 index 0000000..6c7c573 --- /dev/null +++ b/.memory/unprocessed/playground-validity-audit.md @@ -0,0 +1,10 @@ +# Existing playgrounds are not all valid standalone projects + +`E2E_TESTS.md` requires every source-language playground to be valid, but the existing C++ fixture +does not link because `main.cpp` calls undefined `f()` and `g()`. The Rust fixture also cannot be +checked through its own manifest because Cargo treats the nested package as an undeclared member +of the repository's root package workspace. + +The project audit must distinguish source presence from validated buildability. Repair these +fixtures in the dedicated cleanup step rather than silently treating their files as a valid E2E +baseline. diff --git a/Cargo.lock b/Cargo.lock index e577390..6afd233 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -195,6 +195,16 @@ version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" +[[package]] +name = "command-group" +version = "5.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a68fa787550392a9d58f44c21a3022cfb3ea3e2458b7f85d3b399d0ceeccf409" +dependencies = [ + "nix", + "winapi", +] + [[package]] name = "const-oid" version = "0.10.2" @@ -324,6 +334,12 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "fs_extra" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" + [[package]] name = "futures-channel" version = "0.3.32" @@ -762,7 +778,9 @@ version = "0.1.5" dependencies = [ "clap", "clap_complete", + "command-group", "flate2", + "fs_extra", "humantime", "lsp-types", "regex", @@ -776,6 +794,7 @@ dependencies = [ "tempfile", "thiserror", "url", + "wait-timeout", "zip", ] @@ -819,6 +838,17 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "nix" +version = "0.27.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2eb04e9c688eff1c89d72b407f168cf79bb9e867a9d3323ed6c01519eb9cc053" +dependencies = [ + "bitflags 2.11.1", + "cfg-if", + "libc", +] + [[package]] name = "once_cell" version = "1.21.4" @@ -1568,6 +1598,15 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" +[[package]] +name = "wait-timeout" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ac3b126d3914f9849036f826e054cbabdc8519970b8998ddaf3b5bd3c65f11" +dependencies = [ + "libc", +] + [[package]] name = "want" version = "0.3.1" @@ -1719,6 +1758,28 @@ dependencies = [ "rustls-pki-types", ] +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + [[package]] name = "windows-link" version = "0.2.1" diff --git a/Cargo.toml b/Cargo.toml index 59f1230..681aa0c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -30,6 +30,11 @@ thiserror = "2.0.17" url = "2.5.7" zip = { version = "8.6.0", default-features = false, features = ["deflate"] } +[dev-dependencies] +command-group = "5.0.1" +fs_extra = "1.3.0" +wait-timeout = "0.2.1" + [lints] workspace = true diff --git a/E2E_TESTS.md b/E2E_TESTS.md new file mode 100644 index 0000000..8bf88b9 --- /dev/null +++ b/E2E_TESTS.md @@ -0,0 +1,511 @@ +# End-to-end test plan + +## Goal + +Exercise the released `lsp-cli` binary against every supported language, every compatible +supported LSP server, and every top-level subcommand. Keep the suite useful both as a fast pull +request check and as an exhaustive compatibility check. + +The tests must validate user-visible behavior: exit status, stdout, stderr, filesystem effects, +server lifecycle, and semantically relevant LSP results. They must not depend on private Rust APIs. + +## Working definition of supported + +At pinned `lsp-cli-data` revision `59ea88365855ca6a5ab35715c931d30c734e1b6e`, the data tree has: + +- 362 filetype configurations; +- 362 LSP configurations; +- 16 detectable filetype IDs (a non-empty `extensions` or `patterns` list); +- 57 distinct LSP configurations associated with those detectable filetypes; +- 141 compatible detectable-filetype/LSP pairs. + +The working E2E scope is the 16 detectable IDs: + +```text +c cpp cs cuda go gomod gowork java javascript kotlin lua objc objcpp python rust typescript +``` + +The remaining 346 filetype configurations have no detection rules. They are configuration catalog +entries, but cannot currently drive a project-based E2E test. Separately test that the whole data +tree parses and that catalog commands describe it consistently. + +This is a product-policy boundary rather than an implementation fact. Before declaring the suite +complete, the product owner must confirm one of these definitions: + +1. **Detectable support (recommended):** exhaustive real-server tests cover the 16 detectable IDs, + 57 relevant servers, and 141 compatible pairs. +2. **Configured support:** all 362 filetype and server configs are considered supported. This first + requires adding detection rules, test projects, and provisioning for the presently inactive + catalog entries. + +Pros of detectable support: executable now, objective, and automatically derived from the shipped +data. Cons: `lsp-cli languages` currently appears to expose a wider catalog than this definition. + +Pros of configured support: the word “supported” matches every shipped YAML entry. Cons: most of +the required projects and provisioning do not exist, and millions of incompatible Cartesian cases +would still need to be excluded. + +## Non-goals + +- Do not run a Cartesian product of every language, server, command, and option. Only configured + language/server relationships are meaningful. +- Do not require a server to implement an optional LSP capability. +- Do not put language-specific parsing or source-code knowledge into production `lsp-cli` code. +- Do not make tracked playground files writable test state. +- Do not add a Rust dependency without explicit permission. The existing `tempfile`, `serde`, + `serde_json`, and `serde_yaml` dependencies are sufficient for the planned harness. + +## Test projects + +### Existing projects + +Reuse the projects under `playground/` for: + +| Filetype ID | Directory | +|---|---| +| `c` | `playground/c` | +| `cpp` | `playground/cpp` | +| `cs` | `playground/csharp` | +| `go` | `playground/go` | +| `java` | `playground/java` | +| `javascript` | `playground/js` | +| `lua` | `playground/lua` | +| `python` | `playground/python` | +| `rust` | `playground/rust` | +| `typescript` | `playground/typescript` | + +### Additional projects + +Source projects are committed for: + +- `playground/cuda` +- `playground/kotlin` +- `playground/objc` +- `playground/objcpp` + +Minimal detection fixtures are committed under `playground/gomod` and `playground/gowork`. These +IDs describe Go workspace metadata, not source languages, so they can cover detection, file +listing, server selection, initialization, and lifecycle, but cannot independently provide +meaningful symbol or call-hierarchy assertions. + +Every source-language project should be small, valid, and multi-file. Where the language permits, +it should contain: + +- one stable workspace symbol; +- functions and methods; +- a declaration separated from its definition; +- references from more than one file; +- a caller and callee chain; +- types and fields; +- a file whose formatting can be made deterministically incorrect; +- a deterministic source mutation that produces one diagnostic. + +Prefer equivalent domain concepts and symbol names across projects when natural. Do not force a +language into constructs it does not support merely to make fixtures textually identical. + +### Existing playground audit + +The ten existing playgrounds were audited against these requirements on 2026-09-05. `Present` +means the committed source provides the semantic shape; `missing` identifies follow-up work; +`unverified` means the required compiler or runtime was not installed for this audit. Under the +strict declaration rule, an interface, trait, protocol, or declaration file is required when the +language can express one without relying on comments or third-party syntax. + +| Project | Valid, small, multi-file | Stable workspace symbol | Functions and methods | Separate declaration | Cross-file references | Caller/callee chain | Types and fields | Formatting mutation | Diagnostic mutation | +|---|---|---|---|---|---|---|---|---|---| +| C | Present, but compilation database is not portable | `Order` | Functions present; methods not applicable | Present in `order.h` | Present | Present | Present | Missing recipe; baseline is not formatter-clean | Missing recipe | +| C++ | **Invalid:** undefined `f()` and `g()` prevent linking; compilation database is not portable | `playground::Order` | Present | Present in `order.hpp` | Present | Present | Present | Missing recipe; baseline is not formatter-clean | Missing recipe | +| C# | Unverified; `dotnet` unavailable | `Order` | Present | **Missing:** an interface can provide it | Present | Present | Present | Missing recipe | Missing recipe | +| Go | Unverified; `go` unavailable | `Order` | Present | **Missing:** an interface can provide it | Present | Present | Present | Missing recipe; baseline is visibly not `gofmt`-clean | Missing recipe | +| Java | Unverified; JDK and Maven unavailable | `Order` | Present | **Missing:** an interface can provide it | Present | Present | Present | Missing recipe | Missing recipe | +| JavaScript | Present; exercised with Node.js | `Order` | Present | **Missing:** a declaration file can provide it | Present | Present | Present | Missing recipe | Missing recipe | +| Lua | Unverified; Lua unavailable | **Missing:** only local functions are declared | Functions present; methods missing | Not applicable: Lua has no native declaration construct | Present for `format_timestamp` | Present | Partial: a module table field exists, but no structured domain type | Missing recipe | Missing recipe | +| Python | Present; exercised with Python | `Order` | Present | **Missing:** a protocol or abstract base can provide it | Present | Present | Present | Missing recipe; baseline is visibly not formatter-clean | Missing recipe | +| Rust | **Invalid as a standalone project:** Cargo treats it as an undeclared root-workspace member | `Order` | Present | **Missing:** a trait can provide it | Present | Present | Present | Missing recipe | Missing recipe | +| TypeScript | Unverified; local TypeScript compiler unavailable | `Order` | Present | **Missing:** an interface can provide it | Present | Present | Present | Missing recipe | Missing recipe | + +The C sources compile and link, while the C++ sources compile but fail at link time because the +calls added in `main.cpp` have no definitions. JavaScript and Python execute successfully. The Rust +check fails before compilation because the nested package is neither a root-workspace member nor +excluded from that workspace. C and C++ also embed an old absolute checkout path in +`compile_commands.json`; language servers may therefore ignore their intended include paths after +the repository is moved or copied into an isolated E2E sandbox. + +No playground currently defines the exact source edit and expected diagnostic needed for a stable +mutation test. Those recipes should live in manifest data rather than language-specific Rust test +code. The next project-phase item will repair and normalize the fixtures; this audit intentionally +does not mix those changes with the inventory. + +Tests securely create randomized sandboxes with the `tempfile` crate under the user's +`XDG_RUNTIME_DIR`, `XDG_CACHE_HOME`, or `$HOME/.cache`, in that order, then copy a project there +before formatting it or introducing diagnostics. Real test state must not use the ambient system +`/tmp`. This keeps the repository clean, avoids uncontrolled parent root markers, keeps Unix socket +paths short, and allows safe parallel execution. + +Pros of committed playgrounds: humans can reproduce failures with the same projects. Cons: each +language fixture must evolve with its toolchain and server ecosystem. + +An alternative is to generate every project during test setup. That reduces committed files, but +makes failures harder to inspect and manual reproduction less convenient; do not use it for the +baseline projects. + +## Coverage model + +`lsp-cli` currently has 24 canonical top-level subcommands. Factor them by responsibility instead +of multiplying all commands by all language/server pairs. + +| Scope | Subcommands | Required coverage | +|---|---|---| +| Global CLI | `commands`, `languages`, `servers`, `completion`, `agent-skill`, `update` | Focused binary-level cases, independent of real servers | +| Detection and filesystem | `detect`, `list-files` | Every detectable filetype ID | +| LSP requests | `server-capabilities`, `diagnostics`, `format`, `grep`, `list-symbols`, `list-functions`, `references`, `callers`, `callees`, `definition`, `declaration`, `build-index` | Every compatible language/server pair, capability-aware | +| Process lifecycle | `run`, `daemon`, `stop`, `stop-all` | Every distinct relevant server where applicable, with grouped lifecycle scenarios | + +### Capability-aware expectations + +For each compatible pair, first record or inspect the server's initialized capabilities. A command +passes if it either: + +- succeeds and returns the expected semantic result; or +- returns the documented, user-facing unsupported-capability error when the server does not + advertise the required capability. + +Formatting, declarations, diagnostics, workspace symbols, and call hierarchy are optional or vary +substantially between servers. Treating every unsupported operation as a suite failure would test +an assumption the LSP specification does not make. + +Capability advertisement is not enough by itself: when a server advertises a capability, exercise +the corresponding command and assert its behavior. + +### Option coverage + +Distribute option variants across the matrix using explicit cases; do not create another full +cross-product. Cover at least: + +- automatic selection, `--lang`, and `--lsp`; +- text and `--json` output; +- direct execution, `--detach`, and `--no-detach`; +- `--limit`, `--files-with-matches`, and `--full`; +- `--wait-for-index`; +- `format`, `format --check`, and `format --stdout`; +- successful operations, unsupported capabilities, missing executables, server crashes, malformed + replies, and timeouts; +- `--download` once per supported installation mechanism, rather than redundantly for every query. + +JSON assertions should deserialize and compare stable semantic fields. Text assertions should +avoid full snapshots when server versions can legitimately change ordering, signatures, or detail. + +## Harness design + +Use a normal Cargo integration-test crate that invokes the built binary through +`CARGO_BIN_EXE_lsp-cli`. + +Proposed layout: + +```text +tests/ + e2e.rs + e2e/ + harness.rs + manifest.rs + catalog.rs + detection.rs + queries.rs + lifecycle.rs + update.rs + cases/ + suite.yaml + .yaml +``` + +Keep every Rust file under 600 lines. Move repeated process setup and assertions into helpers as +soon as a second test needs them. + +`harness.rs` should provide methods for actions on an E2E context, for example: + +- create an isolated home, configuration root, runtime root, and workspace copy; +- construct an `lsp-cli` process with deterministic environment variables; +- run a command with a deadline and capture stdout/stderr/status; +- parse JSON output; +- introduce a formatting or diagnostic mutation; +- find and terminate remaining child processes; +- stop daemons and report their runtime state after failure. + +Each test process should set at least: + +- `HOME` to an isolated temporary home; +- `XDG_CONFIG_HOME` to an isolated configuration directory; +- `XDG_RUNTIME_DIR` to an isolated daemon directory; +- `LSP_DATA` to the pinned repository submodule; +- `PATH` to the explicitly provisioned toolchain/server environment. + +Do not rely on a developer's user configuration, downloaded server cache, daemon sockets, current +shell, or ambient server versions. + +The manifest directory should include stable case data, provisioning metadata, expected +capabilities, and documented exclusions. `tests/e2e/cases/suite.yaml` owns global command coverage, +while each `tests/e2e/cases/.yaml` owns one project and all of its server pairs. A +validation test should fail when: + +- a detectable filetype lacks a project; +- a compatible pair lacks a manifest entry; +- a manifest entry names a missing data config; +- an exclusion lacks a reason; +- two cases select the same user-visible server ambiguously; +- a new top-level subcommand has no assigned coverage class. + +The version 4 manifest assigns every canonical command to a coverage strategy, derives one +preferred smoke-matrix server for every source-language project from `data/lsp-cli.yaml`, and keeps +`coverage: partial`, which validates every declared language/server entry +against the pinned data without requiring unfinished matrix entries. Phase 4 adds the remaining +entries and switches it to `coverage: complete`; complete mode enforces every detectable language +and compatible pair. + +### Extending the manifest + +To cover an existing filetype, add its small project under `playground/`, add one case file named +after the filetype ID, then add a `pairs` entry there for each compatible server. To introduce a +genuinely new filetype or server, first add its YAML config and commit it in the `data` submodule, +then update the submodule revision and the E2E cases in this repository. + +Pair entries use the LSP YAML filename stem as their stable config ID. The test runner loads the +configured user-visible server name for `--lsp`; do not duplicate it in the manifest. The first +server in each source language's production preference list is also its merge-gate smoke server. +Manifest validation resolves that user-visible name to one compatible LSP config and requires the +corresponding pair to exist. Each optional +`smoke` block declares a generic provisioning method, query kind, semantic expectations, runtime +host programs, and deadlines. Language-specific prerequisites and expected symbols belong in YAML, +not in the Rust runner. The first provisioning method is `download`; add other mechanisms as typed +methods when needed instead of branching on server names. + +### Preferred server matrix + +| Languages | LSP config ID | +| --- | --- | +| C, C++, CUDA, Objective-C, Objective-C++ | `clangd` | +| C# | `roslyn_ls` | +| Go | `gopls` | +| Java | `jdtls` | +| JavaScript, TypeScript | `ts_ls` | +| Kotlin | `kotlin_lsp` | +| Lua | `lua_ls` | +| Python | `pyright` | +| Rust | `rust_analyzer` | + +These selections come from the shipped production preferences, not duplicated manifest flags or +language-specific runner branches. Real-server tests use `--download` with a clean isolated home, +so Mason's current registry release selects and installs the server version on every run. Failure +diagnostics must retain the resolved package source ID so an upstream version change can be +identified after the fact. + +In `coverage: complete` mode, manifest validation makes a new detectable filetype or compatible +filetype/server relationship fail until its project and pair are declared. Partial mode intentionally +allows the matrix to grow incrementally. + +## Special command strategies + +### `run` + +`run` replaces the current process with the language server on Unix. The foundation smoke uses a +deterministic server marker to prove replacement. Real-server coverage should additionally use +piped stdio for a minimal `initialize` / `initialized` / `shutdown` / `exit` exchange. Test +selection and exec errors separately. + +### `daemon`, `stop`, and `stop-all` + +For every applicable server, run a grouped lifecycle scenario: + +1. start a daemon in an isolated runtime directory; +2. issue at least two queries with `--detach` and verify reuse; +3. stop the exact daemon; +4. verify that a later query starts or connects according to documented behavior; +5. start multiple isolated daemons and verify `stop-all` removes all of them. + +On failure, print socket paths, process state, selected command line, workspace root, and bounded +server stderr. Cleanup must run even after an assertion failure. + +### `update` + +The production default uses the lsp-cli-data GitHub release endpoint. The narrowly scoped +`LSP_CLI_DATA_RELEASE_API_URL` override redirects only release metadata lookup, allowing the binary +E2E test to serve metadata and a valid archive locally. This keeps the success path deterministic +without changing normal update behavior. + +### Diagnostics and formatting + +Start from a valid temporary workspace. Apply one language-specific mutation recorded in the +manifest, run the command, assert the expected file/range/message class, and discard the temporary +copy. Do not commit permanently broken source files that could interfere with unrelated queries. + +### Indexing + +`build-index` should be tested against every server that has a usable background-work completion +signal. For other servers, assert the intended bounded timeout or no-op policy. Do not infer +completion from a fixed sleep. + +## Real-server provisioning + +Do not install LSP servers separately. Every real-server case must pass `--download`, allowing the +production Mason integration to select the current registry package, install it inside the case's +isolated home, and return the resolved executable. This applies uniformly to direct archives and +npm, PyPI, Cargo, Go, NuGet, GitHub, or generic package sources supported by the downloader. + +Language SDKs and package-manager runtimes remain explicit host prerequisites. Keep their resolver +commands in the manifest so a missing prerequisite produces a case-specific error rather than a +silent skip. + +Do not silently skip a required pair because its executable is absent. A CI lane either provisions +the server or reports the pair as an explicit, reviewed exclusion. + +Downloading these external test tools requires product-owner approval under the repository's +dependency policy. They need not become Rust package dependencies, but they are still operational +dependencies with maintenance, security, licensing, storage, and network consequences. + +Always using Mason latest detects upstream compatibility changes immediately and avoids maintaining +a second installation path. The tradeoff is a nondeterministic merge gate: a registry or server +release can break an unchanged commit, and reproducing the failure depends on the recorded source +ID remaining available upstream. + +## CI plan + +### Pull requests + +Run: + +- all existing unit tests and checks through `make test`; +- global and detection E2E tests; +- one preferred current-Mason server per source language; +- every relevant subcommand across that smoke matrix; +- manifest/data consistency checks. + +### Nightly exhaustive matrix + +Run all 141 compatible pairs, sharded by language and server installation family. Use fail-fast +disabled so one broken server does not hide the rest of the compatibility report. + +Do not share homes, daemon runtime directories, or mutable workspaces between parallel jobs. CI may +cache immutable download transport data, but each case must retain isolated runtime state and must +not substitute a separately installed server for `--download`. + +### Manual workflow + +Allow selection of one language, one server, or one installation family. This is needed to debug a +nightly failure without rerunning the complete matrix. + +Pros of split CI: fast merge feedback plus exhaustive coverage. Cons: a regression affecting a +non-preferred server may be found the following night rather than on the originating pull request. + +Running all 141 pairs on every pull request gives earlier detection, but has much higher latency, +cost, rate-limit exposure, and upstream-flake risk. It is not the recommended default. + +## Failure policy + +Classify failures as: + +1. `lsp-cli` regression; +2. playground/manifest drift; +3. provisioning or network failure; +4. upstream server behavior change; +5. known server limitation; +6. unsupported LSP capability with the expected user-facing response. + +Only category 6 is an immediate passing outcome. Known limitations must be explicit manifest +entries and, when they concern protocol or server behavior, documented in `GOTCHAS.md`. Do not add +unbounded retries. A retry may cover an identified transient installation/network step, but must +not conceal query or protocol failures. + +If a hard-to-debug defect is fixed, add a focused regression test in addition to the broad matrix. +Also consider whether a type invariant, runtime check, clearer trace, or state-dump helper can make +that class of defect easier to diagnose. + +## Execution phases + +### Phase 0: approve boundaries + +- [x] Confirm detectable support versus configured support: use the 16 detectable IDs. +- [x] Confirm that unsupported optional capabilities count as a passing, asserted outcome. +- [x] Approve latest-Mason external server provisioning through `--download`. +- [x] Approve a narrow HTTP endpoint seam for deterministic `update` E2E coverage. +- [x] Confirm PR smoke plus nightly exhaustive CI cadence. + +### Phase 1: foundation + +- [x] Add `tests/e2e.rs` and compact harness modules. +- [x] Add the initial manifest schema and validation. +- [x] Isolate all environment and runtime state. +- [x] Implement deadlines, cleanup guards, JSON helpers, and useful failure diagnostics. +- [x] Prove the harness with Rust/rust-analyzer. +- [x] Cover all 24 subcommand paths with either a real server or a deterministic local fixture. + +### Phase 2: projects + +- [x] Audit the ten existing playgrounds against the common semantic requirements. +- [x] Remove duplicated setup patterns within each class of fixture. +- [x] Add CUDA, Kotlin, Objective-C, and Objective-C++ projects. +- [x] Add `gomod` and `gowork` detection fixtures. +- [x] Update `playground/README.md` with manual reproduction commands. + +### Phase 3: preferred-server smoke matrix + +Manual LSP verification follows server selection and downloader support so it runs against servers +resolved by the same current Mason registry used in CI rather than ambient installations. + +- [x] Configure one production preference per source language and derive the smoke matrix from it. +- [x] Provision servers through `--download`; add no separate installers or Rust dependencies. +- [ ] Run each relevant command manually against every new project. +- [ ] Implement capability-aware query assertions. +- [ ] Implement direct/detached lifecycle scenarios. +- [ ] Add the pull-request E2E job. + +### Phase 4: exhaustive compatibility + +- [ ] Populate manifest entries for all 141 compatible pairs. +- [ ] Provision every non-excluded server and required SDK. +- [ ] Record reviewed exceptions and platform constraints. +- [ ] Add sharded nightly and manual workflows. +- [ ] Verify failures retain server version, command line, capabilities, stderr summary, and cleanup + state. + +### Phase 5: hardening + +- [ ] Run `make test`. +- [ ] Run the full latest-Mason E2E matrix from a clean environment. +- [ ] Check every new or edited test file for boilerplate and duplication. +- [ ] Check every source file remains below 600 lines. +- [ ] Add regression tests for every bug uncovered during rollout. +- [ ] Add LSP/server-specific discoveries to `GOTCHAS.md`. +- [ ] Document how to identify upstream server versions and triage nightly failures. + +## Definition of done + +The work is complete when: + +- every accepted supported language has a committed project or justified metadata-only fixture; +- every compatible supported language/server pair has an executable manifest entry; +- every top-level subcommand has binary-level E2E coverage in its appropriate scope; +- advertised capabilities are exercised and unsupported capabilities have asserted user-facing + behavior; +- direct, detached, stop, and stop-all lifecycle paths are covered; +- formatting and diagnostics cannot dirty tracked files; +- required servers are provisioned through `--download`, and resolved source IDs are retained; +- PR smoke, nightly exhaustive, and manual targeted workflows are documented and passing; +- `make test` passes; +- known protocol/server deviations are recorded in `GOTCHAS.md`; +- no required case is silently skipped. + +## Architectural consequences and limitations + +- The data catalog becomes an enforceable compatibility contract: adding a detectable filetype or + compatible LSP config requires E2E ownership. +- Real-server E2E tests are inherently slower and less hermetic than protocol tests. Unit tests and + fake-server integration tests remain necessary for precise edge cases. +- Capability-aware results mean “all commands tested” does not mean “all commands succeed on every + server.” It means every applicable success path and every inapplicable user-facing response is + verified. +- Latest-server testing creates recurring upstream-flake, security-review, and reproducibility + risks even when this repository is unchanged. +- Some servers require proprietary, platform-specific, or unusually heavy SDKs. Their treatment + must be an explicit product decision rather than an automatic skip. +- The current difference between the 362 configured filetypes and 16 detectable filetypes may need + a future terminology or behavior change in `languages`; this plan exposes but does not decide + that product question. diff --git a/GOTCHAS.md b/GOTCHAS.md index fc00580..4bb2ae0 100644 --- a/GOTCHAS.md +++ b/GOTCHAS.md @@ -72,6 +72,18 @@ # LSP server implementations +## kotlin-lsp + +- The Mason package exposes Kotlin LSP as `intellij-server`, not `kotlin-lsp`. The data config must + use the packaged launcher name so `--download` can resolve it; a display name or upstream product + name is not necessarily an executable name. + +## roslyn-language-server + +- The Mason package exposes Roslyn through the `roslyn-language-server` .NET tool launcher. A data + config containing a literal installation placeholder cannot work with generic `--download`; + launch the Mason-exposed command and let the NuGet backend manage its concrete installation path. + ## lua-language-server - Opening every source file for symbol discovery also triggers diagnostics. The installed diff --git a/Makefile b/Makefile index f99753a..3e1ea25 100644 --- a/Makefile +++ b/Makefile @@ -1,10 +1,26 @@ -test: +.PHONY: test check-format check-tests check-clippy check-readme check-dependencies test-real-server-e2e gen-readme + +test: check-format check-tests check-clippy check-readme check-dependencies + +check-format: cargo fmt --check - cargo test -q - cargo clippy --all-targets --all-features -- -D warnings + cargo fmt --manifest-path tests/e2e/fixtures/fake-lsp/Cargo.toml --check + +check-tests: + cargo test --locked -q + +check-clippy: + cargo clippy --locked --all-targets --all-features -- -D warnings + cargo clippy --locked --manifest-path tests/e2e/fixtures/fake-lsp/Cargo.toml --target-dir target/e2e-fixtures -- -D warnings + +check-readme: python3 scripts/update_readme_commands.py --check + +check-dependencies: cargo deny check +test-real-server-e2e: + cargo test --locked --test e2e real_servers::manifest_real_server_smoke_cases -- --ignored --exact --nocapture + gen-readme: python3 scripts/update_readme_commands.py - diff --git a/TODO.md b/TODO.md index 4d03809..92633cd 100644 --- a/TODO.md +++ b/TODO.md @@ -9,6 +9,8 @@ - man +- fill filetypes + # Features diff --git a/data b/data index 013a75f..59ea883 160000 --- a/data +++ b/data @@ -1 +1 @@ -Subproject commit 013a75f6412917b710aa4683b9c5761c0c679975 +Subproject commit 59ea88365855ca6a5ab35715c931d30c734e1b6e diff --git a/playground/README.md b/playground/README.md index b9dba32..87ac890 100644 --- a/playground/README.md +++ b/playground/README.md @@ -1,43 +1,133 @@ -This directory contains small multi-file projects for manual `lsp-cli` experiments. +# Playground projects -Each language playground is intentionally small but has enough structure to exercise: +This directory contains small projects for reproducing `lsp-cli` behavior manually. Run commands +from the repository root. Build the current binary and list the configured servers for a language +before choosing one: -- `detect` -- `list-files` -- `list-symbols` -- `list-functions` -- `grep` -- `definition` -- `declaration` -- `references` -- `callers` -- `callees` +```sh +cargo build +cargo run -- servers --lang cuda +``` + +The examples below use one server name for orientation, not as the only supported choice. Replace +it with any compatible server printed by `servers --lang`. The selected executable must already be +available in `PATH`; remove `--no-download` only when its configuration supports downloading. + +## Source-language workflow + +The newest source projects share stable order-domain symbols: + +| Language | Project | Example server | File | Type | Function | +|---|---|---|---|---|---| +| CUDA | `playground/cuda` | `clangd` | `src/main.cu` | `Order` | `format_order` | +| Kotlin | `playground/kotlin` | `kotlin-language-server` | `src/main/kotlin/playground/App.kt` | `Order` | `formatOrder` | +| Objective-C | `playground/objc` | `clangd` | `src/main.m` | `Order` | `format_order` | +| Objective-C++ | `playground/objcpp` | `clangd` | `src/main.mm` | `Order` | `format_order` | + +Set the values from one row. This CUDA example can be changed to any other row without changing +the commands below: + +```sh +language=cuda +project=playground/cuda +server=clangd +file="$project/src/main.cu" +symbol=Order +function=format_order +``` + +Start with detection and file selection. These commands do not start a daemon: + +```sh +cargo run -- detect "$project" --lang "$language" --lsp "$server" --no-download +cargo run -- list-files "$project" --lang "$language" --lsp "$server" +``` + +Use a small shell helper to run each request directly against the chosen server: + +```sh +lsp_cli() { + cargo run -- "$@" --lang "$language" --lsp "$server" --no-download --no-detach +} + +lsp_cli server-capabilities "$project" +lsp_cli diagnostics "$project" +lsp_cli build-index "$project" +lsp_cli list-symbols "$project" +lsp_cli list-functions "$project" +lsp_cli grep "$symbol" "$project" +lsp_cli definition "$function" "$project" +lsp_cli declaration "$function" "$project" +lsp_cli references "$function" "$project" +lsp_cli callers "$function" "$project" +lsp_cli callees "$function" "$project" +lsp_cli format "$file" --stdout +``` + +`format --stdout` leaves the tracked project unchanged. Formatting, declarations, diagnostics, +workspace symbols, and call hierarchy are optional LSP capabilities. A clear unsupported-capability +error is therefore a useful result when the selected server does not advertise an operation. + +Exercise daemon reuse separately: + +```sh +cargo run -- daemon "$project" --lang "$language" --lsp "$server" --no-download +cargo run -- server-capabilities "$project" --lang "$language" --lsp "$server" --no-download --detach +cargo run -- stop "$project" --lang "$language" --lsp "$server" +``` + +## Go metadata workflow + +The metadata-only fixtures deliberately isolate filename detection: + +| Language | Project | Example server | +|---|---|---| +| Go module metadata | `playground/gomod` | `gopls` | +| Go workspace metadata | `playground/gowork` | `gopls` | + +Set `language` to `gomod` or `gowork`, then use the corresponding project: + +```sh +language=gomod +project=playground/gomod +server=gopls + +cargo run -- servers --lang "$language" +cargo run -- detect "$project" --lang "$language" --lsp "$server" --no-download +cargo run -- list-files "$project" --lang "$language" --lsp "$server" +cargo run -- server-capabilities "$project" --lang "$language" --lsp "$server" --no-download --no-detach +cargo run -- daemon "$project" --lang "$language" --lsp "$server" --no-download +cargo run -- stop "$project" --lang "$language" --lsp "$server" +``` + +These fixtures can exercise detection, file listing, server initialization, and lifecycle. They do +not contain source symbols, so symbol, reference, formatting, diagnostic, and call-hierarchy +commands have no meaningful metadata-only expectation. + +## Existing project examples -Suggested commands: +The older playgrounds use the same order domain where it is natural. A few useful starting points +are: ```sh -cargo run -- detect playground/python -cargo run -- detect playground/python --lang python -cargo run -- detect playground/python --lsp pyright-langserver -cargo run -- detect playground/c --lang c -cargo run -- detect playground/cpp --lang cpp -cargo run -- grep Order playground/rust -cargo run -- list-symbols playground/c +cargo run -- grep Order playground/rust --lsp rust-analyzer --no-detach cargo run -- list-symbols playground/java/src/main/java/playground/order/Order.java -cargo run -- definition format_order playground/c +cargo run -- definition format_order playground/c --lang c cargo run -- references OrderFormatter playground/csharp -cargo run -- server-capabilities playground/rust --lsp rust-analyzer -cargo run -- daemon playground/python -cargo run -- stop playground/python ``` -Check automatic server selection with -`cargo run -- definition format_order playground/c --lang c` or -`cargo run -- definition format_order playground/cpp --lang cpp`. -Server selection follows the configured preferences and server availability. +The Lua playground instead exercises discovery of the local `normalize_timestamp` function, which +may be absent from LuaLS workspace-symbol results: -The projects reuse a similar domain across languages so symbol names are easy to remember -while trying different LSP servers. +```sh +cargo run -- references normalize_timestamp playground/lua --lsp lua-language-server --detach +``` + +Compare runs with `max-requests-in-flight: 1` and `max-requests-in-flight: 20` in `lsp-cli.yaml`. +Both should report the call in `timestamp.lua`; the setting changes how many file-symbol requests +can be outstanding, not which files are searched. + +## Daemon latency workflow To measure daemon forwarding overhead with a deterministic immediate-reply server: @@ -70,16 +160,6 @@ to queue during that grace period, so memory use can temporarily exceed both lim Process start, shutdown, exit monitoring, and reaping run through a lifecycle worker; daemon logging uses a bounded worker and a 100-millisecond shutdown flush budget. The check holds the global log lock while forwarding and while stopping, and verifies -that overflow is reported after the lock is released. These measurements describe an immediate-reply fixture, not a -latency guarantee for real language servers or stalled peers. - -The Lua playground exercises discovery of the local `normalize_timestamp` function, -which may be absent from LuaLS workspace-symbol results: - -```sh -cargo run -- references normalize_timestamp playground/lua --lsp lua-language-server --detach -``` - -Compare runs with `max-requests-in-flight: 1` and `max-requests-in-flight: 20` in -`lsp-cli.yaml`. Both should report the call in `timestamp.lua`; the setting changes -how many file-symbol requests can be outstanding, not which files are searched. +that overflow is reported after the lock is released. These measurements describe +an immediate-reply fixture, not a latency guarantee for real language servers or +stalled peers. diff --git a/playground/cuda/compile_commands.json b/playground/cuda/compile_commands.json new file mode 100644 index 0000000..57cb3ea --- /dev/null +++ b/playground/cuda/compile_commands.json @@ -0,0 +1,50 @@ +[ + { + "directory": ".", + "arguments": [ + "clang++", + "-x", + "cuda", + "--cuda-host-only", + "-nocudainc", + "-nocudalib", + "-std=c++17", + "-Iinclude", + "-fsyntax-only", + "src/main.cu" + ], + "file": "src/main.cu" + }, + { + "directory": ".", + "arguments": [ + "clang++", + "-x", + "cuda", + "--cuda-host-only", + "-nocudainc", + "-nocudalib", + "-std=c++17", + "-Iinclude", + "-fsyntax-only", + "src/order.cu" + ], + "file": "src/order.cu" + }, + { + "directory": ".", + "arguments": [ + "clang++", + "-x", + "cuda", + "--cuda-host-only", + "-nocudainc", + "-nocudalib", + "-std=c++17", + "-Iinclude", + "-fsyntax-only", + "src/report/formatter.cu" + ], + "file": "src/report/formatter.cu" + } +] diff --git a/playground/cuda/include/order.cuh b/playground/cuda/include/order.cuh new file mode 100644 index 0000000..f8c71e5 --- /dev/null +++ b/playground/cuda/include/order.cuh @@ -0,0 +1,24 @@ +#ifndef PLAYGROUND_CUDA_ORDER_CUH +#define PLAYGROUND_CUDA_ORDER_CUH + +#include + +struct OrderItem { + const char *name; + int quantity; + double price; + + __host__ __device__ double total() const; +}; + +struct Order { + const char *customer; + const OrderItem *items; + std::size_t item_count; + + __host__ __device__ double total() const; +}; + +Order sample_order(); + +#endif diff --git a/playground/cuda/include/report.cuh b/playground/cuda/include/report.cuh new file mode 100644 index 0000000..24ace10 --- /dev/null +++ b/playground/cuda/include/report.cuh @@ -0,0 +1,10 @@ +#ifndef PLAYGROUND_CUDA_REPORT_CUH +#define PLAYGROUND_CUDA_REPORT_CUH + +#include + +#include "order.cuh" + +std::string format_order(const Order &order); + +#endif diff --git a/playground/cuda/src/main.cu b/playground/cuda/src/main.cu new file mode 100644 index 0000000..eb2c34b --- /dev/null +++ b/playground/cuda/src/main.cu @@ -0,0 +1,14 @@ +#include "order.cuh" +#include "report.cuh" + +#include + +__global__ void calculate_order_total(Order order, double *result) { + *result = order.total(); +} + +int main() { + const Order order = sample_order(); + std::cout << format_order(order) << '\n'; + return 0; +} diff --git a/playground/cuda/src/order.cu b/playground/cuda/src/order.cu new file mode 100644 index 0000000..42b9856 --- /dev/null +++ b/playground/cuda/src/order.cu @@ -0,0 +1,21 @@ +#include "order.cuh" + +__host__ __device__ double OrderItem::total() const { + return static_cast(quantity) * price; +} + +__host__ __device__ double Order::total() const { + double value = 0.0; + for (std::size_t index = 0; index < item_count; ++index) { + value += items[index].total(); + } + return value; +} + +Order sample_order() { + static const OrderItem items[] = { + {"GPU", 1, 499.0}, + {"Power Cable", 2, 12.5}, + }; + return {"Jensen", items, 2}; +} diff --git a/playground/cuda/src/report/formatter.cu b/playground/cuda/src/report/formatter.cu new file mode 100644 index 0000000..d6b8514 --- /dev/null +++ b/playground/cuda/src/report/formatter.cu @@ -0,0 +1,10 @@ +#include "report.cuh" + +#include + +std::string format_order(const Order &order) { + std::ostringstream output; + output << order.customer << " has " << order.item_count << " items worth " + << order.total(); + return output.str(); +} diff --git a/playground/gomod/go.mod b/playground/gomod/go.mod new file mode 100644 index 0000000..a37eedc --- /dev/null +++ b/playground/gomod/go.mod @@ -0,0 +1,3 @@ +module example.com/lsp-cli-playground-gomod + +go 1.22 diff --git a/playground/gowork/go.work b/playground/gowork/go.work new file mode 100644 index 0000000..233b100 --- /dev/null +++ b/playground/gowork/go.work @@ -0,0 +1 @@ +go 1.22 diff --git a/playground/kotlin/settings.gradle.kts b/playground/kotlin/settings.gradle.kts new file mode 100644 index 0000000..27118e9 --- /dev/null +++ b/playground/kotlin/settings.gradle.kts @@ -0,0 +1 @@ +rootProject.name = "lsp-cli-playground-kotlin" diff --git a/playground/kotlin/src/main/kotlin/playground/App.kt b/playground/kotlin/src/main/kotlin/playground/App.kt new file mode 100644 index 0000000..5a77eb2 --- /dev/null +++ b/playground/kotlin/src/main/kotlin/playground/App.kt @@ -0,0 +1,9 @@ +package playground + +import playground.order.sampleOrder +import playground.report.formatOrder + +fun main() { + val order = sampleOrder() + println(formatOrder(order)) +} diff --git a/playground/kotlin/src/main/kotlin/playground/order/Order.kt b/playground/kotlin/src/main/kotlin/playground/order/Order.kt new file mode 100644 index 0000000..afa49eb --- /dev/null +++ b/playground/kotlin/src/main/kotlin/playground/order/Order.kt @@ -0,0 +1,20 @@ +package playground.order + +interface OrderTotal { + fun total(): Double +} + +data class OrderItem( + val name: String, + val quantity: Int, + val price: Double +) : OrderTotal { + override fun total(): Double = quantity * price +} + +data class Order( + val customer: String, + val items: List +) : OrderTotal { + override fun total(): Double = items.map { item -> item.total() }.sum() +} diff --git a/playground/kotlin/src/main/kotlin/playground/order/OrderService.kt b/playground/kotlin/src/main/kotlin/playground/order/OrderService.kt new file mode 100644 index 0000000..f92875c --- /dev/null +++ b/playground/kotlin/src/main/kotlin/playground/order/OrderService.kt @@ -0,0 +1,9 @@ +package playground.order + +fun sampleOrder(): Order = Order( + customer = "JetBrains", + items = listOf( + OrderItem(name = "Keyboard", quantity = 1, price = 120.0), + OrderItem(name = "Keycap", quantity = 4, price = 3.5) + ) +) diff --git a/playground/kotlin/src/main/kotlin/playground/report/OrderFormatter.kt b/playground/kotlin/src/main/kotlin/playground/report/OrderFormatter.kt new file mode 100644 index 0000000..f65e3ef --- /dev/null +++ b/playground/kotlin/src/main/kotlin/playground/report/OrderFormatter.kt @@ -0,0 +1,6 @@ +package playground.report + +import playground.order.Order + +fun formatOrder(order: Order): String = + "${order.customer} has ${order.items.size} items worth ${"%.2f".format(order.total())}" diff --git a/playground/objc/compile_commands.json b/playground/objc/compile_commands.json new file mode 100644 index 0000000..7c6e5d2 --- /dev/null +++ b/playground/objc/compile_commands.json @@ -0,0 +1,17 @@ +[ + { + "directory": ".", + "arguments": ["clang", "-x", "objective-c", "-Iinclude", "-fsyntax-only", "src/main.m"], + "file": "src/main.m" + }, + { + "directory": ".", + "arguments": ["clang", "-x", "objective-c", "-Iinclude", "-fsyntax-only", "src/Order.m"], + "file": "src/Order.m" + }, + { + "directory": ".", + "arguments": ["clang", "-x", "objective-c", "-Iinclude", "-fsyntax-only", "src/OrderFormatter.m"], + "file": "src/OrderFormatter.m" + } +] diff --git a/playground/objc/include/Order.h b/playground/objc/include/Order.h new file mode 100644 index 0000000..c383d5c --- /dev/null +++ b/playground/objc/include/Order.h @@ -0,0 +1,32 @@ +#ifndef PLAYGROUND_OBJC_ORDER_H +#define PLAYGROUND_OBJC_ORDER_H + +#include + +typedef struct { + const char *name; + int quantity; + double price; +} OrderItem; + +__attribute__((objc_root_class)) +@interface Order { +@private + const char *_customer; + const OrderItem *_items; + size_t _itemCount; +} + +- (instancetype)initWithCustomer:(const char *)customer + items:(const OrderItem *)items + count:(size_t)itemCount; +- (const char *)customer; +- (size_t)itemCount; +- (double)total; + +@end + +double item_total(OrderItem item); +Order *sample_order(void); + +#endif diff --git a/playground/objc/include/OrderFormatter.h b/playground/objc/include/OrderFormatter.h new file mode 100644 index 0000000..33258ae --- /dev/null +++ b/playground/objc/include/OrderFormatter.h @@ -0,0 +1,10 @@ +#ifndef PLAYGROUND_OBJC_ORDER_FORMATTER_H +#define PLAYGROUND_OBJC_ORDER_FORMATTER_H + +#include + +#include "Order.h" + +void format_order(Order *order, char *buffer, size_t buffer_size); + +#endif diff --git a/playground/objc/src/Order.m b/playground/objc/src/Order.m new file mode 100644 index 0000000..83af5f8 --- /dev/null +++ b/playground/objc/src/Order.m @@ -0,0 +1,41 @@ +#include "Order.h" + +@implementation Order + +- (instancetype)initWithCustomer:(const char *)customer + items:(const OrderItem *)items + count:(size_t)itemCount { + _customer = customer; + _items = items; + _itemCount = itemCount; + return self; +} + +- (const char *)customer { + return _customer; +} + +- (size_t)itemCount { + return _itemCount; +} + +- (double)total { + double value = 0.0; + for (size_t index = 0; index < _itemCount; ++index) { + value += item_total(_items[index]); + } + return value; +} + +@end + +double item_total(OrderItem item) { return item.quantity * item.price; } + +Order *sample_order(void) { + static const OrderItem items[] = { + {"Display", 1, 600.0}, + {"Stand", 1, 80.0}, + }; + Order *order = (Order *)0; + return [order initWithCustomer:"Brad" items:items count:2]; +} diff --git a/playground/objc/src/OrderFormatter.m b/playground/objc/src/OrderFormatter.m new file mode 100644 index 0000000..16c96b6 --- /dev/null +++ b/playground/objc/src/OrderFormatter.m @@ -0,0 +1,12 @@ +#include "OrderFormatter.h" + +#include + +void format_order(Order *order, char *buffer, size_t buffer_size) { + if (order == (Order *)0) { + snprintf(buffer, buffer_size, "empty order"); + return; + } + snprintf(buffer, buffer_size, "%s has %zu items worth %.2f", [order customer], + [order itemCount], [order total]); +} diff --git a/playground/objc/src/main.m b/playground/objc/src/main.m new file mode 100644 index 0000000..5057ece --- /dev/null +++ b/playground/objc/src/main.m @@ -0,0 +1,12 @@ +#include "Order.h" +#include "OrderFormatter.h" + +#include + +int main(void) { + Order *order = sample_order(); + char summary[128]; + format_order(order, summary, sizeof(summary)); + puts(summary); + return 0; +} diff --git a/playground/objcpp/compile_commands.json b/playground/objcpp/compile_commands.json new file mode 100644 index 0000000..9fbd37b --- /dev/null +++ b/playground/objcpp/compile_commands.json @@ -0,0 +1,17 @@ +[ + { + "directory": ".", + "arguments": ["clang++", "-x", "objective-c++", "-std=c++17", "-Iinclude", "-fsyntax-only", "src/main.mm"], + "file": "src/main.mm" + }, + { + "directory": ".", + "arguments": ["clang++", "-x", "objective-c++", "-std=c++17", "-Iinclude", "-fsyntax-only", "src/Order.mm"], + "file": "src/Order.mm" + }, + { + "directory": ".", + "arguments": ["clang++", "-x", "objective-c++", "-std=c++17", "-Iinclude", "-fsyntax-only", "src/OrderFormatter.mm"], + "file": "src/OrderFormatter.mm" + } +] diff --git a/playground/objcpp/include/Order.hpp b/playground/objcpp/include/Order.hpp new file mode 100644 index 0000000..9c2778d --- /dev/null +++ b/playground/objcpp/include/Order.hpp @@ -0,0 +1,36 @@ +#ifndef PLAYGROUND_OBJCPP_ORDER_HPP +#define PLAYGROUND_OBJCPP_ORDER_HPP + +#include +#include + +struct OrderItem { + std::string name; + int quantity; + double price; + + double total() const; +}; + +@protocol OrderTotaling +- (double)total; +@end + +__attribute__((objc_root_class)) +@interface Order { +@private + std::string _customer; + std::vector _items; +} + +- (instancetype)initWithCustomer:(std::string)customer + items:(std::vector)items; +- (const std::string &)customer; +- (const std::vector &)items; +- (double)total; + +@end + +Order *sample_order(); + +#endif diff --git a/playground/objcpp/include/OrderFormatter.hpp b/playground/objcpp/include/OrderFormatter.hpp new file mode 100644 index 0000000..3c5e0dc --- /dev/null +++ b/playground/objcpp/include/OrderFormatter.hpp @@ -0,0 +1,10 @@ +#ifndef PLAYGROUND_OBJCPP_ORDER_FORMATTER_HPP +#define PLAYGROUND_OBJCPP_ORDER_FORMATTER_HPP + +#include + +#include "Order.hpp" + +std::string format_order(Order *order); + +#endif diff --git a/playground/objcpp/src/Order.mm b/playground/objcpp/src/Order.mm new file mode 100644 index 0000000..de2ed17 --- /dev/null +++ b/playground/objcpp/src/Order.mm @@ -0,0 +1,43 @@ +#include "Order.hpp" + +#include + +double OrderItem::total() const { + return static_cast(quantity) * price; +} + +@implementation Order + +- (instancetype)initWithCustomer:(std::string)customer + items:(std::vector)items { + _customer = std::move(customer); + _items = std::move(items); + return self; +} + +- (const std::string &)customer { + return _customer; +} + +- (const std::vector &)items { + return _items; +} + +- (double)total { + double value = 0.0; + for (const OrderItem &item : _items) { + value += item.total(); + } + return value; +} + +@end + +Order *sample_order() { + Order *order = (Order *)nullptr; + std::vector items = { + {"Compiler", 1, 75.0}, + {"Book", 2, 42.0}, + }; + return [order initWithCustomer:std::string("Bjarne") items:std::move(items)]; +} diff --git a/playground/objcpp/src/OrderFormatter.mm b/playground/objcpp/src/OrderFormatter.mm new file mode 100644 index 0000000..9e650e0 --- /dev/null +++ b/playground/objcpp/src/OrderFormatter.mm @@ -0,0 +1,13 @@ +#include "OrderFormatter.hpp" + +#include + +std::string format_order(Order *order) { + if (order == nullptr) { + return "empty order"; + } + std::ostringstream output; + output << [order customer] << " has " << [order items].size() + << " items worth " << [order total]; + return output.str(); +} diff --git a/playground/objcpp/src/main.mm b/playground/objcpp/src/main.mm new file mode 100644 index 0000000..15dc38c --- /dev/null +++ b/playground/objcpp/src/main.mm @@ -0,0 +1,10 @@ +#include "Order.hpp" +#include "OrderFormatter.hpp" + +#include + +int main() { + Order *order = sample_order(); + std::cout << format_order(order) << '\n'; + return 0; +} diff --git a/src/env_vars.rs b/src/env_vars.rs index b94f5e8..bc8092b 100644 --- a/src/env_vars.rs +++ b/src/env_vars.rs @@ -19,6 +19,9 @@ pub(crate) const PATH: &str = "PATH"; /// Current interactive shell used for shell auto-detection in completion output. pub(crate) const SHELL: &str = "SHELL"; +/// Override the release metadata endpoint, primarily for isolated update testing. +pub(crate) const DATA_RELEASE_API_URL: &str = "LSP_CLI_DATA_RELEASE_API_URL"; + #[cfg(test)] /// Test-only override that tells fake npm installs which executable to materialize. pub(crate) const TEST_FAKE_NPM_PROGRAM: &str = "LSP_CLI_TEST_FAKE_NPM_PROGRAM"; @@ -51,6 +54,10 @@ pub(crate) fn shell() -> Option { std::env::var_os(SHELL) } +pub(crate) fn data_release_api_url() -> Option { + std::env::var(DATA_RELEASE_API_URL).ok() +} + #[cfg(test)] pub(crate) fn fake_npm_program() -> Option { std::env::var_os(TEST_FAKE_NPM_PROGRAM) diff --git a/src/mason/install.rs b/src/mason/install.rs index 8b5bafe..a1116ff 100644 --- a/src/mason/install.rs +++ b/src/mason/install.rs @@ -36,7 +36,8 @@ pub(crate) fn resolve_cached_program( SourceId::Npm { .. } | SourceId::Pypi { .. } | SourceId::Cargo { .. } - | SourceId::Golang { .. } => { + | SourceId::Golang { .. } + | SourceId::Nuget { .. } => { let resolved_program = resolve_program(package, program, state, &TemplateContext::empty())?; Ok(if is_resolved_program_runnable(&resolved_program) { @@ -78,6 +79,10 @@ pub(crate) fn resolve_or_install_program( module_path, version, } => install_golang_package(state, package, &module_path, &version, program), + SourceId::Nuget { + package_name, + version, + } => install_nuget_package(state, package, &package_name, &version, program), SourceId::Github { repository, version, @@ -319,6 +324,55 @@ fn install_golang_package( ) } +fn install_nuget_package( + state: &RuntimeState, + package: &MasonPackage, + package_name: &str, + version: &str, + program: &str, +) -> Result { + use_cached_program_or( + package, + program, + state, + &TemplateContext::empty(), + |resolved_program| { + require_command("dotnet", package, program)?; + let install_dir = prepare_install_dir(state, package)?.join("bin"); + crate::fs::create_dir_all(&install_dir)?; + + let mut cmd = nuget_install_command(package_name, version, &install_dir); + run_install_command(&mut cmd, package, "dotnet tool install")?; + + finalize_install( + state, + package, + program, + &resolved_program, + &TemplateContext::empty(), + "dotnet did not produce a runnable", + ) + }, + ) +} + +fn nuget_install_command( + package_name: &str, + version: &str, + install_dir: &std::path::Path, +) -> Command { + let mut command = Command::new("dotnet"); + command + .arg("tool") + .arg("install") + .arg(package_name) + .arg("--tool-path") + .arg(install_dir) + .arg("--version") + .arg(version); + command +} + fn install_github_package( state: &RuntimeState, package: &MasonPackage, diff --git a/src/mason/install/tests.rs b/src/mason/install/tests.rs index 25a40f6..da8b9e3 100644 --- a/src/mason/install/tests.rs +++ b/src/mason/install/tests.rs @@ -1,4 +1,15 @@ -use super::artifacts::parse_archive_file_spec; +use std::path::Path; + +#[cfg(unix)] +use std::fs; + +use super::{ + artifacts::parse_archive_file_spec, nuget_install_command, resolve_or_install_program, +}; +#[cfg(unix)] +use crate::runtime_state::RuntimeState; +#[cfg(unix)] +use crate::test_support::{TestDir, env_var, make_executable, roslyn_package, with_env_vars}; #[test] fn parses_archive_file_spec() { @@ -14,3 +25,59 @@ fn parses_archive_file_spec() { ("clangd-linux-22.1.0.zip", None) ); } + +#[test] +fn builds_exact_nuget_tool_install_command() { + let command = nuget_install_command( + "roslyn-language-server", + "5.11.0-1.26380.4", + Path::new("managed/bin"), + ); + + assert_eq!(command.get_program(), "dotnet"); + assert_eq!( + command.get_args().collect::>(), + [ + "tool", + "install", + "roslyn-language-server", + "--tool-path", + "managed/bin", + "--version", + "5.11.0-1.26380.4", + ] + ); +} + +#[cfg(unix)] +#[test] +fn installs_and_caches_nuget_tool_with_a_receipt() { + let dir = TestDir::new("mason-nuget-install"); + let state = RuntimeState::new(dir.path().join("state")); + let bin_dir = dir.path().join("bin"); + fs::create_dir_all(&bin_dir).expect("fake program directory should exist"); + let dotnet = dir.write_file( + "bin/dotnet", + "#!/bin/sh\nwhile [ \"$#\" -gt 0 ]; do\n if [ \"$1\" = \"--tool-path\" ]; then\n shift\n tool_path=$1\n fi\n shift\ndone\n/bin/mkdir -p \"$tool_path\"\n: > \"$tool_path/roslyn-language-server\"\n/bin/chmod 755 \"$tool_path/roslyn-language-server\"\n", + ); + make_executable(&dotnet); + + let installed = with_env_vars(&[env_var("PATH", &bin_dir)], || { + resolve_or_install_program(&state, &roslyn_package(), "roslyn-language-server") + .expect("NuGet tool should install") + }); + + assert_eq!( + installed, + state + .package_dir("roslyn-language-server") + .join("bin/roslyn-language-server") + ); + let receipt = fs::read_to_string(state.receipt_path("roslyn-language-server")) + .expect("install receipt should exist"); + assert!(receipt.contains("pkg:nuget/roslyn-language-server@5.11.0-1.26380.4")); + + let cached = resolve_or_install_program(&state, &roslyn_package(), "roslyn-language-server") + .expect("installed NuGet tool should be reusable from cache"); + assert_eq!(cached, installed); +} diff --git a/src/mason/link.rs b/src/mason/link.rs index 6f2eeb5..effe801 100644 --- a/src/mason/link.rs +++ b/src/mason/link.rs @@ -115,6 +115,16 @@ pub(crate) fn resolve_program( )); } + if let Some(relative) = rendered.strip_prefix("nuget:") { + let executable = format!("{relative}{}", std::env::consts::EXE_SUFFIX); + return Ok(ResolvedProgram::Direct( + state + .package_dir(&package.name) + .join("bin") + .join(executable), + )); + } + if let Some(relative) = rendered.strip_prefix("python:") { return Ok(ResolvedProgram::Wrapper(WrapperProgram { launcher_path: state.bin_dir().join(program), diff --git a/src/mason/link/tests.rs b/src/mason/link/tests.rs index 1347e5b..3d8eb80 100644 --- a/src/mason/link/tests.rs +++ b/src/mason/link/tests.rs @@ -9,8 +9,8 @@ use crate::mason::registry::{ use crate::mason::template::TemplateContext; use crate::runtime_state::RuntimeState; use crate::test_support::{ - TestDir, env_var, jdtls_package, make_executable, pyright_package, suggested_language, - with_env_vars, + TestDir, env_var, jdtls_package, make_executable, pyright_package, roslyn_package, + suggested_language, with_env_vars, }; use std::collections::BTreeMap; use std::fs; @@ -196,6 +196,29 @@ fn resolves_npm_program_path() { ); } +#[test] +fn resolves_nuget_program_path() { + let dir = TestDir::new("mason-link"); + let state = RuntimeState::new(dir.path().join("state")); + + assert_eq!( + resolve_program_path( + &roslyn_package(), + "roslyn-language-server", + &state, + &TemplateContext::empty(), + ) + .expect("NuGet path should resolve"), + state + .package_dir("roslyn-language-server") + .join("bin") + .join(format!( + "roslyn-language-server{}", + std::env::consts::EXE_SUFFIX + )) + ); +} + #[test] fn resolves_github_template_program_path() { let dir = TestDir::new("mason-link"); diff --git a/src/mason/source.rs b/src/mason/source.rs index dab4a05..dba556a 100644 --- a/src/mason/source.rs +++ b/src/mason/source.rs @@ -19,6 +19,10 @@ pub(crate) enum SourceId { module_path: String, version: String, }, + Nuget { + package_name: String, + version: String, + }, Github { repository: String, version: String, @@ -77,6 +81,10 @@ pub(crate) fn parse_source_id(source_id: &str) -> Result { module_path: decoded_name, version, }, + "nuget" => SourceId::Nuget { + package_name: decoded_name, + version, + }, "github" => SourceId::Github { repository: decoded_name, version, @@ -186,6 +194,14 @@ mod tests { version: "v0.21.1".to_string(), } ); + assert_eq!( + parse_source_id("pkg:nuget/roslyn-language-server@5.11.0-1.26380.4") + .expect("NuGet source should parse"), + SourceId::Nuget { + package_name: "roslyn-language-server".to_string(), + version: "5.11.0-1.26380.4".to_string(), + } + ); } #[test] diff --git a/src/suggest.rs b/src/suggest.rs index f7a2ed9..47b6fcb 100644 --- a/src/suggest.rs +++ b/src/suggest.rs @@ -191,7 +191,7 @@ mod tests { LspConfig { id: "example_lsp".to_string(), filetypes: vec!["alpha".to_string(), "beta".to_string()], - root_markers: vec![".workspace-root".to_string(), ".git".to_string()], + root_markers: vec![".workspace-root".to_string()], name: "example-lsp".to_string(), cmdline: "example-lsp --stdio $WORKSPACE".to_string(), wait_for_index: false, diff --git a/src/test_support.rs b/src/test_support.rs index 52c38cb..64a8d61 100644 --- a/src/test_support.rs +++ b/src/test_support.rs @@ -1,4 +1,5 @@ pub(crate) mod lsp_peer; +mod temp_root; use crate::detect::DetectionResult; use crate::mason::registry::{MasonDownload, MasonNeovim, MasonPackage, MasonSource, OneOrMany}; @@ -12,6 +13,8 @@ use std::sync::{Mutex, OnceLock}; use std::time::{SystemTime, UNIX_EPOCH}; use tempfile::TempDir; +use self::temp_root::test_temp_root; + pub(crate) const LOCAL_SHARE_LSP_CLI: &str = ".local/share/lsp-cli"; #[cfg(test)] pub(crate) const SUBPROCESS_HELPER_MODE_ENV: &str = "LSP_CLI_TEST_HELPER_MODE"; @@ -30,9 +33,11 @@ pub(crate) struct TestDir { impl TestDir { pub(crate) fn new(prefix: &str) -> Self { + let test_temp_root = test_temp_root().expect("secure test temp root should be selected"); + fs::create_dir_all(&test_temp_root).expect("test temp root should be created"); let dir = tempfile::Builder::new() .prefix(&format!("lsp-cli-{prefix}-test-")) - .tempdir() + .tempdir_in(test_temp_root) .expect("temp dir should be created"); Self { dir } } @@ -173,6 +178,28 @@ pub(crate) fn pyright_package() -> MasonPackage { } } +pub(crate) fn roslyn_package() -> MasonPackage { + MasonPackage { + name: "roslyn-language-server".to_string(), + categories: vec!["LSP".to_string()], + source: MasonSource { + id: "pkg:nuget/roslyn-language-server@5.11.0-1.26380.4".to_string(), + extra_packages: Vec::new(), + asset: None, + download: None, + version_overrides: Vec::new(), + }, + bin: BTreeMap::from([( + "roslyn-language-server".to_string(), + "nuget:roslyn-language-server".to_string(), + )]), + share: BTreeMap::new(), + neovim: MasonNeovim { + lspconfig: Some("roslyn_ls".to_string()), + }, + } +} + pub(crate) fn jdtls_package() -> MasonPackage { MasonPackage { name: "jdtls".to_string(), @@ -279,11 +306,22 @@ pub(crate) fn make_executable(path: &Path) { mod tests { use super::{ SUBPROCESS_HELPER_EXIT_CODE_ENV, SUBPROCESS_HELPER_MODE_ENV, - SUBPROCESS_HELPER_OUTPUT_PATH_ENV, SUBPROCESS_HELPER_STDERR_ENV, + SUBPROCESS_HELPER_OUTPUT_PATH_ENV, SUBPROCESS_HELPER_STDERR_ENV, TestDir, test_temp_root, }; use std::fs; use std::io::Write as _; + #[test] + fn creates_test_directories_in_secure_selected_root() { + let dir = TestDir::new("secure-root"); + + assert!( + dir.path() + .starts_with(test_temp_root().expect("secure test temp root should be selected")) + ); + assert!(!dir.path().starts_with("/tmp")); + } + #[test] #[ignore = "subprocess helper"] fn subprocess_helper() { diff --git a/src/test_support/temp_root.rs b/src/test_support/temp_root.rs new file mode 100644 index 0000000..244b7ff --- /dev/null +++ b/src/test_support/temp_root.rs @@ -0,0 +1,26 @@ +use std::io; +use std::path::{Path, PathBuf}; + +pub(crate) fn test_temp_root() -> io::Result { + Ok(test_temp_base()?.join("lsp-cli/test-tmp")) +} + +pub(crate) fn test_temp_base() -> io::Result { + option_env!("XDG_RUNTIME_DIR") + .map(PathBuf::from) + .filter(|path| acceptable_test_temp_base(path)) + .or_else(|| option_env!("XDG_CACHE_HOME").map(PathBuf::from)) + .filter(|path| acceptable_test_temp_base(path)) + .or_else(|| option_env!("HOME").map(|home| PathBuf::from(home).join(".cache"))) + .filter(|path| acceptable_test_temp_base(path)) + .ok_or_else(|| { + io::Error::new( + io::ErrorKind::NotFound, + "an absolute XDG_RUNTIME_DIR, XDG_CACHE_HOME, or HOME outside /tmp is required", + ) + }) +} + +fn acceptable_test_temp_base(path: &Path) -> bool { + path.is_absolute() && !path.starts_with("/tmp") +} diff --git a/src/update.rs b/src/update.rs index a6c2abb..2ebc84a 100644 --- a/src/update.rs +++ b/src/update.rs @@ -259,11 +259,7 @@ struct ReleaseDownload { } fn fetch_release(client: &Client, version: &str) -> Result { - let url = if version == "latest" { - format!("https://api.github.com/repos/{DATA_REPOSITORY}/releases/latest") - } else { - format!("https://api.github.com/repos/{DATA_REPOSITORY}/releases/tags/{version}") - }; + let url = crate::env_vars::data_release_api_url().unwrap_or_else(|| release_url(version)); let release: GithubRelease = client .get(url) .send() @@ -292,5 +288,13 @@ fn fetch_release(client: &Client, version: &str) -> Result { }) } +fn release_url(version: &str) -> String { + if version == "latest" { + format!("https://api.github.com/repos/{DATA_REPOSITORY}/releases/latest") + } else { + format!("https://api.github.com/repos/{DATA_REPOSITORY}/releases/tags/{version}") + } +} + #[cfg(test)] mod tests; diff --git a/src/update/tests.rs b/src/update/tests.rs index eb1dfc3..1668ce6 100644 --- a/src/update/tests.rs +++ b/src/update/tests.rs @@ -1,4 +1,4 @@ -use super::{install_downloaded_data, locate_data_root}; +use super::{install_downloaded_data, locate_data_root, release_url}; use crate::runtime_state::RuntimeState; use crate::test_support::TestDir; use flate2::Compression; @@ -29,6 +29,12 @@ fn archive_with_files(files: &[(&str, &[u8])]) -> Vec { encoder.finish().expect("gzip should finish") } +#[test] +fn builds_latest_and_tagged_release_urls() { + assert!(release_url("latest").ends_with("/releases/latest")); + assert!(release_url("v1.2.3").ends_with("/releases/tags/v1.2.3")); +} + #[test] fn installs_valid_downloaded_data_into_runtime_data_dir() { let dir = TestDir::new("update-install"); diff --git a/tests/e2e.rs b/tests/e2e.rs new file mode 100644 index 0000000..3c1ed29 --- /dev/null +++ b/tests/e2e.rs @@ -0,0 +1,41 @@ +#![expect( + clippy::panic, + reason = "E2E assertion helpers panic with captured process diagnostics." +)] +#![expect( + clippy::expect_used, + reason = "E2E fixtures and assertions fail immediately with contextual expectation messages." +)] + +#[path = "e2e/case_files.rs"] +mod case_files; +#[path = "e2e/catalog.rs"] +mod catalog; +#[path = "e2e/filesystem.rs"] +mod filesystem; +#[path = "e2e/fixture.rs"] +mod fixture; +#[path = "e2e/harness.rs"] +mod harness; +#[path = "e2e/lifecycle.rs"] +mod lifecycle; +#[path = "e2e/local_fixture.rs"] +mod local_fixture; +#[path = "e2e/manifest.rs"] +mod manifest; +#[path = "e2e/manifest_data.rs"] +mod manifest_data; +#[path = "e2e/process.rs"] +mod process; +#[path = "e2e/queries.rs"] +mod queries; +#[path = "e2e/real_servers.rs"] +mod real_servers; +#[path = "e2e/update.rs"] +mod update; + +use std::path::Path; + +pub(crate) fn repository_root() -> &'static Path { + Path::new(env!("CARGO_MANIFEST_DIR")) +} diff --git a/tests/e2e/case_files.rs b/tests/e2e/case_files.rs new file mode 100644 index 0000000..56c47da --- /dev/null +++ b/tests/e2e/case_files.rs @@ -0,0 +1,33 @@ +use std::fs; +use std::path::{Path, PathBuf}; + +use serde::de::DeserializeOwned; + +pub(crate) fn yaml_paths(directory: &Path) -> Result, String> { + let entries = fs::read_dir(directory) + .map_err(|error| format!("failed to read {}: {error}", directory.display()))?; + let mut paths = entries + .map(|entry| { + entry + .map(|entry| entry.path()) + .map_err(|error| format!("failed to read {}: {error}", directory.display())) + }) + .collect::, _>>()?; + paths.retain(|path| path.extension().and_then(|extension| extension.to_str()) == Some("yaml")); + paths.sort(); + Ok(paths) +} + +pub(crate) fn read_yaml(path: &Path) -> Result { + let contents = fs::read_to_string(path) + .map_err(|error| format!("failed to read {}: {error}", path.display()))?; + serde_yaml::from_str(&contents) + .map_err(|error| format!("failed to parse {}: {error}", path.display())) +} + +pub(crate) fn file_stem(path: &Path) -> Result { + path.file_stem() + .and_then(|stem| stem.to_str()) + .map(str::to_owned) + .ok_or_else(|| format!("{} has no UTF-8 file stem", path.display())) +} diff --git a/tests/e2e/cases/c.yaml b/tests/e2e/cases/c.yaml new file mode 100644 index 0000000..88cdb47 --- /dev/null +++ b/tests/e2e/cases/c.yaml @@ -0,0 +1,8 @@ +language: + id: c + kind: source + project: playground/c + +pairs: + - language: c + server: clangd diff --git a/tests/e2e/cases/cpp.yaml b/tests/e2e/cases/cpp.yaml new file mode 100644 index 0000000..1c27cf5 --- /dev/null +++ b/tests/e2e/cases/cpp.yaml @@ -0,0 +1,8 @@ +language: + id: cpp + kind: source + project: playground/cpp + +pairs: + - language: cpp + server: clangd diff --git a/tests/e2e/cases/cs.yaml b/tests/e2e/cases/cs.yaml new file mode 100644 index 0000000..55d061f --- /dev/null +++ b/tests/e2e/cases/cs.yaml @@ -0,0 +1,8 @@ +language: + id: cs + kind: source + project: playground/csharp + +pairs: + - language: cs + server: roslyn_ls diff --git a/tests/e2e/cases/cuda.yaml b/tests/e2e/cases/cuda.yaml new file mode 100644 index 0000000..2b8741d --- /dev/null +++ b/tests/e2e/cases/cuda.yaml @@ -0,0 +1,8 @@ +language: + id: cuda + kind: source + project: playground/cuda + +pairs: + - language: cuda + server: clangd diff --git a/tests/e2e/cases/go.yaml b/tests/e2e/cases/go.yaml new file mode 100644 index 0000000..8200a4e --- /dev/null +++ b/tests/e2e/cases/go.yaml @@ -0,0 +1,8 @@ +language: + id: go + kind: source + project: playground/go + +pairs: + - language: go + server: gopls diff --git a/tests/e2e/cases/gomod.yaml b/tests/e2e/cases/gomod.yaml new file mode 100644 index 0000000..83deac0 --- /dev/null +++ b/tests/e2e/cases/gomod.yaml @@ -0,0 +1,4 @@ +language: + id: gomod + kind: metadata + project: playground/gomod diff --git a/tests/e2e/cases/gowork.yaml b/tests/e2e/cases/gowork.yaml new file mode 100644 index 0000000..b6696f0 --- /dev/null +++ b/tests/e2e/cases/gowork.yaml @@ -0,0 +1,4 @@ +language: + id: gowork + kind: metadata + project: playground/gowork diff --git a/tests/e2e/cases/java.yaml b/tests/e2e/cases/java.yaml new file mode 100644 index 0000000..1e8bae9 --- /dev/null +++ b/tests/e2e/cases/java.yaml @@ -0,0 +1,8 @@ +language: + id: java + kind: source + project: playground/java + +pairs: + - language: java + server: jdtls diff --git a/tests/e2e/cases/javascript.yaml b/tests/e2e/cases/javascript.yaml new file mode 100644 index 0000000..9e585fd --- /dev/null +++ b/tests/e2e/cases/javascript.yaml @@ -0,0 +1,8 @@ +language: + id: javascript + kind: source + project: playground/js + +pairs: + - language: javascript + server: ts_ls diff --git a/tests/e2e/cases/kotlin.yaml b/tests/e2e/cases/kotlin.yaml new file mode 100644 index 0000000..518587f --- /dev/null +++ b/tests/e2e/cases/kotlin.yaml @@ -0,0 +1,8 @@ +language: + id: kotlin + kind: source + project: playground/kotlin + +pairs: + - language: kotlin + server: kotlin_lsp diff --git a/tests/e2e/cases/lua.yaml b/tests/e2e/cases/lua.yaml new file mode 100644 index 0000000..e4b6e93 --- /dev/null +++ b/tests/e2e/cases/lua.yaml @@ -0,0 +1,8 @@ +language: + id: lua + kind: source + project: playground/lua + +pairs: + - language: lua + server: lua_ls diff --git a/tests/e2e/cases/objc.yaml b/tests/e2e/cases/objc.yaml new file mode 100644 index 0000000..32e11af --- /dev/null +++ b/tests/e2e/cases/objc.yaml @@ -0,0 +1,8 @@ +language: + id: objc + kind: source + project: playground/objc + +pairs: + - language: objc + server: clangd diff --git a/tests/e2e/cases/objcpp.yaml b/tests/e2e/cases/objcpp.yaml new file mode 100644 index 0000000..5f4af3c --- /dev/null +++ b/tests/e2e/cases/objcpp.yaml @@ -0,0 +1,8 @@ +language: + id: objcpp + kind: source + project: playground/objcpp + +pairs: + - language: objcpp + server: clangd diff --git a/tests/e2e/cases/python.yaml b/tests/e2e/cases/python.yaml new file mode 100644 index 0000000..729fde3 --- /dev/null +++ b/tests/e2e/cases/python.yaml @@ -0,0 +1,8 @@ +language: + id: python + kind: source + project: playground/python + +pairs: + - language: python + server: pyright diff --git a/tests/e2e/cases/rust.yaml b/tests/e2e/cases/rust.yaml new file mode 100644 index 0000000..95e624c --- /dev/null +++ b/tests/e2e/cases/rust.yaml @@ -0,0 +1,25 @@ +language: + id: rust + kind: source + project: playground/rust + +pairs: + - language: rust + server: rust_analyzer + smoke: + provision: + method: download + query: + kind: list-symbols + expected-names: + - Order + - OrderItem + - sample_order + - format_order + host-programs: + - name: cargo + resolve: [rustup, which, cargo] + - name: rustc + resolve: [rustup, which, rustc] + lsp-timeout-seconds: 30 + deadline-seconds: 180 diff --git a/tests/e2e/cases/suite.yaml b/tests/e2e/cases/suite.yaml new file mode 100644 index 0000000..f4352f5 --- /dev/null +++ b/tests/e2e/cases/suite.yaml @@ -0,0 +1,28 @@ +schema-version: 4 +coverage: partial + +commands: + - { name: commands, strategy: catalog } + - { name: languages, strategy: catalog } + - { name: servers, strategy: catalog } + - { name: completion, strategy: catalog } + - { name: agent-skill, strategy: catalog } + - { name: detect, strategy: filesystem } + - { name: list-files, strategy: filesystem } + - { name: server-capabilities, strategy: lsp-fixture } + - { name: diagnostics, strategy: lsp-fixture } + - { name: format, strategy: lsp-fixture } + - { name: grep, strategy: lsp-fixture } + - { name: list-symbols, strategy: lsp-fixture } + - { name: list-functions, strategy: lsp-fixture } + - { name: references, strategy: lsp-fixture } + - { name: callers, strategy: lsp-fixture } + - { name: callees, strategy: lsp-fixture } + - { name: definition, strategy: lsp-fixture } + - { name: declaration, strategy: lsp-fixture } + - { name: build-index, strategy: lsp-fixture } + - { name: daemon, strategy: lifecycle } + - { name: stop, strategy: lifecycle } + - { name: stop-all, strategy: lifecycle } + - { name: run, strategy: lifecycle } + - { name: update, strategy: update-fixture } diff --git a/tests/e2e/cases/typescript.yaml b/tests/e2e/cases/typescript.yaml new file mode 100644 index 0000000..46fe591 --- /dev/null +++ b/tests/e2e/cases/typescript.yaml @@ -0,0 +1,8 @@ +language: + id: typescript + kind: source + project: playground/typescript + +pairs: + - language: typescript + server: ts_ls diff --git a/tests/e2e/catalog.rs b/tests/e2e/catalog.rs new file mode 100644 index 0000000..d66b32c --- /dev/null +++ b/tests/e2e/catalog.rs @@ -0,0 +1,31 @@ +use crate::fixture::E2eFixture; +use crate::manifest::CommandStrategy; +use std::collections::BTreeSet; + +#[test] +fn catalog_command_paths_are_covered() { + let fixture = E2eFixture::new().expect("E2E fixture should initialize"); + let context = fixture.context(); + + let commands = context.run(&["commands"]); + commands.assert_success(); + let actual = commands.stdout_text().lines().collect::>(); + assert_eq!(actual, fixture.manifest().command_names()); + + for command in fixture.commands_for(CommandStrategy::Catalog) { + let output = match command { + "commands" => continue, + "languages" => context.run(&[command]), + "servers" => context.run(&[command, "--lang", "rust"]), + "completion" => context.run(&[command, "bash"]), + "agent-skill" => context.run(&[command]), + other => panic!("catalog strategy has no scenario for {other:?}"), + }; + output.assert_success(); + assert!( + !output.stdout_text().is_empty(), + "{command} should produce identifying output" + ); + assert!(output.stderr_text().is_empty()); + } +} diff --git a/tests/e2e/filesystem.rs b/tests/e2e/filesystem.rs new file mode 100644 index 0000000..3919fa4 --- /dev/null +++ b/tests/e2e/filesystem.rs @@ -0,0 +1,29 @@ +use serde_json::Value; + +use crate::local_fixture::LocalFixture; +use crate::manifest::CommandStrategy; + +#[test] +fn filesystem_command_paths_are_covered() { + let fixture = LocalFixture::new().expect("local fixture should initialize"); + + for command in fixture.commands_for(CommandStrategy::Filesystem) { + let output = + fixture + .context() + .run(&[command, ".", "--lsp", fixture.server_name(), "--json"]); + output.assert_success(); + let value: Value = output.json(); + match command { + "detect" => assert_eq!( + value.pointer("/servers/0/server").and_then(Value::as_str), + Some(fixture.server_name()) + ), + "list-files" => assert_eq!( + value.pointer("/files/0").and_then(Value::as_str), + Some("./main.fake") + ), + other => panic!("filesystem strategy has no scenario for {other:?}"), + } + } +} diff --git a/tests/e2e/fixture.rs b/tests/e2e/fixture.rs new file mode 100644 index 0000000..d7977f3 --- /dev/null +++ b/tests/e2e/fixture.rs @@ -0,0 +1,41 @@ +use std::path::PathBuf; + +use crate::harness::E2eContext; +use crate::manifest::{CommandStrategy, Manifest}; + +pub(crate) struct E2eFixture { + context: E2eContext, + manifest: Manifest, +} + +impl E2eFixture { + pub(crate) fn new() -> Result { + Self::initialize(None) + } + + pub(crate) fn new_with_data_dir(data_dir: PathBuf) -> Result { + Self::initialize(Some(data_dir)) + } + + pub(crate) fn context(&self) -> &E2eContext { + &self.context + } + + pub(crate) fn manifest(&self) -> &Manifest { + &self.manifest + } + + pub(crate) fn commands_for(&self, strategy: CommandStrategy) -> impl Iterator { + self.manifest.commands_for(strategy) + } + + fn initialize(data_dir: Option) -> Result { + let manifest = Manifest::load_repository()?; + let mut context = + E2eContext::new().map_err(|error| format!("failed to create E2E context: {error}"))?; + if let Some(data_dir) = data_dir { + context = context.with_data_dir(data_dir); + } + Ok(Self { context, manifest }) + } +} diff --git a/tests/e2e/fixtures/data/filetypes/fake.yaml b/tests/e2e/fixtures/data/filetypes/fake.yaml new file mode 100644 index 0000000..95fd7c3 --- /dev/null +++ b/tests/e2e/fixtures/data/filetypes/fake.yaml @@ -0,0 +1,3 @@ +extensions: + - "fake" +patterns: [] diff --git a/tests/e2e/fixtures/data/lsp/fake.yaml b/tests/e2e/fixtures/data/lsp/fake.yaml new file mode 100644 index 0000000..2cd6466 --- /dev/null +++ b/tests/e2e/fixtures/data/lsp/fake.yaml @@ -0,0 +1,7 @@ +filetypes: + - "fake" +root_markers: + - "fixture.root" +name: "e2e-fake-lsp" +cmdline: "e2e-fake-lsp" +wait-for-index: false diff --git a/tests/e2e/fixtures/fake-lsp/.gitignore b/tests/e2e/fixtures/fake-lsp/.gitignore new file mode 100644 index 0000000..b83d222 --- /dev/null +++ b/tests/e2e/fixtures/fake-lsp/.gitignore @@ -0,0 +1 @@ +/target/ diff --git a/tests/e2e/fixtures/fake-lsp/Cargo.lock b/tests/e2e/fixtures/fake-lsp/Cargo.lock new file mode 100644 index 0000000..a828bcc --- /dev/null +++ b/tests/e2e/fixtures/fake-lsp/Cargo.lock @@ -0,0 +1,105 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "lsp-cli-e2e-fake-lsp" +version = "0.0.0" +dependencies = [ + "serde_json", +] + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.150" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "syn" +version = "3.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12df2e0110f65b775f769bb17ef989067a1d931b2eb822bd4346631eeada89f9" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/tests/e2e/fixtures/fake-lsp/Cargo.toml b/tests/e2e/fixtures/fake-lsp/Cargo.toml new file mode 100644 index 0000000..91f335a --- /dev/null +++ b/tests/e2e/fixtures/fake-lsp/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "lsp-cli-e2e-fake-lsp" +version = "0.0.0" +edition = "2024" +rust-version = "1.91" +publish = false + +[dependencies] +serde_json = "=1.0.150" + +[workspace] diff --git a/tests/e2e/fixtures/fake-lsp/src/main.rs b/tests/e2e/fixtures/fake-lsp/src/main.rs new file mode 100644 index 0000000..85e1865 --- /dev/null +++ b/tests/e2e/fixtures/fake-lsp/src/main.rs @@ -0,0 +1,176 @@ +use std::env; +use std::io::{self, BufRead, BufReader, Write}; + +use serde_json::{Value, json}; + +fn main() -> Result<(), Box> { + if env::var_os("LSP_CLI_E2E_RUN_MARKER").is_some() { + println!("fake LSP server replaced lsp-cli"); + return Ok(()); + } + + let mut input = BufReader::new(io::stdin().lock()); + let mut output = io::stdout().lock(); + let mut root_uri = String::new(); + let mut report_status = false; + + while let Some(message) = read_message(&mut input)? { + let method = message.get("method").and_then(Value::as_str).unwrap_or(""); + if method == "exit" { + break; + } + if method == "initialized" { + if report_status { + write_message( + &mut output, + &json!({ + "jsonrpc": "2.0", + "method": "experimental/serverStatus", + "params": {"health": "ok", "quiescent": true} + }), + )?; + } + continue; + } + let Some(id) = message.get("id").cloned() else { + continue; + }; + + let result = match method { + "initialize" => { + root_uri = message["params"]["rootUri"] + .as_str() + .unwrap_or("file:///") + .to_string(); + report_status = + message["params"]["capabilities"]["experimental"]["serverStatusNotification"] + .as_bool() + .unwrap_or(false); + initialize_result() + } + "workspace/symbol" | "textDocument/documentSymbol" => { + json!([symbol(&root_uri)]) + } + "textDocument/diagnostic" => json!({ + "kind": "full", + "items": [{ + "range": range(1, 2, 13), + "severity": 2, + "code": "fixture", + "source": "e2e-fake-lsp", + "message": "synthetic diagnostic" + }] + }), + "textDocument/formatting" => json!([{ + "range": range(1, 0, 13), + "newText": " formatted" + }]), + "textDocument/references" | "textDocument/definition" | "textDocument/declaration" => { + json!([location(&root_uri)]) + } + "textDocument/prepareCallHierarchy" => json!([call_item(&root_uri)]), + "callHierarchy/incomingCalls" => { + json!([{"from": call_item(&root_uri), "fromRanges": []}]) + } + "callHierarchy/outgoingCalls" => { + json!([{"to": call_item(&root_uri), "fromRanges": []}]) + } + "shutdown" => Value::Null, + _ => { + write_message( + &mut output, + &json!({ + "jsonrpc": "2.0", + "id": id, + "error": {"code": -32601, "message": format!("unsupported method {method}")} + }), + )?; + continue; + } + }; + write_message( + &mut output, + &json!({"jsonrpc": "2.0", "id": id, "result": result}), + )?; + } + Ok(()) +} + +fn initialize_result() -> Value { + json!({ + "capabilities": { + "textDocumentSync": 1, + "workspaceSymbolProvider": true, + "documentSymbolProvider": true, + "definitionProvider": true, + "declarationProvider": true, + "referencesProvider": true, + "callHierarchyProvider": true, + "documentFormattingProvider": true, + "diagnosticProvider": { + "identifier": "fixture", + "interFileDependencies": false, + "workspaceDiagnostics": false + } + }, + "serverInfo": {"name": "e2e-fake-lsp", "version": "1"} + }) +} + +fn symbol(root_uri: &str) -> Value { + json!({ + "name": "Target", + "kind": 12, + "location": location(root_uri) + }) +} + +fn location(root_uri: &str) -> Value { + json!({"uri": format!("{root_uri}/main.fake"), "range": range(0, 3, 9)}) +} + +fn call_item(root_uri: &str) -> Value { + json!({ + "name": "Target", + "kind": 12, + "uri": format!("{root_uri}/main.fake"), + "range": range(0, 0, 13), + "selectionRange": range(0, 3, 9) + }) +} + +fn range(line: u32, start: u32, end: u32) -> Value { + json!({ + "start": {"line": line, "character": start}, + "end": {"line": line, "character": end} + }) +} + +fn read_message(reader: &mut impl BufRead) -> io::Result> { + let mut content_length = None; + loop { + let mut header = String::new(); + if reader.read_line(&mut header)? == 0 { + return Ok(None); + } + if header == "\r\n" { + break; + } + if let Some(value) = header.strip_prefix("Content-Length:") { + content_length = Some(value.trim().parse::().map_err(io::Error::other)?); + } + } + let length = content_length.ok_or_else(|| io::Error::other("missing Content-Length"))?; + let mut body = vec![0; length]; + reader.read_exact(&mut body)?; + serde_json::from_slice(&body) + .map(Some) + .map_err(io::Error::other) +} + +fn write_message(writer: &mut impl Write, message: &Value) -> io::Result<()> { + let body = serde_json::to_vec(message).map_err(io::Error::other)?; + write!(writer, "Content-Length: {}\r\n\r\n", body.len())?; + writer.write_all(&body)?; + writer.flush() +} diff --git a/tests/e2e/fixtures/project/fixture.root b/tests/e2e/fixtures/project/fixture.root new file mode 100644 index 0000000..ee8c1ee --- /dev/null +++ b/tests/e2e/fixtures/project/fixture.root @@ -0,0 +1 @@ +fixture diff --git a/tests/e2e/fixtures/project/main.fake b/tests/e2e/fixtures/project/main.fake new file mode 100644 index 0000000..c92e8e0 --- /dev/null +++ b/tests/e2e/fixtures/project/main.fake @@ -0,0 +1,3 @@ +fn Target() { + unformatted +} diff --git a/tests/e2e/harness.rs b/tests/e2e/harness.rs new file mode 100644 index 0000000..537ef35 --- /dev/null +++ b/tests/e2e/harness.rs @@ -0,0 +1,548 @@ +use std::ffi::OsStr; +use std::fs; +use std::io; +use std::path::{Path, PathBuf}; +use std::process::Command; +use std::time::Duration; + +use fs_extra::dir::CopyOptions; +use serde::de::DeserializeOwned; +use tempfile::TempDir; + +use crate::process::{self, ProcessOutput}; + +#[path = "../../src/test_support/temp_root.rs"] +mod temp_root; + +use self::temp_root::{test_temp_base, test_temp_root}; + +const DEFAULT_COMMAND_DEADLINE: Duration = Duration::from_secs(30); +const DAEMON_CLEANUP_DEADLINE: Duration = Duration::from_secs(5); + +pub(crate) struct E2eContext { + _sandbox: TempDir, + _runtime_sandbox: TempDir, + home: PathBuf, + config_home: PathBuf, + runtime_dir: PathBuf, + workspace: PathBuf, + bin_dir: PathBuf, + build_dir: PathBuf, + data_dir: PathBuf, +} + +pub(crate) struct E2eOutput { + process: ProcessOutput, + runtime_dir: PathBuf, +} + +impl E2eContext { + pub(crate) fn new() -> io::Result { + let test_temp_root = test_temp_root()?; + fs::create_dir_all(&test_temp_root)?; + let sandbox = tempfile::Builder::new() + .prefix("lsp-cli-e2e-") + .tempdir_in(test_temp_root)?; + let test_temp_base = test_temp_base()?; + fs::create_dir_all(&test_temp_base)?; + // Keep this prefix and hierarchy short: daemon socket paths have a small OS limit. + let runtime_sandbox = tempfile::Builder::new() + .prefix("e-") + .tempdir_in(test_temp_base)?; + let home = sandbox.path().join("home"); + let config_home = sandbox.path().join("config"); + let workspace = sandbox.path().join("workspace"); + let bin_dir = sandbox.path().join("bin"); + let build_dir = sandbox.path().join("build"); + let runtime_dir = runtime_sandbox.path().to_path_buf(); + + for directory in [&home, &config_home, &workspace, &bin_dir, &build_dir] { + fs::create_dir(directory)?; + } + + Ok(Self { + _sandbox: sandbox, + _runtime_sandbox: runtime_sandbox, + home, + config_home, + runtime_dir, + workspace, + bin_dir, + build_dir, + data_dir: PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("data"), + }) + } + + pub(crate) fn copy_project(&self, source: &Path) -> Result<(), String> { + let options = CopyOptions::new().content_only(true); + fs_extra::dir::copy(source, &self.workspace, &options) + .map(|_copied_bytes| ()) + .map_err(|error| { + format!( + "failed to copy E2E project {} into {}: {error}", + source.display(), + self.workspace.display() + ) + }) + } + + pub(crate) fn with_data_dir(mut self, data_dir: PathBuf) -> Self { + self.data_dir = data_dir; + self + } + + pub(crate) fn stage_program(&self, name: &str, source: &Path) -> Result<(), String> { + let source = source + .canonicalize() + .map_err(|error| format!("failed to resolve {}: {error}", source.display()))?; + self.link_host_program(&source, &self.bin_dir.join(name)) + .map_err(|error| format!("failed to stage {} as {name:?}: {error}", source.display())) + } + + pub(crate) fn stage_host_program( + &self, + name: &str, + resolver: &[String], + deadline: Duration, + ) -> Result<(), String> { + let (program, args) = resolver + .split_first() + .ok_or_else(|| format!("host program {name:?} has no resolver command"))?; + let mut command = Command::new(program); + command.args(args).current_dir(env!("CARGO_MANIFEST_DIR")); + let output = process::run(&mut command, deadline) + .map_err(|failure| failure.diagnostic(&runtime_state(&self.runtime_dir)))?; + if !output.status().success() { + return Err(output.diagnostic( + &format!("failed to resolve required host program {name:?}"), + &runtime_state(&self.runtime_dir), + )); + } + let stdout = std::str::from_utf8(output.stdout()).map_err(|error| { + output.diagnostic( + &format!("resolver output for host program {name:?} is not UTF-8: {error}"), + &runtime_state(&self.runtime_dir), + ) + })?; + let paths = stdout + .lines() + .filter(|line| !line.is_empty()) + .collect::>(); + let [resolved] = paths.as_slice() else { + return Err(output.diagnostic( + &format!( + "resolver for host program {name:?} must print exactly one non-empty path" + ), + &runtime_state(&self.runtime_dir), + )); + }; + let resolved = Path::new(resolved); + if !resolved.is_file() { + return Err(output.diagnostic( + &format!( + "resolver for host program {name:?} returned {}, which is not a file", + resolved.display() + ), + &runtime_state(&self.runtime_dir), + )); + } + let resolved = resolved.canonicalize().map_err(|error| { + output.diagnostic( + &format!( + "failed to resolve host program {name:?} path {}: {error}", + resolved.display() + ), + &runtime_state(&self.runtime_dir), + ) + })?; + + self.link_host_program(&resolved, &self.bin_dir.join(name)) + .map_err(|error| { + format!( + "failed to stage host program {name:?} from {}: {error}", + resolved.display() + ) + }) + } + + #[cfg(unix)] + fn link_host_program(&self, source: &Path, destination: &Path) -> io::Result<()> { + std::os::unix::fs::symlink(source, destination) + } + + #[cfg(not(unix))] + fn link_host_program(&self, source: &Path, destination: &Path) -> io::Result<()> { + fs::copy(source, destination).map(|_copied_bytes| ()) + } + + pub(crate) fn home(&self) -> &Path { + &self.home + } + + pub(crate) fn workspace(&self) -> &Path { + &self.workspace + } + + pub(crate) fn installed_data(&self) -> PathBuf { + self.home.join(".local/share/lsp-cli/data") + } + + pub(crate) fn run(&self, args: &[&str]) -> E2eOutput { + self.run_with_deadline(args, DEFAULT_COMMAND_DEADLINE) + } + + pub(crate) fn run_with_deadline(&self, args: &[&str], deadline: Duration) -> E2eOutput { + self.try_run_with_deadline(args, deadline) + .unwrap_or_else(|diagnostic| panic!("{diagnostic}")) + } + + pub(crate) fn try_run_with_deadline( + &self, + args: &[&str], + deadline: Duration, + ) -> Result { + let mut command = self.command(); + command.args(args); + self.run_command(&mut command, deadline) + } + + pub(crate) fn run_with_env(&self, args: &[&str], environment: &[(&str, &str)]) -> E2eOutput { + let mut command = self.command(); + command.args(args).envs(environment.iter().copied()); + self.run_command(&mut command, DEFAULT_COMMAND_DEADLINE) + .unwrap_or_else(|diagnostic| panic!("{diagnostic}")) + } + + fn command(&self) -> Command { + self.command_for(env!("CARGO_BIN_EXE_lsp-cli")) + } + + fn command_for(&self, program: impl AsRef) -> Command { + let mut command = Command::new(program); + command + .env_clear() + .env("HOME", &self.home) + .env("CARGO_TARGET_DIR", &self.build_dir) + .env("XDG_CONFIG_HOME", &self.config_home) + .env("XDG_RUNTIME_DIR", &self.runtime_dir) + .env("LSP_DATA", &self.data_dir) + .env("PATH", &self.bin_dir) + .env("LANG", "C") + .env("LC_ALL", "C") + .env("TZ", "UTC") + .current_dir(&self.workspace); + command + } + + fn run_command(&self, command: &mut Command, deadline: Duration) -> Result { + process::run(command, deadline) + .map(|process| E2eOutput { + process, + runtime_dir: self.runtime_dir.clone(), + }) + .map_err(|failure| failure.diagnostic(&runtime_state(&self.runtime_dir))) + } + + #[cfg(test)] + fn run_test_program( + &self, + program: impl AsRef, + args: &[&str], + deadline: Duration, + ) -> Result { + let mut command = self.command_for(program); + command.args(args); + self.run_command(&mut command, deadline) + } +} + +impl Drop for E2eContext { + fn drop(&mut self) { + let daemon_root = self.runtime_dir.join("lsp-cli"); + if !daemon_root.exists() { + return; + } + + // Detached daemons outlive command process groups, so the context must stop them explicitly. + let mut command = self.command(); + command.args(["stop-all", "--debug"]); + let cleanup = process::run(&mut command, DAEMON_CLEANUP_DEADLINE); + let diagnostic = match cleanup { + Ok(output) if output.status().success() => return, + Ok(output) => output.diagnostic( + "E2E daemon cleanup exited unsuccessfully", + &runtime_state(&self.runtime_dir), + ), + Err(failure) => failure.diagnostic(&runtime_state(&self.runtime_dir)), + }; + if std::thread::panicking() { + eprintln!("E2E daemon cleanup failed:\n{diagnostic}"); + } else { + panic!("E2E daemon cleanup failed:\n{diagnostic}"); + } + } +} + +impl E2eOutput { + pub(crate) fn assert_success(&self) { + self.ensure_success() + .unwrap_or_else(|diagnostic| panic!("{diagnostic}")); + } + + pub(crate) fn ensure_success(&self) -> Result<(), String> { + if self.process.status().success() { + Ok(()) + } else { + Err(self.diagnostic("lsp-cli exited unsuccessfully")) + } + } + + pub(crate) fn stdout_text(&self) -> &str { + std::str::from_utf8(self.process.stdout()).unwrap_or_else(|error| { + panic!( + "{}", + self.diagnostic(&format!("stdout is not valid UTF-8: {error}")) + ) + }) + } + + pub(crate) fn stderr_text(&self) -> &str { + std::str::from_utf8(self.process.stderr()).unwrap_or_else(|error| { + panic!( + "{}", + self.diagnostic(&format!("stderr is not valid UTF-8: {error}")) + ) + }) + } + + pub(crate) fn json(&self) -> T { + self.try_json() + .unwrap_or_else(|diagnostic| panic!("{diagnostic}")) + } + + fn diagnostic(&self, reason: &str) -> String { + self.process + .diagnostic(reason, &runtime_state(&self.runtime_dir)) + } + + pub(crate) fn try_json(&self) -> Result { + serde_json::from_slice(self.process.stdout()) + .map_err(|error| self.diagnostic(&format!("stdout is not valid JSON: {error}"))) + } +} + +fn runtime_state(runtime_dir: &std::path::Path) -> String { + let daemon_root = runtime_dir.join("lsp-cli"); + let entries = match fs::read_dir(&daemon_root) { + Ok(entries) => entries, + Err(error) if error.kind() == io::ErrorKind::NotFound => { + return format!("{} does not exist", daemon_root.display()); + } + Err(error) => return format!("failed to read {}: {error}", daemon_root.display()), + }; + let mut paths = entries + .map(|entry| match entry { + Ok(entry) => entry.path().display().to_string(), + Err(error) => format!(""), + }) + .collect::>(); + paths.sort(); + if paths.is_empty() { + format!("{} is empty", daemon_root.display()) + } else { + paths.join("\n") + } +} + +#[cfg(test)] +mod tests { + use std::collections::BTreeMap; + use std::ffi::OsString; + #[cfg(target_os = "linux")] + use std::path::Path; + #[cfg(target_os = "linux")] + use std::time::Instant; + + use super::*; + + fn context() -> E2eContext { + E2eContext::new().expect("E2E context should initialize") + } + + fn run_shell( + context: &E2eContext, + script: &str, + deadline: Duration, + ) -> Result { + context.run_test_program("/bin/sh", &["-c", script], deadline) + } + + #[cfg(target_os = "linux")] + fn assert_recorded_process_is_gone(pid_file: &Path) { + let pid = fs::read_to_string(pid_file).expect("descendant PID should be recorded"); + let process = Path::new("/proc").join(&pid); + let reaping_deadline = Instant::now() + Duration::from_secs(1); + while process.exists() && Instant::now() < reaping_deadline { + std::thread::sleep(Duration::from_millis(10)); + } + assert!( + !process.exists(), + "descendant process {pid} survived group cleanup" + ); + } + + #[test] + fn command_isolated_from_ambient_process_state() { + let context = context(); + let command = context.command(); + let actual = command + .get_envs() + .map(|(name, value)| (name.to_os_string(), value.map(OsString::from))) + .collect::>(); + let expected = [ + ( + "CARGO_TARGET_DIR", + context.build_dir.as_os_str().to_os_string(), + ), + ("HOME", context.home.as_os_str().to_os_string()), + ("LANG", OsString::from("C")), + ("LC_ALL", OsString::from("C")), + ("LSP_DATA", context.data_dir.as_os_str().to_os_string()), + ("PATH", context.bin_dir.as_os_str().to_os_string()), + ("TZ", OsString::from("UTC")), + ( + "XDG_CONFIG_HOME", + context.config_home.as_os_str().to_os_string(), + ), + ( + "XDG_RUNTIME_DIR", + context.runtime_dir.as_os_str().to_os_string(), + ), + ] + .into_iter() + .map(|(name, value)| (OsString::from(name), Some(value))) + .collect::>(); + + assert_eq!(actual, expected); + assert_eq!(command.get_current_dir(), Some(context.workspace.as_path())); + assert!(!context.runtime_dir.starts_with(context._sandbox.path())); + } + + #[cfg(unix)] + #[test] + fn runtime_directory_has_room_for_daemon_socket_name() { + use std::os::unix::net::UnixListener; + + let context = context(); + let daemon_root = context.runtime_dir.join("lsp-cli"); + fs::create_dir(&daemon_root).expect("daemon root should be created"); + let socket_path = daemon_root.join(format!("{}-{}.sock", "s".repeat(32), "f".repeat(24))); + let _listener = UnixListener::bind(&socket_path).unwrap_or_else(|error| { + panic!( + "E2E runtime path {} cannot hold a daemon socket: {error}", + socket_path.display() + ) + }); + } + + #[test] + fn captures_large_stdout_and_stderr_without_deadlock() { + let context = context(); + let output = run_shell( + &context, + "i=0; while [ \"$i\" -lt 100000 ]; do printf o; printf e >&2; i=$((i + 1)); done", + Duration::from_secs(5), + ) + .expect("output fixture should finish"); + + assert_eq!(output.process.stdout().len(), 100_000); + assert_eq!(output.process.stderr().len(), 100_000); + } + + #[test] + fn deadline_kills_the_command_process_group() { + let context = context(); + let pid_file = context.workspace.join("descendant.pid"); + let script = format!( + "/bin/sleep 30 & child=$!; printf '%s' \"$child\" > {}; wait", + pid_file.display() + ); + let diagnostic = run_shell(&context, &script, Duration::from_millis(100)) + .err() + .expect("stalled fixture should exceed its deadline"); + + assert!(diagnostic.contains("process exceeded its deadline")); + assert!(diagnostic.contains("process group killed and reaped")); + #[cfg(target_os = "linux")] + assert_recorded_process_is_gone(&pid_file); + } + + #[test] + fn deadline_includes_output_pipes_held_by_descendants() { + let context = context(); + let pid_file = context.workspace.join("pipe-holder.pid"); + let script = format!( + "/bin/sleep 30 & child=$!; printf '%s' \"$child\" > {}", + pid_file.display() + ); + let diagnostic = run_shell(&context, &script, Duration::from_millis(100)) + .err() + .expect("inherited pipe should keep the process group beyond its deadline"); + + assert!(diagnostic.contains("remained open after the command deadline")); + assert!(diagnostic.contains("process group killed and reaped")); + #[cfg(target_os = "linux")] + assert_recorded_process_is_gone(&pid_file); + } + + #[test] + fn parses_json_output_into_requested_type() { + let context = context(); + let output = run_shell( + &context, + "printf '%s' '{\"answer\":42}'", + Duration::from_secs(1), + ) + .expect("JSON fixture should finish"); + let value: serde_json::Value = output.json(); + + assert_eq!(value, serde_json::json!({"answer": 42})); + } + + #[test] + fn invalid_json_reports_command_and_captured_output() { + let context = context(); + let output = run_shell(&context, "printf not-json", Duration::from_secs(1)) + .expect("invalid JSON fixture should finish"); + let diagnostic = output + .try_json::() + .expect_err("invalid JSON should be rejected"); + + assert!(diagnostic.contains("stdout is not valid JSON")); + assert!(diagnostic.contains("command: \"/bin/sh\" \"-c\"")); + assert!(diagnostic.contains("not-json")); + } + + #[test] + fn failed_command_diagnostic_includes_execution_context_and_output() { + let context = context(); + let output = run_shell( + &context, + "printf stdout-marker; printf stderr-marker >&2; exit 7", + Duration::from_secs(1), + ) + .expect("failure fixture should finish"); + let diagnostic = output.diagnostic("fixture failed"); + + assert!(diagnostic.contains("fixture failed")); + assert!(diagnostic.contains("command: \"/bin/sh\" \"-c\"")); + assert!(diagnostic.contains(&format!( + "working directory: {}", + context.workspace.display() + ))); + assert!(diagnostic.contains("status: exit status: 7")); + assert!(diagnostic.contains("stdout-marker")); + assert!(diagnostic.contains("stderr-marker")); + assert!(diagnostic.contains("runtime state:")); + } +} diff --git a/tests/e2e/lifecycle.rs b/tests/e2e/lifecycle.rs new file mode 100644 index 0000000..bba0378 --- /dev/null +++ b/tests/e2e/lifecycle.rs @@ -0,0 +1,54 @@ +use crate::local_fixture::LocalFixture; +use crate::manifest::CommandStrategy; +use std::collections::BTreeSet; + +#[test] +fn lifecycle_command_paths_are_covered() { + let fixture = LocalFixture::new().expect("local fixture should initialize"); + assert_eq!( + fixture + .commands_for(CommandStrategy::Lifecycle) + .collect::>(), + BTreeSet::from(["daemon", "run", "stop", "stop-all"]) + ); + let context = fixture.context(); + let server = fixture.server_name(); + + let daemon = context.run(&[ + "daemon", + ".", + "--lsp", + server, + "--no-download", + "--idle-timeout", + "10", + ]); + daemon.assert_success(); + assert!(daemon.stdout_text().trim_end().ends_with(".sock")); + + let stop = context.run(&["stop", ".", "--lsp", server]); + stop.assert_success(); + assert!(stop.stdout_text().contains("stopped")); + + context + .run(&[ + "daemon", + ".", + "--lsp", + server, + "--no-download", + "--idle-timeout", + "10", + ]) + .assert_success(); + let stop_all = context.run(&["stop-all"]); + stop_all.assert_success(); + assert!(stop_all.stdout_text().contains("stopped")); + + let run = context.run_with_env( + &["run", ".", "--lsp", server, "--no-download"], + &[("LSP_CLI_E2E_RUN_MARKER", "1")], + ); + run.assert_success(); + assert_eq!(run.stdout_text(), "fake LSP server replaced lsp-cli\n"); +} diff --git a/tests/e2e/local_fixture.rs b/tests/e2e/local_fixture.rs new file mode 100644 index 0000000..f2c25ac --- /dev/null +++ b/tests/e2e/local_fixture.rs @@ -0,0 +1,89 @@ +use std::path::{Path, PathBuf}; +use std::process::Command; +use std::sync::OnceLock; +use std::time::Duration; + +use crate::fixture::E2eFixture; +use crate::harness::E2eContext; +use crate::manifest::CommandStrategy; +use crate::process; +use crate::repository_root; + +const SERVER_NAME: &str = "e2e-fake-lsp"; +const BINARY_NAME: &str = "lsp-cli-e2e-fake-lsp"; +const BUILD_DEADLINE: Duration = Duration::from_secs(120); + +pub(crate) struct LocalFixture { + fixture: E2eFixture, +} + +impl LocalFixture { + pub(crate) fn new() -> Result { + let root = repository_root(); + let fixtures = root.join("tests/e2e/fixtures"); + let fixture = E2eFixture::new_with_data_dir(fixtures.join("data"))?; + fixture.context().copy_project(&fixtures.join("project"))?; + fixture + .context() + .stage_program(SERVER_NAME, fake_server_binary()?)?; + Ok(Self { fixture }) + } + + pub(crate) fn context(&self) -> &E2eContext { + self.fixture.context() + } + + pub(crate) fn commands_for(&self, strategy: CommandStrategy) -> impl Iterator { + self.fixture.commands_for(strategy) + } + + pub(crate) fn server_name(&self) -> &'static str { + SERVER_NAME + } +} + +fn fake_server_binary() -> Result<&'static Path, String> { + static BINARY: OnceLock> = OnceLock::new(); + match BINARY.get_or_init(build_fake_server) { + Ok(path) => Ok(path), + Err(error) => Err(error.clone()), + } +} + +fn build_fake_server() -> Result { + let root = repository_root(); + let manifest = root.join("tests/e2e/fixtures/fake-lsp/Cargo.toml"); + let target = root.join("target/e2e-fixtures"); + let cargo = option_env!("CARGO").unwrap_or("cargo"); + let mut command = Command::new(cargo); + command.args([ + "build", + "--quiet", + "--locked", + "--manifest-path", + manifest + .to_str() + .ok_or_else(|| format!("fixture manifest path is not UTF-8: {}", manifest.display()))?, + "--target-dir", + target + .to_str() + .ok_or_else(|| format!("fixture target path is not UTF-8: {}", target.display()))?, + ]); + let output = process::run(&mut command, BUILD_DEADLINE) + .map_err(|failure| failure.diagnostic("fake LSP build has no daemon runtime"))?; + if !output.status().success() { + return Err(output.diagnostic( + "failed to build the fake LSP fixture", + "fake LSP build has no daemon runtime", + )); + } + let binary = target.join("debug").join(BINARY_NAME); + if binary.is_file() { + Ok(binary) + } else { + Err(format!( + "fake LSP build did not create {}", + binary.display() + )) + } +} diff --git a/tests/e2e/manifest.rs b/tests/e2e/manifest.rs new file mode 100644 index 0000000..81dc928 --- /dev/null +++ b/tests/e2e/manifest.rs @@ -0,0 +1,554 @@ +use std::collections::BTreeSet; +use std::path::{Component, Path, PathBuf}; + +use serde::Deserialize; + +use crate::case_files::{file_stem, read_yaml, yaml_paths}; +use crate::manifest_data::{ + FiletypeConfig, LspConfig, PairKey, compatible_pairs, detectable_languages, preferred_pairs, +}; +use crate::repository_root; + +const MANIFEST_SCHEMA_VERSION: u32 = 4; + +#[derive(Clone, Debug)] +pub(crate) struct Manifest { + schema_version: u32, + coverage: Coverage, + commands: Vec, + languages: Vec, + pairs: Vec, +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "kebab-case", deny_unknown_fields)] +struct SuiteFile { + schema_version: u32, + coverage: Coverage, + commands: Vec, +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "kebab-case", deny_unknown_fields)] +struct LanguageFile { + language: LanguageCase, + #[serde(default)] + pairs: Vec, +} + +#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq)] +#[serde(rename_all = "kebab-case")] +enum Coverage { + Partial, + Complete, +} + +#[derive(Clone, Debug, Deserialize)] +#[serde(rename_all = "kebab-case", deny_unknown_fields)] +struct CommandCase { + name: String, + strategy: CommandStrategy, +} + +#[derive(Clone, Copy, Debug, Deserialize, Eq, Ord, PartialEq, PartialOrd)] +#[serde(rename_all = "kebab-case")] +pub(crate) enum CommandStrategy { + Catalog, + Filesystem, + LspFixture, + Lifecycle, + UpdateFixture, +} + +#[derive(Clone, Debug, Deserialize)] +#[serde(rename_all = "kebab-case", deny_unknown_fields)] +struct LanguageCase { + id: String, + kind: ProjectKind, + project: PathBuf, +} + +#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq)] +#[serde(rename_all = "kebab-case")] +enum ProjectKind { + Source, + Metadata, +} + +impl ProjectKind { + fn label(self) -> &'static str { + match self { + Self::Source => "source", + Self::Metadata => "metadata", + } + } +} + +#[derive(Clone, Debug, Deserialize)] +#[serde(rename_all = "kebab-case", deny_unknown_fields)] +struct PairCase { + language: String, + server: String, + smoke: Option, +} + +#[derive(Clone, Debug, Deserialize)] +#[serde(rename_all = "kebab-case", deny_unknown_fields)] +struct SmokeCase { + provision: Provision, + query: Query, + expected_names: Vec, + #[serde(default)] + host_programs: Vec, + lsp_timeout_seconds: u64, + deadline_seconds: u64, +} + +#[derive(Clone, Copy, Debug, Deserialize)] +#[serde(rename_all = "kebab-case", deny_unknown_fields)] +struct Provision { + method: ProvisionMethod, +} + +#[derive(Clone, Copy, Debug, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub(crate) enum ProvisionMethod { + Download, +} + +#[derive(Clone, Copy, Debug, Deserialize)] +#[serde(rename_all = "kebab-case", deny_unknown_fields)] +struct Query { + kind: QueryKind, +} + +#[derive(Clone, Copy, Debug, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub(crate) enum QueryKind { + ListSymbols, +} + +#[derive(Clone, Debug, Deserialize)] +#[serde(rename_all = "kebab-case", deny_unknown_fields)] +struct HostProgram { + name: String, + resolve: Vec, +} + +pub(crate) struct RealServerCase<'a> { + language: &'a LanguageCase, + pair: &'a PairCase, + smoke: &'a SmokeCase, +} + +impl Manifest { + fn load() -> Result { + Self::load_cases(repository_root()) + } + + fn load_cases(repository: &Path) -> Result { + let directory = repository.join("tests/e2e/cases"); + let suite_path = directory.join("suite.yaml"); + let suite: SuiteFile = read_yaml(&suite_path)?; + let mut languages = Vec::new(); + let mut pairs = Vec::new(); + + for path in yaml_paths(&directory)? { + if path == suite_path { + continue; + } + let case_id = file_stem(&path)?; + let case: LanguageFile = read_yaml(&path)?; + let (language, mut language_pairs) = case.into_parts(&case_id, &path)?; + languages.push(language); + pairs.append(&mut language_pairs); + } + + Ok(Self { + schema_version: suite.schema_version, + coverage: suite.coverage, + commands: suite.commands, + languages, + pairs, + }) + } + + fn validate(&self, repository: &Path) -> Result<(), String> { + if self.schema_version != MANIFEST_SCHEMA_VERSION { + return Err(format!( + "unsupported E2E manifest schema version {}; expected {MANIFEST_SCHEMA_VERSION}", + self.schema_version + )); + } + if self.languages.is_empty() { + return Err("E2E manifest must declare at least one language".to_string()); + } + self.validate_commands()?; + if self.pairs.is_empty() { + return Err("E2E manifest must declare at least one language/server pair".to_string()); + } + + let data = repository.join("data"); + let declared_languages = self.validate_languages(repository, &data)?; + let declared_pairs = self.validate_pairs(&data, &declared_languages)?; + self.validate_preferred_servers(&data, &declared_pairs)?; + + if self.coverage == Coverage::Complete { + Self::validate_complete_coverage(&data, &declared_languages, &declared_pairs)?; + } + Ok(()) + } + + pub(crate) fn load_validated(repository: &Path) -> Result { + let manifest = Self::load_cases(repository)?; + manifest.validate(repository)?; + Ok(manifest) + } + + pub(crate) fn load_repository() -> Result { + Self::load_validated(repository_root()) + } + + pub(crate) fn real_server_smoke_cases(&self) -> impl Iterator> { + self.pairs.iter().filter_map(|pair| { + let smoke = pair.smoke.as_ref()?; + let language = self + .languages + .iter() + .find(|language| language.id == pair.language)?; + Some(RealServerCase { + language, + pair, + smoke, + }) + }) + } + + pub(crate) fn command_names(&self) -> BTreeSet<&str> { + self.commands + .iter() + .map(|command| command.name.as_str()) + .collect() + } + + pub(crate) fn commands_for(&self, strategy: CommandStrategy) -> impl Iterator { + self.commands + .iter() + .filter(move |command| command.strategy == strategy) + .map(|command| command.name.as_str()) + } + + fn validate_commands(&self) -> Result<(), String> { + if self.commands.is_empty() { + return Err("E2E manifest must assign command coverage".to_string()); + } + let mut names = BTreeSet::new(); + for command in &self.commands { + validate_config_id("command", &command.name)?; + if !names.insert(&command.name) { + return Err(format!( + "E2E manifest assigns command {:?} more than once", + command.name + )); + } + } + Ok(()) + } + + fn validate_languages( + &self, + repository: &Path, + data: &Path, + ) -> Result, String> { + let mut declared = BTreeSet::new(); + for language in &self.languages { + validate_config_id("language", &language.id)?; + if !declared.insert(language.id.clone()) { + return Err(format!( + "E2E manifest declares language {:?} more than once", + language.id + )); + } + + let filetype_path = data.join("filetypes").join(format!("{}.yaml", language.id)); + let filetype: FiletypeConfig = read_yaml(&filetype_path)?; + if !filetype.is_detectable() { + return Err(format!( + "E2E language {:?} has no extension or filename-pattern detection rule", + language.id + )); + } + language.validate_project(repository)?; + } + Ok(declared) + } + + fn validate_pairs( + &self, + data: &Path, + declared_languages: &BTreeSet, + ) -> Result, String> { + let mut declared = BTreeSet::new(); + for pair in &self.pairs { + validate_config_id("server", &pair.server)?; + if !declared_languages.contains(&pair.language) { + return Err(format!( + "E2E pair {}/{} references an undeclared language", + pair.language, pair.server + )); + } + if !declared.insert(pair.key()) { + return Err(format!( + "E2E manifest declares pair {}/{} more than once", + pair.language, pair.server + )); + } + + let lsp_path = data.join("lsp").join(format!("{}.yaml", pair.server)); + let lsp: LspConfig = read_yaml(&lsp_path)?; + if !lsp.filetypes.contains(&pair.language) { + return Err(format!( + "LSP config {:?} does not support language {:?}", + pair.server, pair.language + )); + } + if let Some(smoke) = &pair.smoke { + smoke.validate(pair)?; + } + } + Ok(declared) + } + + fn validate_preferred_servers( + &self, + data: &Path, + declared_pairs: &BTreeSet, + ) -> Result<(), String> { + let source_languages = self + .languages + .iter() + .filter(|language| language.kind == ProjectKind::Source) + .map(|language| language.id.clone()) + .collect::>(); + for pair in preferred_pairs(data, &source_languages)? { + if !declared_pairs.contains(&pair) { + return Err(format!( + "E2E source language {:?} is missing its data-preferred server pair {:?}", + pair.language, pair.server + )); + } + } + Ok(()) + } + + fn validate_complete_coverage( + data: &Path, + declared_languages: &BTreeSet, + declared_pairs: &BTreeSet, + ) -> Result<(), String> { + let detectable = detectable_languages(data)?; + let missing_languages = detectable + .difference(declared_languages) + .cloned() + .collect::>(); + if !missing_languages.is_empty() { + return Err(format!( + "complete E2E manifest is missing languages: {}", + missing_languages.join(", ") + )); + } + + let compatible = compatible_pairs(data, &detectable)?; + let missing_pairs = compatible + .difference(declared_pairs) + .map(|pair| format!("{}/{}", pair.language, pair.server)) + .collect::>(); + if !missing_pairs.is_empty() { + return Err(format!( + "complete E2E manifest is missing pairs: {}", + missing_pairs.join(", ") + )); + } + Ok(()) + } +} + +impl LanguageFile { + fn into_parts( + self, + case_id: &str, + path: &Path, + ) -> Result<(LanguageCase, Vec), String> { + if self.language.id != case_id { + return Err(format!( + "E2E case filename {case_id:?} does not match language ID {:?} in {}", + self.language.id, + path.display() + )); + } + if let Some(pair) = self + .pairs + .iter() + .find(|pair| pair.language != self.language.id) + { + return Err(format!( + "E2E case {} for language {:?} contains pair for language {:?}", + path.display(), + self.language.id, + pair.language + )); + } + Ok((self.language, self.pairs)) + } +} + +impl PairCase { + fn key(&self) -> PairKey { + PairKey { + language: self.language.clone(), + server: self.server.clone(), + } + } +} + +impl SmokeCase { + fn validate(&self, pair: &PairCase) -> Result<(), String> { + let label = format!("{}/{}", pair.language, pair.server); + if self.expected_names.is_empty() || self.expected_names.iter().any(String::is_empty) { + return Err(format!( + "E2E smoke case {label} must declare non-empty expected names" + )); + } + if self.lsp_timeout_seconds == 0 || self.deadline_seconds == 0 { + return Err(format!("E2E smoke case {label} deadlines must be positive")); + } + if self.deadline_seconds < self.lsp_timeout_seconds { + return Err(format!( + "E2E smoke case {label} deadline must not be shorter than its LSP timeout" + )); + } + + let mut names = BTreeSet::new(); + for program in &self.host_programs { + validate_config_id("host program", &program.name)?; + if !names.insert(&program.name) { + return Err(format!( + "E2E smoke case {label} declares host program {:?} more than once", + program.name + )); + } + if program.resolve.is_empty() || program.resolve.iter().any(String::is_empty) { + return Err(format!( + "E2E host program {:?} for {label} must have a non-empty resolver command", + program.name + )); + } + } + Ok(()) + } +} + +impl RealServerCase<'_> { + pub(crate) fn label(&self) -> String { + format!("{}/{}", self.pair.language, self.pair.server) + } + + pub(crate) fn language(&self) -> &str { + &self.language.id + } + + pub(crate) fn server_name(&self, repository: &Path) -> Result { + let path = repository + .join("data/lsp") + .join(format!("{}.yaml", self.pair.server)); + let config: LspConfig = read_yaml(&path)?; + Ok(config.name) + } + + pub(crate) fn project(&self) -> &Path { + &self.language.project + } + + pub(crate) fn provision_method(&self) -> ProvisionMethod { + self.smoke.provision.method + } + + pub(crate) fn query_kind(&self) -> QueryKind { + self.smoke.query.kind + } + + pub(crate) fn expected_names(&self) -> &[String] { + &self.smoke.expected_names + } + + pub(crate) fn host_programs(&self) -> impl Iterator { + self.smoke + .host_programs + .iter() + .map(|program| (program.name.as_str(), program.resolve.as_slice())) + } + + pub(crate) fn lsp_timeout_seconds(&self) -> u64 { + self.smoke.lsp_timeout_seconds + } + + pub(crate) fn deadline_seconds(&self) -> u64 { + self.smoke.deadline_seconds + } +} + +impl LanguageCase { + fn validate_project(&self, repository: &Path) -> Result<(), String> { + if self.project.is_absolute() + || self + .project + .components() + .any(|component| !matches!(component, Component::Normal(_))) + { + return Err(format!( + "E2E project path {} must be a normalized relative path", + self.project.display() + )); + } + + let project = repository.join(&self.project); + if !project.is_dir() { + return Err(format!( + "E2E {} project {} for language {:?} is not a directory", + self.kind.label(), + self.project.display(), + self.id + )); + } + let canonical_repository = repository + .canonicalize() + .map_err(|error| format!("failed to resolve {}: {error}", repository.display()))?; + let canonical_project = project + .canonicalize() + .map_err(|error| format!("failed to resolve {}: {error}", project.display()))?; + if !canonical_project.starts_with(canonical_repository) { + return Err(format!( + "E2E project {} resolves outside the repository", + self.project.display() + )); + } + Ok(()) + } +} + +fn validate_config_id(kind: &str, value: &str) -> Result<(), String> { + let mut components = Path::new(value).components(); + if value.is_empty() + || !matches!(components.next(), Some(Component::Normal(_))) + || components.next().is_some() + { + return Err(format!( + "E2E {kind} config ID {value:?} must be one normalized path component" + )); + } + Ok(()) +} + +#[cfg(test)] +#[path = "manifest_tests.rs"] +mod tests; diff --git a/tests/e2e/manifest_data.rs b/tests/e2e/manifest_data.rs new file mode 100644 index 0000000..6cf6b05 --- /dev/null +++ b/tests/e2e/manifest_data.rs @@ -0,0 +1,112 @@ +use std::collections::{BTreeMap, BTreeSet}; +use std::path::Path; + +use serde::Deserialize; + +use crate::case_files::{file_stem, read_yaml, yaml_paths}; + +#[derive(Clone, Debug, Eq, Ord, PartialEq, PartialOrd)] +pub(crate) struct PairKey { + pub(crate) language: String, + pub(crate) server: String, +} + +#[derive(Deserialize)] +pub(crate) struct FiletypeConfig { + #[serde(default)] + extensions: Vec, + #[serde(default)] + patterns: Vec, +} + +impl FiletypeConfig { + pub(crate) fn is_detectable(&self) -> bool { + !self.extensions.is_empty() || !self.patterns.is_empty() + } +} + +#[derive(Deserialize)] +pub(crate) struct LspConfig { + #[serde(default)] + pub(crate) filetypes: Vec, + pub(crate) name: String, +} + +#[derive(Deserialize)] +struct CliConfig { + #[serde(default)] + lsp: BTreeMap>, +} + +pub(crate) fn detectable_languages(data: &Path) -> Result, String> { + let mut languages = BTreeSet::new(); + for path in yaml_paths(&data.join("filetypes"))? { + let config: FiletypeConfig = read_yaml(&path)?; + if config.is_detectable() { + languages.insert(file_stem(&path)?); + } + } + Ok(languages) +} + +pub(crate) fn compatible_pairs( + data: &Path, + detectable: &BTreeSet, +) -> Result, String> { + let mut pairs = BTreeSet::new(); + for path in yaml_paths(&data.join("lsp"))? { + let config: LspConfig = read_yaml(&path)?; + let server = file_stem(&path)?; + for language in config.filetypes { + if detectable.contains(&language) { + pairs.insert(PairKey { + language, + server: server.clone(), + }); + } + } + } + Ok(pairs) +} + +pub(crate) fn preferred_pairs( + data: &Path, + languages: &BTreeSet, +) -> Result, String> { + let config: CliConfig = read_yaml(&data.join("lsp-cli.yaml"))?; + let mut lsps = Vec::new(); + for path in yaml_paths(&data.join("lsp"))? { + lsps.push((file_stem(&path)?, read_yaml::(&path)?)); + } + + languages + .iter() + .map(|language| { + let preferred_name = config + .lsp + .get(language) + .and_then(|servers| servers.first()) + .ok_or_else(|| { + format!("E2E source language {language:?} has no preferred server in data") + })?; + let matching = lsps + .iter() + .filter(|(_id, lsp)| { + lsp.name == *preferred_name && lsp.filetypes.contains(language) + }) + .collect::>(); + match matching.as_slice() { + [(server, _lsp)] => Ok(PairKey { + language: language.clone(), + server: server.clone(), + }), + [] => Err(format!( + "preferred server {preferred_name:?} for E2E source language {language:?} has no compatible LSP config" + )), + _ => Err(format!( + "preferred server {preferred_name:?} for E2E source language {language:?} is ambiguous" + )), + } + }) + .collect() +} diff --git a/tests/e2e/manifest_tests.rs b/tests/e2e/manifest_tests.rs new file mode 100644 index 0000000..2edf996 --- /dev/null +++ b/tests/e2e/manifest_tests.rs @@ -0,0 +1,206 @@ +use super::*; +use crate::repository_root; + +fn first_smoke(manifest: &mut Manifest) -> &mut SmokeCase { + manifest + .pairs + .iter_mut() + .find(|pair| pair.smoke.is_some()) + .expect("manifest should contain a smoke pair") + .smoke + .as_mut() + .expect("selected pair should have a smoke case") +} + +#[test] +fn partial_manifest_matches_pinned_data() { + Manifest::load() + .expect("E2E manifest should parse") + .validate(repository_root()) + .expect("E2E manifest should be valid"); +} + +#[test] +fn source_languages_select_the_approved_preferred_servers() { + let manifest = Manifest::load().expect("E2E manifest should parse"); + let expected = [ + ("c", "clangd"), + ("cpp", "clangd"), + ("cs", "roslyn_ls"), + ("cuda", "clangd"), + ("go", "gopls"), + ("java", "jdtls"), + ("javascript", "ts_ls"), + ("kotlin", "kotlin_lsp"), + ("lua", "lua_ls"), + ("objc", "clangd"), + ("objcpp", "clangd"), + ("python", "pyright"), + ("rust", "rust_analyzer"), + ("typescript", "ts_ls"), + ]; + + let languages = expected + .iter() + .map(|(language, _server)| (*language).to_string()) + .collect(); + let preferred = preferred_pairs(&repository_root().join("data"), &languages) + .expect("data preferences should resolve"); + let actual = preferred + .iter() + .map(|pair| (pair.language.as_str(), pair.server.as_str())) + .collect::>(); + assert_eq!(actual, expected.as_slice()); + + manifest + .validate(repository_root()) + .expect("preferred data pairs should be declared"); +} + +#[test] +fn complete_mode_rejects_the_partial_matrix() { + let mut manifest = Manifest::load().expect("E2E manifest should parse"); + manifest.coverage = Coverage::Complete; + + let error = manifest + .validate(repository_root()) + .expect_err("partial matrix should not satisfy complete coverage"); + + assert!(error.contains("complete E2E manifest is missing pairs")); +} + +#[test] +fn complete_mode_rejects_missing_server_pairs() { + let data = repository_root().join("data"); + let detectable = detectable_languages(&data).expect("filetype configs should load"); + let declared_pairs = BTreeSet::from([PairKey { + language: "rust".to_string(), + server: "rust_analyzer".to_string(), + }]); + + let error = Manifest::validate_complete_coverage(&data, &detectable, &declared_pairs) + .expect_err("partial server matrix should not satisfy complete coverage"); + + assert!(error.contains("complete E2E manifest is missing pairs")); +} + +#[test] +fn manifest_rejects_unknown_fields() { + let error = serde_yaml::from_str::( + "schema-version: 4\ncoverage: partial\ncommands: []\nunknown: true\n", + ) + .expect_err("unknown manifest fields should fail"); + + assert!(error.to_string().contains("unknown field `unknown`")); +} + +#[test] +fn manifest_rejects_a_source_language_without_a_preferred_server() { + let mut manifest = Manifest::load().expect("E2E manifest should parse"); + manifest + .pairs + .retain(|pair| !(pair.language == "c" && pair.server == "clangd")); + + let error = manifest + .validate(repository_root()) + .expect_err("source language should require one preferred server"); + + assert!(error.contains("missing its data-preferred server pair")); +} + +#[test] +fn language_case_filename_must_match_its_id() { + let case = serde_yaml::from_str::( + "language:\n id: gomod\n kind: metadata\n project: playground/gomod\n", + ) + .expect("language case should parse"); + + let error = case + .into_parts("wrong", Path::new("cases/wrong.yaml")) + .expect_err("case filename should match language ID"); + + assert!(error.contains("does not match language ID")); +} + +#[test] +fn language_case_must_own_its_pairs() { + let case = serde_yaml::from_str::( + "language:\n id: gomod\n kind: metadata\n project: playground/gomod\npairs:\n - language: gowork\n server: gopls\n", + ) + .expect("language case should parse"); + + let error = case + .into_parts("gomod", Path::new("cases/gomod.yaml")) + .expect_err("case should contain only its own language pairs"); + + assert!(error.contains("contains pair for language")); +} + +#[test] +fn manifest_rejects_duplicate_command_coverage() { + let mut manifest = Manifest::load().expect("E2E manifest should parse"); + let duplicate = manifest + .commands + .first() + .expect("manifest should contain command coverage") + .clone(); + manifest.commands.push(duplicate); + + let error = manifest + .validate(repository_root()) + .expect_err("duplicate command coverage should fail"); + + assert!(error.contains("more than once")); +} + +#[test] +fn manifest_rejects_config_path_traversal() { + let mut manifest = Manifest::load().expect("E2E manifest should parse"); + manifest + .languages + .first_mut() + .expect("manifest should contain a language") + .id = "../rust".to_string(); + manifest + .pairs + .first_mut() + .expect("manifest should contain a pair") + .language = "../rust".to_string(); + + let error = manifest + .validate(repository_root()) + .expect_err("config path traversal should fail"); + + assert!(error.contains("must be one normalized path component")); +} + +#[test] +fn manifest_rejects_invalid_smoke_deadlines() { + let mut manifest = Manifest::load().expect("E2E manifest should parse"); + let smoke = first_smoke(&mut manifest); + smoke.deadline_seconds = smoke.lsp_timeout_seconds.saturating_sub(1); + + let error = manifest + .validate(repository_root()) + .expect_err("short overall deadline should fail"); + + assert!(error.contains("deadline must not be shorter")); +} + +#[test] +fn manifest_rejects_duplicate_host_programs() { + let mut manifest = Manifest::load().expect("E2E manifest should parse"); + let smoke = first_smoke(&mut manifest); + let duplicate = smoke + .host_programs + .first() + .expect("smoke case should have a host program") + .clone(); + smoke.host_programs.push(duplicate); + + let error = manifest + .validate(repository_root()) + .expect_err("duplicate host programs should fail"); + + assert!(error.contains("more than once")); +} diff --git a/tests/e2e/process.rs b/tests/e2e/process.rs new file mode 100644 index 0000000..3f3c97d --- /dev/null +++ b/tests/e2e/process.rs @@ -0,0 +1,397 @@ +use std::ffi::OsStr; +use std::io; +use std::path::PathBuf; +use std::process::{Command, ExitStatus, Stdio}; +use std::sync::mpsc::{self, Receiver, RecvTimeoutError}; +use std::thread::{self, JoinHandle}; +use std::time::{Duration, Instant}; + +use command_group::{CommandGroup as _, GroupChild}; +use wait_timeout::ChildExt as _; + +const OUTPUT_EXCERPT_LIMIT: usize = 16 * 1024; +const READER_CLEANUP_DEADLINE: Duration = Duration::from_secs(1); + +pub(crate) struct ProcessOutput { + details: ProcessDetails, + status: ExitStatus, +} + +pub(crate) struct ProcessFailure { + details: Box, + reason: String, +} + +struct ProcessDetails { + command: String, + cwd: PathBuf, + deadline: Duration, + elapsed: Duration, + stdout: Vec, + stderr: Vec, +} + +struct RunningGroup { + child: Option, + stdout_reader: OutputReader, + stderr_reader: OutputReader, +} + +struct OutputReader { + result: Option>>>, + thread: Option>, + output: Option>, +} + +pub(crate) fn run( + command: &mut Command, + deadline: Duration, +) -> Result { + let command_line = render_command(command); + let cwd = command + .get_current_dir() + .map_or_else(|| PathBuf::from(""), PathBuf::from); + let started = Instant::now(); + let absolute_deadline = started.checked_add(deadline).unwrap_or(started); + command + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + + let child = command.group_spawn().map_err(|error| ProcessFailure { + details: Box::new(ProcessDetails::empty( + command_line.clone(), + cwd.clone(), + deadline, + started.elapsed(), + )), + reason: format!("failed to start process group: {error}"), + })?; + let mut running = RunningGroup::new(child).map_err(|(mut child, error)| { + let cleanup = terminate_group(&mut child); + ProcessFailure { + details: Box::new(ProcessDetails::empty( + command_line.clone(), + cwd.clone(), + deadline, + started.elapsed(), + )), + reason: format!("failed to capture process output: {error}; {cleanup}"), + } + })?; + + // `wait-timeout` extends `Child`, so borrow the group leader only for the bounded wait. + // If it expires, `GroupChild::kill` still terminates the complete process group. + let wait_result = running.child_mut().and_then(|child| { + child + .inner() + .wait_timeout(deadline.saturating_sub(started.elapsed())) + .map_err(|error| format!("failed while waiting for process group: {error}")) + }); + + match wait_result { + Ok(Some(status)) => { + let (stdout, stderr) = match running.finish_readers_before(absolute_deadline) { + Ok(output) => output, + Err(reason) => { + let cleanup = running.terminate(); + let (stdout, stderr) = running.finish_readers_after_termination(); + return Err(ProcessFailure { + details: Box::new(ProcessDetails { + command: command_line, + cwd, + deadline, + elapsed: started.elapsed(), + stdout, + stderr, + }), + reason: format!("{reason}; {cleanup}"), + }); + } + }; + running.disarm_child(); + Ok(ProcessOutput { + details: ProcessDetails { + command: command_line, + cwd, + deadline, + elapsed: started.elapsed(), + stdout, + stderr, + }, + status, + }) + } + Ok(None) => { + let cleanup = running.terminate(); + let (stdout, stderr) = running.finish_readers_after_termination(); + Err(ProcessFailure { + details: Box::new(ProcessDetails { + command: command_line, + cwd, + deadline, + elapsed: started.elapsed(), + stdout, + stderr, + }), + reason: format!("process exceeded its deadline; {cleanup}"), + }) + } + Err(reason) => { + let cleanup = running.terminate(); + let (stdout, stderr) = running.finish_readers_after_termination(); + Err(ProcessFailure { + details: Box::new(ProcessDetails { + command: command_line, + cwd, + deadline, + elapsed: started.elapsed(), + stdout, + stderr, + }), + reason: format!("{reason}; {cleanup}"), + }) + } + } +} + +impl ProcessOutput { + pub(crate) fn status(&self) -> ExitStatus { + self.status + } + + pub(crate) fn stdout(&self) -> &[u8] { + &self.details.stdout + } + + pub(crate) fn stderr(&self) -> &[u8] { + &self.details.stderr + } + + pub(crate) fn diagnostic(&self, reason: &str, runtime_state: &str) -> String { + self.details + .diagnostic(reason, Some(self.status), runtime_state) + } +} + +impl ProcessFailure { + pub(crate) fn diagnostic(&self, runtime_state: &str) -> String { + self.details.diagnostic(&self.reason, None, runtime_state) + } +} + +impl ProcessDetails { + fn empty(command: String, cwd: PathBuf, deadline: Duration, elapsed: Duration) -> Self { + Self { + command, + cwd, + deadline, + elapsed, + stdout: Vec::new(), + stderr: Vec::new(), + } + } + + fn diagnostic(&self, reason: &str, status: Option, runtime_state: &str) -> String { + let status = status.map_or_else(|| "not available".to_string(), |value| value.to_string()); + format!( + "{reason}\ncommand: {}\nworking directory: {}\nstatus: {status}\nelapsed: {:?}\ndeadline: {:?}\nruntime state:\n{runtime_state}\nstdout:\n{}\nstderr:\n{}", + self.command, + self.cwd.display(), + self.elapsed, + self.deadline, + output_excerpt(&self.stdout), + output_excerpt(&self.stderr), + ) + } +} + +impl RunningGroup { + fn new(mut child: GroupChild) -> Result { + let Some(stdout) = child.inner().stdout.take() else { + return Err((child, io::Error::other("stdout pipe was not created"))); + }; + let Some(stderr) = child.inner().stderr.take() else { + return Err((child, io::Error::other("stderr pipe was not created"))); + }; + + Ok(Self { + child: Some(child), + stdout_reader: OutputReader::new(stdout), + stderr_reader: OutputReader::new(stderr), + }) + } + + fn child_mut(&mut self) -> Result<&mut GroupChild, String> { + self.child + .as_mut() + .ok_or_else(|| "running process group lost its child handle".to_string()) + } + + fn disarm_child(&mut self) { + let _child = self.child.take(); + } + + fn terminate(&mut self) -> String { + let Some(mut child) = self.child.take() else { + return "process group already reaped".to_string(); + }; + terminate_group(&mut child) + } + + fn finish_readers_before(&mut self, deadline: Instant) -> Result<(Vec, Vec), String> { + let stdout = self.stdout_reader.finish_before(deadline, "stdout"); + let stderr = self.stderr_reader.finish_before(deadline, "stderr"); + match (stdout, stderr) { + (Ok(()), Ok(())) => Ok(self.take_output()), + (Err(stdout), Ok(())) => Err(stdout), + (Ok(()), Err(stderr)) => Err(stderr), + (Err(stdout), Err(stderr)) => Err(format!("{stdout}; {stderr}")), + } + } + + fn finish_readers_after_termination(&mut self) -> (Vec, Vec) { + let deadline = Instant::now() + .checked_add(READER_CLEANUP_DEADLINE) + .unwrap_or_else(Instant::now); + let error = self.finish_readers_before(deadline).err(); + let (stdout, mut stderr) = self.take_output(); + if let Some(error) = error { + stderr.extend_from_slice(format!("\nfailed to capture output: {error}").as_bytes()); + } + (stdout, stderr) + } + + fn take_output(&mut self) -> (Vec, Vec) { + ( + self.stdout_reader.take_output(), + self.stderr_reader.take_output(), + ) + } +} + +impl Drop for RunningGroup { + fn drop(&mut self) { + // This guard is necessary because test assertions may unwind while descendants are alive. + if self.child.is_some() { + let _cleanup_result = self.terminate(); + } + let _captured_output = self.finish_readers_after_termination(); + } +} + +impl OutputReader { + fn new(reader: impl io::Read + Send + 'static) -> Self { + let (sender, result) = mpsc::channel(); + let thread = thread::spawn(move || { + let _send_result = sender.send(read_all(reader)); + }); + Self { + result: Some(result), + thread: Some(thread), + output: None, + } + } + + fn finish_before(&mut self, deadline: Instant, stream: &str) -> Result<(), String> { + if self.output.is_some() { + return Ok(()); + } + let Some(result) = self.result.as_ref() else { + return Err(format!("{stream} capture result is unavailable")); + }; + let remaining = deadline.saturating_duration_since(Instant::now()); + let output = match result.recv_timeout(remaining) { + Ok(output) => output.map_err(|error| format!("failed to read {stream}: {error}"))?, + Err(RecvTimeoutError::Timeout) => { + return Err(format!("{stream} remained open after the command deadline")); + } + Err(RecvTimeoutError::Disconnected) => { + return Err(format!("{stream} capture thread exited without a result")); + } + }; + self.result = None; + if let Some(thread) = self.thread.take() + && thread.join().is_err() + { + return Err(format!("{stream} capture thread panicked")); + } + self.output = Some(output); + Ok(()) + } + + fn take_output(&mut self) -> Vec { + self.output.take().unwrap_or_default() + } +} + +fn terminate_group(child: &mut GroupChild) -> String { + let kill_result = child.kill(); + let wait_result = child.wait(); + match (kill_result, wait_result) { + (Ok(()), Ok(status)) => format!("process group killed and reaped with {status}"), + (Err(kill_error), Ok(status)) if kill_error.kind() == io::ErrorKind::InvalidInput => { + format!("process group exited during cleanup with {status}") + } + (Err(kill_error), Ok(status)) => { + format!("process group reaped with {status}, but kill failed: {kill_error}") + } + (Ok(()), Err(wait_error)) => { + format!("process group was killed but could not be reaped: {wait_error}") + } + (Err(kill_error), Err(wait_error)) => { + format!("process group could not be killed ({kill_error}) or reaped ({wait_error})") + } + } +} + +fn read_all(mut reader: impl io::Read) -> io::Result> { + let mut output = Vec::new(); + reader.read_to_end(&mut output)?; + Ok(output) +} + +fn render_command(command: &Command) -> String { + std::iter::once(command.get_program()) + .chain(command.get_args()) + .map(render_argument) + .collect::>() + .join(" ") +} + +fn render_argument(argument: &OsStr) -> String { + format!("{argument:?}") +} + +fn output_excerpt(output: &[u8]) -> String { + let omitted = output.len().saturating_sub(OUTPUT_EXCERPT_LIMIT); + let excerpt = output + .get(omitted..) + .map_or(output, |bounded_output| bounded_output); + if omitted == 0 { + String::from_utf8_lossy(excerpt).into_owned() + } else { + format!( + "<{} earlier bytes omitted>\n{}", + omitted, + String::from_utf8_lossy(excerpt) + ) + } +} + +#[cfg(test)] +mod tests { + use super::{OUTPUT_EXCERPT_LIMIT, output_excerpt}; + + #[test] + fn diagnostic_output_keeps_only_a_bounded_tail() { + let output = vec![b'x'; OUTPUT_EXCERPT_LIMIT + 7]; + let excerpt = output_excerpt(&output); + + assert!(excerpt.starts_with("<7 earlier bytes omitted>\n")); + assert_eq!( + excerpt.bytes().filter(|byte| *byte == b'x').count(), + OUTPUT_EXCERPT_LIMIT + ); + } +} diff --git a/tests/e2e/queries.rs b/tests/e2e/queries.rs new file mode 100644 index 0000000..82808d4 --- /dev/null +++ b/tests/e2e/queries.rs @@ -0,0 +1,79 @@ +use serde_json::Value; + +use crate::local_fixture::LocalFixture; +use crate::manifest::CommandStrategy; + +#[test] +fn lsp_fixture_command_paths_are_covered() { + let fixture = LocalFixture::new().expect("local fixture should initialize"); + + for command in fixture.commands_for(CommandStrategy::LspFixture) { + run_query(&fixture, command); + } +} + +fn run_query(fixture: &LocalFixture, command: &str) { + let mut args = command_prefix(command); + args.extend([ + "--lsp".to_string(), + fixture.server_name().to_string(), + "--no-download".to_string(), + "--no-detach".to_string(), + "--timeout".to_string(), + "5".to_string(), + ]); + if !matches!(command, "server-capabilities" | "format" | "build-index") { + args.push("--json".to_string()); + } + if command == "format" { + args.push("--stdout".to_string()); + } + let refs = args.iter().map(String::as_str).collect::>(); + let output = fixture.context().run(&refs); + output.assert_success(); + + match command { + "server-capabilities" => assert!(output.stdout_text().contains("workspace symbols")), + "build-index" => assert!(output.stdout_text().is_empty()), + "format" => { + assert!(output.stdout_text().contains(" formatted")); + assert!( + std::fs::read_to_string(fixture.context().workspace().join("main.fake")) + .expect("fixture source should remain readable") + .contains(" unformatted") + ); + } + "diagnostics" => { + let value: Value = output.json(); + assert_eq!( + value + .pointer("/diagnostics/0/message") + .and_then(Value::as_str), + Some("synthetic diagnostic") + ); + } + other => { + let value: Value = output.json(); + assert_eq!( + value.pointer("/matches/0/name").and_then(Value::as_str), + Some("Target"), + "command {other}" + ); + } + } +} + +fn command_prefix(command: &str) -> Vec { + match command { + "grep" | "references" | "callers" | "callees" | "definition" | "declaration" => { + vec![command.to_string(), "Target".to_string(), ".".to_string()] + } + "format" => vec![command.to_string(), "main.fake".to_string()], + "server-capabilities" + | "diagnostics" + | "list-symbols" + | "list-functions" + | "build-index" => vec![command.to_string(), ".".to_string()], + other => panic!("LSP fixture strategy has no scenario for {other:?}"), + } +} diff --git a/tests/e2e/real_servers.rs b/tests/e2e/real_servers.rs new file mode 100644 index 0000000..c9f65ea --- /dev/null +++ b/tests/e2e/real_servers.rs @@ -0,0 +1,156 @@ +use std::collections::BTreeSet; +use std::path::Path; +use std::time::{Duration, Instant}; + +use serde::Deserialize; + +use crate::harness::E2eContext; +use crate::manifest::{Manifest, ProvisionMethod, QueryKind, RealServerCase}; +use crate::repository_root; + +struct RealServerTest<'a> { + case: RealServerCase<'a>, + repository: &'a Path, +} + +#[derive(Deserialize)] +struct ListSymbolsOutput { + detected: BTreeSet, + server: ServerOutput, + matches: Vec, +} + +#[derive(Deserialize)] +struct ServerOutput { + command: Vec, +} + +#[derive(Deserialize)] +struct SymbolOutput { + name: String, +} + +impl<'a> RealServerTest<'a> { + fn new(case: RealServerCase<'a>, repository: &'a Path) -> Self { + Self { case, repository } + } + + fn run(self) -> Result<(), String> { + self.run_inner() + .map_err(|error| format!("E2E case {} failed:\n{error}", self.case.label())) + } + + fn run_inner(&self) -> Result<(), String> { + let started = Instant::now(); + let deadline = Duration::from_secs(self.case.deadline_seconds()); + let context = E2eContext::new() + .map_err(|error| format!("failed to create an isolated E2E context: {error}"))?; + context.copy_project(&self.repository.join(self.case.project()))?; + + for (name, resolver) in self.case.host_programs() { + context.stage_host_program(name, resolver, remaining(started, deadline)?)?; + } + + let server_name = self.case.server_name(self.repository)?; + let args = self.command_args(&server_name); + let arg_refs = args.iter().map(String::as_str).collect::>(); + let output = context.try_run_with_deadline(&arg_refs, remaining(started, deadline)?)?; + output.ensure_success()?; + + match self.case.query_kind() { + QueryKind::ListSymbols => { + let response: ListSymbolsOutput = output.try_json()?; + self.validate_list_symbols(&context, &response) + } + } + } + + fn command_args(&self, server_name: &str) -> Vec { + let mut args = match self.case.query_kind() { + QueryKind::ListSymbols => vec!["list-symbols".to_string(), ".".to_string()], + }; + args.extend(["--lsp".to_string(), server_name.to_string()]); + match self.case.provision_method() { + ProvisionMethod::Download => args.push("--download".to_string()), + } + args.extend([ + "--no-detach".to_string(), + "--json".to_string(), + "--timeout".to_string(), + self.case.lsp_timeout_seconds().to_string(), + ]); + args + } + + fn validate_list_symbols( + &self, + context: &E2eContext, + response: &ListSymbolsOutput, + ) -> Result<(), String> { + if !response.detected.contains(self.case.language()) { + return Err(format!( + "expected detected languages {:?} to contain {:?}", + response.detected, + self.case.language() + )); + } + + let Some(program) = response.server.command.first() else { + return Err("selected server reported an empty command".to_string()); + }; + if !Path::new(program).starts_with(context.home()) { + return Err(format!( + "downloaded server program {} is outside isolated home {}", + program, + context.home().display() + )); + } + + let actual = response + .matches + .iter() + .map(|matched| matched.name.as_str()) + .collect::>(); + let missing = self + .case + .expected_names() + .iter() + .filter(|expected| !actual.contains(expected.as_str())) + .collect::>(); + if missing.is_empty() { + Ok(()) + } else { + Err(format!( + "missing expected symbols {missing:?}; returned symbols: {actual:?}" + )) + } + } +} + +fn remaining(started: Instant, deadline: Duration) -> Result { + let remaining = deadline.saturating_sub(started.elapsed()); + if remaining.is_zero() { + Err(format!( + "case exceeded its overall deadline of {deadline:?}" + )) + } else { + Ok(remaining) + } +} + +#[test] +#[ignore = "downloads and runs real LSP servers; executed explicitly in CI"] +fn manifest_real_server_smoke_cases() { + let repository = repository_root(); + let manifest = Manifest::load_validated(repository).expect("E2E manifest should be valid"); + let failures = manifest + .real_server_smoke_cases() + .filter_map(|case| RealServerTest::new(case, repository).run().err()) + .collect::>(); + + assert!( + failures.is_empty(), + "real-server E2E failures:\n{}", + failures.join("\n\n") + ); +} diff --git a/tests/e2e/update.rs b/tests/e2e/update.rs new file mode 100644 index 0000000..1add672 --- /dev/null +++ b/tests/e2e/update.rs @@ -0,0 +1,175 @@ +use std::io::{Read, Write}; +use std::net::{TcpListener, TcpStream}; +use std::thread; +use std::time::{Duration, Instant}; + +use flate2::Compression; +use flate2::write::GzEncoder; +use serde_json::json; + +use crate::fixture::E2eFixture; +use crate::manifest::CommandStrategy; + +#[test] +fn update_command_path_uses_local_release_fixture() { + let fixture = E2eFixture::new().expect("E2E fixture should initialize"); + assert_eq!( + fixture + .commands_for(CommandStrategy::UpdateFixture) + .collect::>(), + ["update"] + ); + let archive = data_archive(); + let server = HttpFixture::start(archive); + let context = fixture.context(); + + let output = context.run_with_env( + &["update"], + &[("LSP_CLI_DATA_RELEASE_API_URL", server.release_url())], + ); + output.assert_success(); + assert_eq!(output.stdout_text(), "updated lsp-cli data to e2e-v1\n"); + assert!( + context + .installed_data() + .join("filetypes/fake.yaml") + .is_file() + ); + server.finish(); +} + +struct HttpFixture { + release_url: String, + thread: Option>>, +} + +impl HttpFixture { + fn start(archive: Vec) -> Self { + let listener = TcpListener::bind("127.0.0.1:0").expect("HTTP fixture should bind"); + listener + .set_nonblocking(true) + .expect("HTTP fixture should become nonblocking"); + let address = listener + .local_addr() + .expect("HTTP fixture should have an address"); + let base = format!("http://{address}"); + let release_url = format!("{base}/release"); + let thread = thread::spawn(move || serve(listener, &base, &archive)); + Self { + release_url, + thread: Some(thread), + } + } + + fn release_url(&self) -> &str { + &self.release_url + } + + fn finish(mut self) { + self.thread + .take() + .expect("HTTP fixture thread should be present") + .join() + .expect("HTTP fixture thread should not panic") + .expect("HTTP fixture should serve both requests"); + } +} + +impl Drop for HttpFixture { + fn drop(&mut self) { + // Join here because an assertion may unwind before `finish`, leaving a fixture thread alive. + if let Some(thread) = self.thread.take() { + let _result = thread.join(); + } + } +} + +fn serve(listener: TcpListener, base: &str, archive: &[u8]) -> Result<(), String> { + let deadline = Instant::now() + Duration::from_secs(10); + let mut served = 0; + while served < 2 && Instant::now() < deadline { + match listener.accept() { + Ok((mut stream, _address)) => { + serve_request(&mut stream, base, archive)?; + served += 1; + } + Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => { + thread::sleep(Duration::from_millis(10)); + } + Err(error) => return Err(format!("HTTP fixture accept failed: {error}")), + } + } + if served == 2 { + Ok(()) + } else { + Err(format!( + "HTTP fixture served {served} of 2 expected requests" + )) + } +} + +fn serve_request(stream: &mut TcpStream, base: &str, archive: &[u8]) -> Result<(), String> { + stream + .set_read_timeout(Some(Duration::from_secs(2))) + .map_err(|error| error.to_string())?; + let mut request = [0; 4096]; + let length = stream + .read(&mut request) + .map_err(|error| error.to_string())?; + let request = String::from_utf8_lossy( + request + .get(..length) + .expect("read byte count should fit its buffer"), + ); + let (content_type, body) = if request.starts_with("GET /release ") { + let body = json!({ + "tag_name": "e2e-v1", + "tarball_url": format!("{base}/archive"), + "zipball_url": null + }) + .to_string() + .into_bytes(); + ("application/json", body) + } else if request.starts_with("GET /archive ") { + ("application/gzip", archive.to_vec()) + } else { + return Err(format!("unexpected HTTP fixture request: {request}")); + }; + write!( + stream, + "HTTP/1.1 200 OK\r\nContent-Type: {content_type}\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", + body.len() + ) + .and_then(|()| stream.write_all(&body)) + .map_err(|error| error.to_string()) +} + +fn data_archive() -> Vec { + let encoder = GzEncoder::new(Vec::new(), Compression::default()); + let mut archive = tar::Builder::new(encoder); + append( + &mut archive, + "data/filetypes/fake.yaml", + b"extensions: [fake]\n", + ); + append( + &mut archive, + "data/lsp/fake.yaml", + b"filetypes: [fake]\nroot_markers: []\nname: fake\ncmdline: fake\n", + ); + archive + .into_inner() + .expect("archive should finish") + .finish() + .expect("gzip stream should finish") +} + +fn append(archive: &mut tar::Builder>>, path: &str, contents: &[u8]) { + let mut header = tar::Header::new_gnu(); + header.set_size(u64::try_from(contents.len()).expect("fixture size should fit u64")); + header.set_mode(0o644); + header.set_cksum(); + archive + .append_data(&mut header, path, contents) + .expect("fixture archive entry should append"); +}