From 8b6172f2e8aefcabe330f0514e0cc7714caab310 Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Sat, 26 Sep 2026 20:21:22 +1000 Subject: [PATCH 01/11] prek `cargo-doc` to manual stage --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 939717d2d9..cde89207ec 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -87,7 +87,7 @@ repos: types: [rust] language: system pass_filenames: false - stages: [pre-push] + stages: [manual] # Prettier - repo: https://github.com/rbubley/mirrors-prettier From fabce8a6074aed64700fdf69e817fb33d684d37f Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Fri, 25 Sep 2026 19:59:52 +1000 Subject: [PATCH 02/11] leave a tree a build wrote out of the project it was built from, recognised by the manifest at its top wherever --out put it, in the walk and in the watcher's changes, so by check and workspace diagnostics no longer check and report every generated copy of the project, and mark the output before the first file lands in it --- crates/by_stage/src/project.rs | 141 ++---------------- crates/by_stage/src/staging.rs | 117 ++++++++++++++- crates/by_stage/src/verbatim.rs | 12 +- crates/ty/src/by_commands.rs | 4 + crates/ty/tests/by_e2e.rs | 98 +++++++++++++ crates/ty/tests/file_watching.rs | 48 ++++++- crates/ty_project/src/build_output.rs | 165 ++++++++++++++++++++++ crates/ty_project/src/db/changes.rs | 45 +++++- crates/ty_project/src/lib.rs | 1 + crates/ty_project/src/walk.rs | 20 ++- crates/ty_server/tests/e2e/build_trees.rs | 139 ++++++++++++++++++ crates/ty_server/tests/e2e/main.rs | 1 + 12 files changed, 650 insertions(+), 141 deletions(-) create mode 100644 crates/ty_project/src/build_output.rs create mode 100644 crates/ty_server/tests/e2e/build_trees.rs diff --git a/crates/by_stage/src/project.rs b/crates/by_stage/src/project.rs index 326bd1e454..50cc973ec4 100644 --- a/crates/by_stage/src/project.rs +++ b/crates/by_stage/src/project.rs @@ -5,8 +5,10 @@ //! project reports errors for files the project deliberately excludes — and, when //! this was first written, wrote nothing for them either. So the file set is the //! project's own (`Project::files`), narrowed only by things a *build* has to -//! exclude and a check does not: a hidden directory holding copies, and the last -//! build's own output. +//! exclude and a check does not: a hidden directory holding copies, and the +//! directory this build writes to. Every tree a build already wrote carries a +//! manifest, and the project leaves those out itself — see +//! [`ty_project::build_output`]. //! //! It lives here rather than in the `by` binary because the language server needs //! the same answers. A single file re-staged into a build tree has to land where @@ -14,13 +16,11 @@ //! build would have transpiled it, and the only way to guarantee that is for there //! to be one implementation of each. -use std::collections::HashMap; use std::ffi::OsStr; use std::path::{Path, PathBuf}; use anyhow::Context; -use ruff_db::Db as _; -use ruff_db::system::{OsSystem, System, SystemPath, SystemPathBuf}; +use ruff_db::system::{OsSystem, SystemPath, SystemPathBuf}; use ty_project::{Db, ProjectDatabase, ProjectMetadata}; /// Everything needed to build a project db a second time. @@ -310,8 +310,7 @@ pub(crate) fn project_sources( root: &Path, output: Option<&Path>, ) -> Vec<(PathBuf, ruff_db::files::File)> { - let candidates: Vec<(PathBuf, ruff_db::files::File)> = db - .project() + db.project() .files(db) .into_iter() .filter(|file| { @@ -327,79 +326,16 @@ pub(crate) fn project_sources( // dependencies, not this project's sources — emitting them would write // a parallel tree nobody asked for .filter(|(path, _)| !is_hidden_within(path, root)) - // nor is the last build's output. it holds a copy of every `.by` source - // this build is about to read, and reading those instead would build the - // project into itself, one directory deeper each time + // nor is the directory this build writes to. it holds a copy of every + // `.by` source this build is about to read, and reading those instead + // would build the project into itself, one directory deeper each time. + // a tree a build finished carries a manifest and is never among the + // project's files to begin with, whatever `--out` called it; this is for + // the one this run was told to write to, which may not have one yet .filter(|(path, _)| output.is_none_or(|output| !path.starts_with(output))) - .collect(); - // and nor is any *other* build's output. `--out` can name any directory, and - // a project can have several, so the one this run was given is not the only - // one that has to be turned away - let mut known = HashMap::new(); - candidates - .into_iter() - .filter(|(path, _)| !inside_build_output(db.system(), path, root, &mut known)) .collect() } -/// Whether `path` sits inside a tree that some build already wrote. -/// -/// A build output is self-describing: it holds a `.by-manifest` naming everything -/// that build put there. That marker is what identifies it, rather than the -/// directory's name. [`NON_SOURCE_DIRS`] already turns away the two default names, -/// but `--out` can say anything, and only the tree this run was *given* is known -/// to be an output from the arguments — a project that builds into two of them has -/// to have the other recognised on sight. -/// -/// It has to be turned away because the tree holds a *copy* of the project: every -/// hand-written `.py`, and every `.by` too when `build.sources` is on. Reading one -/// back means each of those modules is claimed twice — once where it was written -/// and once where it was copied — and the build stops, telling the author that two -/// files they never put side by side would compile to the same module. -/// -/// The project root is never itself an output, whatever it holds: a `.by-manifest` -/// left at the root would otherwise empty the project of every file it has. -fn inside_build_output( - system: &dyn System, - path: &Path, - root: &Path, - known: &mut HashMap, -) -> bool { - let Some(parent) = path.parent() else { - return false; - }; - let mut current = parent; - while current != root && current.starts_with(root) { - let is_output = match known.get(current) { - Some(answer) => *answer, - None => { - let answer = holds_manifest(system, current); - known.insert(current.to_path_buf(), answer); - answer - } - }; - if is_output { - return true; - } - match current.parent() { - Some(next) => current = next, - None => return false, - } - } - false -} - -/// Whether `directory` carries the manifest that marks it a build output. -/// -/// Asked of the db's [`System`] rather than of `std::fs`, so that it is the same -/// file system every other question in a build is asked — the language server's -/// re-stage runs this on a warm db, and an in-memory test system has to be able -/// to answer it. -pub(crate) fn holds_manifest(system: &dyn System, directory: &Path) -> bool { - SystemPath::from_std_path(directory) - .is_some_and(|path| system.is_file(&path.join(crate::staging::MANIFEST_FILENAME))) -} - /// The project's first-party module roots, longest first, as absolute paths. /// /// These are the directories a module name is resolved against — for a @@ -420,60 +356,9 @@ pub fn module_roots(db: &ProjectDatabase, cwd: &Path) -> Vec { #[cfg(test)] mod tests { - use super::{inside_build_output, is_hidden_within}; - use ruff_db::system::{OsSystem, SystemPath}; - use std::collections::HashMap; + use super::is_hidden_within; use std::path::Path; - /// `--out` can name any directory and a project can hold several, so a build - /// output is recognised by the manifest it carries rather than by its name. - #[test] - fn a_directory_holding_a_manifest_is_a_build_output() { - let directory = tempfile::tempdir().expect("tempdir"); - let root = directory.path(); - let output = root.join("anything"); - std::fs::create_dir_all(output.join("pkg")).expect("create"); - std::fs::write(output.join(".by-manifest"), "").expect("write"); - let plain = root.join("src"); - std::fs::create_dir_all(&plain).expect("create"); - - let system = - OsSystem::new(SystemPath::from_std_path(root).expect("the temp directory is utf-8")); - let mut known = HashMap::new(); - assert!(inside_build_output( - &system, - &output.join("a.by"), - root, - &mut known - )); - assert!( - inside_build_output(&system, &output.join("pkg").join("a.by"), root, &mut known), - "a file deeper inside the output is inside it too" - ); - assert!( - !inside_build_output(&system, &plain.join("a.by"), root, &mut known), - "a directory with no manifest is ordinary source" - ); - } - - /// A manifest at the root would otherwise empty the project of every file. - #[test] - fn the_project_root_is_never_itself_an_output() { - let directory = tempfile::tempdir().expect("tempdir"); - let root = directory.path(); - std::fs::write(root.join(".by-manifest"), "").expect("write"); - - let system = - OsSystem::new(SystemPath::from_std_path(root).expect("the temp directory is utf-8")); - let mut known = HashMap::new(); - assert!(!inside_build_output( - &system, - &root.join("a.by"), - root, - &mut known - )); - } - #[test] fn a_hidden_directory_is_not_project_source() { let root = Path::new("/p"); diff --git a/crates/by_stage/src/staging.rs b/crates/by_stage/src/staging.rs index ea08eee095..7413b94a9d 100644 --- a/crates/by_stage/src/staging.rs +++ b/crates/by_stage/src/staging.rs @@ -8,6 +8,7 @@ use std::collections::{BTreeMap, BTreeSet}; use std::fs; +use std::io::Write as _; use std::path::{Path, PathBuf}; use anyhow::Context; @@ -20,7 +21,13 @@ use anyhow::Context; /// until one shadows a module that moved. The manifest is what makes the output a /// mirror rather than a pile: what the previous build wrote and this one did not /// is deleted. -pub(crate) const MANIFEST_FILENAME: &str = ".by-manifest"; +/// +/// Being there is also what marks the tree as a build output: the project the build +/// was made from leaves it out of its own files — see [`ty_project::build_output`]. +const MANIFEST_FILENAME: &str = ty_project::build_output::BUILD_MANIFEST; + +/// the line a manifest starts with, before the paths it records +const MANIFEST_HEADER: &str = "# written by `by build`; delete it and stale output stays\n"; /// An output tree being written. pub struct Staging { @@ -30,6 +37,8 @@ pub struct Staging { /// destinations another writer produced, held apart from `written` so a /// source that would land on one is reported rather than overwriting it recorded: BTreeSet, + /// whether the tree is known to carry a manifest already — see [`Self::mark_as_output`] + marked: bool, } impl Staging { @@ -38,9 +47,46 @@ impl Staging { out: out.to_path_buf(), written: BTreeMap::new(), recorded: BTreeSet::new(), + marked: false, } } + /// mark the tree as a build output before the first file lands in it + /// + /// the manifest [`Self::finish`] writes is also what tells the project this tree is not + /// part of it. a first build writes into a directory that has none yet, and a language + /// server watching the project would otherwise see a copy of every source appear, take them + /// for the project's own, and check each one until the build finished and the manifest + /// arrived. an empty manifest says the same as a missing one to the next build (nothing + /// was written here before), and one already there is left for `finish` to read + /// + /// writing and copying through the staging mark it themselves, and so does [`Self::record`]. + /// a writer that lays files out itself, as `by compile` does, calls this before it writes + pub fn mark_as_output(&mut self) -> anyhow::Result<()> { + if self.marked { + return Ok(()); + } + let manifest = self.out.join(MANIFEST_FILENAME); + create_parent(&manifest)?; + match fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(&manifest) + { + Ok(mut file) => { + file.write_all(MANIFEST_HEADER.as_bytes()) + .with_context(|| format!("could not write {}", manifest.display()))?; + } + Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => {} + Err(error) => { + return Err(error) + .with_context(|| format!("could not write {}", manifest.display())); + } + } + self.marked = true; + Ok(()) + } + pub(crate) fn out(&self) -> &Path { &self.out } @@ -76,6 +122,7 @@ impl Staging { contents: &str, ) -> anyhow::Result<()> { self.claim(relative, source)?; + self.mark_as_output()?; let destination = self.out.join(relative); create_parent(&destination)?; fs::write(&destination, contents) @@ -104,6 +151,7 @@ impl Staging { claimant(source.as_deref()), ); } + self.mark_as_output()?; self.recorded.insert(relative.to_path_buf()); Ok(()) } @@ -111,6 +159,7 @@ impl Staging { /// Copy `source` to `relative` verbatim. pub(crate) fn copy(&mut self, relative: &Path, source: &Path) -> anyhow::Result<()> { self.claim(relative, Some(source))?; + self.mark_as_output()?; let destination = self.out.join(relative); create_parent(&destination)?; fs::copy(source, &destination).with_context(|| { @@ -190,8 +239,7 @@ impl Staging { } create_parent(&manifest)?; - let mut rendered = - String::from("# written by `by build`; delete it and stale output stays\n"); + let mut rendered = String::from(MANIFEST_HEADER); for path in current { rendered.push_str(&portable(path)); rendered.push('\n'); @@ -426,6 +474,69 @@ mod tests { assert!(!directory.path().join("mine.py").exists()); } + /// the manifest is what keeps the tree out of the project it was built from, so it is + /// there before the first file is, not only once the build is done + #[test] + fn a_first_build_marks_its_output_before_writing_into_it() { + let directory = tempfile::tempdir().expect("tempdir"); + let out = directory.path().join("build"); + let mut staging = Staging::new(&out); + staging + .write(Path::new("a.py"), None, "x = 1\n") + .expect("write"); + assert!( + ty_project::build_output::is_build_output( + &ruff_db::system::OsSystem::new( + ruff_db::system::SystemPath::from_std_path(directory.path()).expect("utf-8") + ), + ruff_db::system::SystemPath::from_std_path(&out).expect("utf-8"), + ), + "a tree being written is an output before the build finishes" + ); + staging.finish().expect("finish"); + assert!(read_manifest(&out.join(MANIFEST_FILENAME)).contains(Path::new("a.py"))); + } + + /// a file another writer put into the tree marks it too + #[test] + fn recording_an_artifact_marks_the_output() { + let directory = tempfile::tempdir().expect("tempdir"); + let out = directory.path().join("build"); + let mut staging = Staging::new(&out); + staging.record(Path::new("a.so")).expect("record"); + assert!( + ty_project::build_output::is_build_output( + &ruff_db::system::OsSystem::new( + ruff_db::system::SystemPath::from_std_path(directory.path()).expect("utf-8") + ), + ruff_db::system::SystemPath::from_std_path(&out).expect("utf-8"), + ), + "a tree with an artifact recorded in it is an output before the build finishes" + ); + } + + /// marking a tree the last build left does not forget what that build wrote + #[test] + fn marking_an_output_keeps_the_previous_manifest() { + let directory = tempfile::tempdir().expect("tempdir"); + let mut first = Staging::new(directory.path()); + first + .write(Path::new("gone.py"), None, "x = 1\n") + .expect("write"); + first.finish().expect("finish"); + + let mut second = Staging::new(directory.path()); + second + .write(Path::new("kept.py"), None, "x = 1\n") + .expect("write"); + assert!( + read_manifest(&directory.path().join(MANIFEST_FILENAME)).contains(Path::new("gone.py")), + "the previous build's record survives until this one replaces it" + ); + second.finish().expect("finish"); + assert!(!directory.path().join("gone.py").exists()); + } + #[test] fn a_manifest_round_trips_through_its_portable_form() { let directory = tempfile::tempdir().expect("tempdir"); diff --git a/crates/by_stage/src/verbatim.rs b/crates/by_stage/src/verbatim.rs index cab6374dcd..7285ca3256 100644 --- a/crates/by_stage/src/verbatim.rs +++ b/crates/by_stage/src/verbatim.rs @@ -18,10 +18,11 @@ use std::path::{Path, PathBuf}; use ruff_db::Db as _; use ruff_db::system::SystemPath; +use ty_project::build_output::is_build_output; use ty_project::{Db, ProjectDatabase}; use walkdir::WalkDir; -use crate::project::{holds_manifest, may_hold_build_content}; +use crate::project::may_hold_build_content; use crate::staging::{Staging, relative_destination}; /// One file carried into the output unchanged. @@ -62,11 +63,14 @@ fn verbatim_files( // copy of every file this walk carries, so one that was not turned // away would be copied into the next wholesale, a tree deep in a // tree. it is recognised by the manifest it carries rather than by - // its name, because `--out` can say anything. the root is exempt: - // a manifest there would otherwise carry nothing over at all + // its name, because `--out` can say anything, and asked of the db's + // file system, which the language server's re-stage and an in-memory + // test system answer too. the root is exempt: a manifest there would + // otherwise carry nothing over at all if entry.file_type().is_dir() && entry.path() != root - && holds_manifest(db.system(), entry.path()) + && SystemPath::from_std_path(entry.path()) + .is_some_and(|path| is_build_output(db.system(), path)) { return false; } diff --git a/crates/ty/src/by_commands.rs b/crates/ty/src/by_commands.rs index 55628c3d0f..eabf23b338 100644 --- a/crates/ty/src/by_commands.rs +++ b/crates/ty/src/by_commands.rs @@ -1325,6 +1325,10 @@ pub(crate) fn cmd_compile( // manifest left describing the run before it would have the *next* run prune // against a tree that no longer exists let compiling: anyhow::Result<()> = 'compiling: { + // `by_build` writes each artifact before it is recorded, so the tree is marked first + if let Err(error) = staging.mark_as_output() { + break 'compiling Err(error); + } for ((path, file), name) in planned { let source = match fs::read_to_string(path) .with_context(|| format!("could not read {}", path.display())) diff --git a/crates/ty/tests/by_e2e.rs b/crates/ty/tests/by_e2e.rs index 07dfd7b554..833d244324 100644 --- a/crates/ty/tests/by_e2e.rs +++ b/crates/ty/tests/by_e2e.rs @@ -6847,3 +6847,101 @@ fn a_restaged_module_finds_a_helper_the_build_never_used() { String::from_utf8_lossy(&restaged.stderr) ); } + +/// `by check` in `dir`, as the path, relative to `dir`, of each diagnostic it reports +fn checked_paths(dir: &Path, extra: &[&str]) -> Vec { + let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") + .args(["check", "--output-format", "concise"]) + .args(extra) + .current_dir(dir) + .output() + .expect("failed to spawn by"); + let stdout = String::from_utf8_lossy(&output.stdout); + let mut paths: Vec = stdout + .lines() + .filter(|line| !line.starts_with("Found ") && !line.trim().is_empty()) + .filter_map(|line| { + line.split_once(':') + .map(|(path, _)| path.replace('\\', "/")) + }) + .collect(); + paths.sort(); + paths +} + +/// a project whose one `.by` module and one hand-written `.py` module each assign a `str` to an +/// `int`, so that a copy of either in a build output would be reported again +fn project_with_an_error_in_each_module(name: &str) -> Scratch { + let dir = Scratch::new(name); + fs::create_dir_all(dir.join("pkg")).unwrap(); + fs::write( + dir.join("pyproject.toml"), + "[project]\nname=\"s\"\nversion=\"0\"\nrequires-python=\">=3.13\"\n", + ) + .unwrap(); + fs::write(dir.join("pkg").join("__init__.py"), "").unwrap(); + fs::write(dir.join("pkg").join("main.by"), "x: int = \"by\"\n").unwrap(); + fs::write(dir.join("pkg").join("helper.py"), "y: int = \"py\"\n").unwrap(); + dir +} + +/// `by build` in `dir` with `args`, which has to write `pkg/main.py` into `out` +/// +/// the build reports the project's type errors and exits non-zero for them, and writes the tree +/// all the same, which is the tree these tests are about +fn build_into(dir: &Path, out: &str, args: &[&str]) { + let build = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") + .arg("build") + .args(args) + .current_dir(dir) + .output() + .expect("failed to spawn by"); + assert!( + dir.join(out).join("pkg").join("main.py").exists(), + "by build wrote no tree:\n{}", + String::from_utf8_lossy(&build.stderr) + ); +} + +#[test] +fn check_does_not_check_the_tree_a_build_wrote() { + // the build output is a copy of the project: a `.py` for every `.by` and every hand-written + // `.py` carried over. checked as part of the project, each copy is checked again and every + // diagnostic is reported twice, once at a path nobody edits — in a project that imports + // torch that was most of what a check cost + let dir = project_with_an_error_in_each_module("by_cli_check_skips_build"); + build_into(&dir, "build", &[]); + + assert_eq!( + checked_paths(&dir, &[]), + ["pkg/helper.py", "pkg/main.by"], + "only the sources are checked, not their copies under build/" + ); + + // a tree asked for by name is still checked: naming it is asking for it + let named = checked_paths(&dir, &["build"]); + assert!( + named.iter().any(|path| path.starts_with("build/")), + "the output named on the command line was not checked: {named:?}" + ); +} + +#[test] +fn check_follows_the_build_to_another_directory_and_keeps_a_package_called_build() { + // the output is recognised by the manifest the build leaves, not by its name: `--out` can + // write anywhere, and a directory of the author's own that happens to be called `build` is + // part of the project when the build writes somewhere else + let dir = project_with_an_error_in_each_module("by_cli_check_skips_out"); + fs::create_dir_all(dir.join("build")).unwrap(); + fs::write(dir.join("build").join("tool.py"), "z: int = \"mine\"\n").unwrap(); + + build_into(&dir, "elsewhere", &["--out", "elsewhere"]); + + assert_eq!( + checked_paths(&dir, &[]), + ["build/tool.py", "pkg/helper.py", "pkg/main.by"], + "the sources and the author's own `build/`, and nothing the build wrote" + ); +} diff --git a/crates/ty/tests/file_watching.rs b/crates/ty/tests/file_watching.rs index 3cf0b1e711..fa3ddc6afb 100644 --- a/crates/ty/tests/file_watching.rs +++ b/crates/ty/tests/file_watching.rs @@ -18,7 +18,7 @@ use ty_project::metadata::options::{EnvironmentOptions, Options, SrcOptions}; use ty_project::metadata::pyproject::{PyProject, Tool}; use ty_project::metadata::python_version::SupportedPythonVersion; use ty_project::metadata::value::{RelativeGlobPattern, RelativePathBuf}; -use ty_project::watch::{ChangeEvent, ProjectWatcher, directory_watcher}; +use ty_project::watch::{ChangeEvent, CreatedKind, DeletedKind, ProjectWatcher, directory_watcher}; use ty_project::{ChangeResult, Db, ProjectDatabase, ProjectMetadata}; use ty_python_core::platform::PythonPlatform; use ty_static::EnvVars; @@ -975,6 +975,52 @@ fn changed_file() -> anyhow::Result<()> { Ok(()) } +/// a manifest written into a directory of the project makes the directory a build's output, +/// which is not part of the project +#[test] +fn a_build_manifest_created_takes_its_directory_out_of_the_project() -> anyhow::Result<()> { + let mut case = setup([("app.py", ""), ("build/app.py", "")])?; + let app = case.system_file(case.project_path("app.py"))?; + let built = case.system_file(case.project_path("build/app.py"))?; + case.assert_indexed_project_files([app, built]); + + let manifest = case.project_path("build/.by-manifest"); + std::fs::write(manifest.as_std_path(), "app.py\n")?; + case.apply_changes(&[ChangeEvent::Created { + path: manifest, + kind: CreatedKind::File, + }]); + + case.assert_indexed_project_files([app]); + + Ok(()) +} + +/// a build's output whose manifest is deleted is a directory like any other, and part of the +/// project again +#[test] +fn a_build_manifest_deleted_returns_its_directory_to_the_project() -> anyhow::Result<()> { + let mut case = setup([ + ("app.py", ""), + ("build/.by-manifest", "app.py\n"), + ("build/app.py", ""), + ])?; + let app = case.system_file(case.project_path("app.py"))?; + case.assert_indexed_project_files([app]); + + let manifest = case.project_path("build/.by-manifest"); + std::fs::remove_file(manifest.as_std_path())?; + case.apply_changes(&[ChangeEvent::Deleted { + path: manifest, + kind: DeletedKind::File, + }]); + + let built = case.system_file(case.project_path("build/app.py"))?; + case.assert_indexed_project_files([app, built]); + + Ok(()) +} + #[test] fn scripts_to_synchronize_after_file_and_directory_changes() -> anyhow::Result<()> { let script = dedent( diff --git a/crates/ty_project/src/build_output.rs b/crates/ty_project/src/build_output.rs new file mode 100644 index 0000000000..9f55804308 --- /dev/null +++ b/crates/ty_project/src/build_output.rs @@ -0,0 +1,165 @@ +//! the trees `by build` and `by compile` write, which are not part of the project they were built +//! from +//! +//! a build output holds a copy of the project: a `.py` for every `.by`, every hand-written `.py` +//! carried over verbatim, and the `.by` sources too when `build.sources` is on. taken for part of +//! the project, each of those would be checked a second time, and every diagnostic in a source +//! reported again against its copy, at a path the author never edits +//! +//! an output is recognised by the manifest the build leaves at its top, not by its name. the +//! default output is `build/` in the project root, but `--out` can name any directory, and a +//! project can have several; each one describes itself. a directory the author happens to call +//! `build` holds no manifest, so it stays part of the project wherever the build writes +//! +//! the exclusion applies where the project's files are discovered (the walk, and the file +//! watcher's changes) the way an ignore file's does, so that `by check`, the language server's +//! workspace diagnostics, and everything else that reads [`crate::Project::files`] agree. it +//! does not make the output unreadable: module resolution reads the file system, not the +//! project's file list, and a generated file an editor opens is still a file it can ask about + +use ruff_db::system::{System, SystemPath, SystemPathBuf}; + +/// the file a build writes at the top of its output, recording what it wrote there, and, by +/// being there, marking the directory as an output +pub const BUILD_MANIFEST: &str = ".by-manifest"; + +/// whether `directory` is the top of a build output +pub fn is_build_output(system: &dyn System, directory: &SystemPath) -> bool { + system.is_file(&directory.join(BUILD_MANIFEST)) +} + +/// whether `path` lies inside a build output below the walk root that holds it +/// +/// what the walk decides for a path it reaches from one of `walk_roots` (the project root, or +/// the paths a command line named), decided for one path on its own, as the file watcher's +/// changes need. a walk root is never itself skipped: the project root is never an output, +/// whatever it holds, and a directory a command line names is checked because it was asked for, +/// output or not. only the directories between the walk root and `path` are asked +pub(crate) fn is_within_build_output( + system: &dyn System, + path: &SystemPath, + walk_roots: &[SystemPathBuf], +) -> bool { + let Some(walk_root) = innermost_walk_root(path, walk_roots) else { + return false; + }; + path.ancestors() + .skip(1) + .take_while(|directory| *directory != walk_root) + .any(|directory| is_build_output(system, directory)) +} + +/// whether `path` is the manifest of a build output inside one of `walk_roots`: its appearing or +/// disappearing changes which of the files under its directory belong to the project +pub(crate) fn is_build_manifest(path: &SystemPath, walk_roots: &[SystemPathBuf]) -> bool { + path.file_name() == Some(BUILD_MANIFEST) + && path.parent().is_some_and(|directory| { + innermost_walk_root(directory, walk_roots) + .is_some_and(|walk_root| directory != walk_root) + }) +} + +/// the walk root nearest above `path`, or `path` itself if it is one +/// +/// the walk starts again from each of its roots, one nested in another included, so a path is +/// reached from every root above it, and from the innermost one through the fewest directories. +/// a path any of the walks keeps, that one keeps +fn innermost_walk_root<'a>( + path: &SystemPath, + walk_roots: &'a [SystemPathBuf], +) -> Option<&'a SystemPath> { + walk_roots + .iter() + .filter(|walk_root| path.starts_with(walk_root)) + .max_by_key(|walk_root| walk_root.components().count()) + .map(SystemPathBuf::as_path) +} + +#[cfg(test)] +mod tests { + use ruff_db::system::{SystemPath, SystemPathBuf, TestSystem}; + + use super::{BUILD_MANIFEST, is_build_manifest, is_within_build_output}; + + fn system_with(files: &[&str]) -> TestSystem { + let system = TestSystem::default(); + for file in files { + system + .memory_file_system() + .write_file_all(file, "") + .expect("an in-memory write succeeds"); + } + system + } + + fn roots(paths: &[&str]) -> Vec { + paths + .iter() + .map(|path| SystemPathBuf::from(*path)) + .collect() + } + + /// `--out` can name any directory, and a directory the author calls `build` is theirs until + /// a build writes a manifest into it + #[test] + fn an_output_is_recognised_by_its_manifest_not_its_name() { + let system = system_with(&["/p/anything/.by-manifest", "/p/build/tool.py"]); + let walk_roots = roots(&["/p"]); + assert!(is_within_build_output( + &system, + SystemPath::new("/p/anything/a.py"), + &walk_roots + )); + assert!( + is_within_build_output( + &system, + SystemPath::new("/p/anything/pkg/a.py"), + &walk_roots + ), + "a file deeper inside the output is inside it too" + ); + assert!(!is_within_build_output( + &system, + SystemPath::new("/p/build/tool.py"), + &walk_roots + )); + } + + /// a manifest left at the root would otherwise empty the project of every file, and a + /// directory named on the command line is checked because it was asked for + #[test] + fn a_walk_root_is_never_itself_an_output() { + let system = system_with(&["/p/.by-manifest", "/p/build/.by-manifest"]); + assert!(!is_within_build_output( + &system, + SystemPath::new("/p/a.py"), + &roots(&["/p"]) + )); + assert!(!is_within_build_output( + &system, + SystemPath::new("/p/build/a.py"), + &roots(&["/p/build"]) + )); + assert!(!is_build_manifest( + &SystemPath::new("/p").join(BUILD_MANIFEST), + &roots(&["/p"]) + )); + assert!(is_build_manifest( + &SystemPath::new("/p/build").join(BUILD_MANIFEST), + &roots(&["/p"]) + )); + + // named inside another walk root, it is still a root of its own: the walk from it keeps + // what the walk from the root above skips + let nested = roots(&["/p", "/p/build"]); + assert!(!is_within_build_output( + &system, + SystemPath::new("/p/build/a.py"), + &nested + )); + assert!(!is_build_manifest( + &SystemPath::new("/p/build").join(BUILD_MANIFEST), + &nested + )); + } +} diff --git a/crates/ty_project/src/db/changes.rs b/crates/ty_project/src/db/changes.rs index 0c1dc7ed33..dc55a9760b 100644 --- a/crates/ty_project/src/db/changes.rs +++ b/crates/ty_project/src/db/changes.rs @@ -1,3 +1,4 @@ +use crate::build_output::{is_build_manifest, is_within_build_output}; use crate::db::{Db, ProjectDatabase}; use crate::metadata::CONFIG_FILE_NAMES; use crate::script::script_tag; @@ -134,9 +135,13 @@ impl ProjectDatabase { // over to the `ignore` crate, since the `ignore` crate will respect // the settings provided in `create_walker`. ---AG let respect_ignore_files = project.settings(self).src().respect_ignore_files; - let ignore_walk_roots = - respect_ignore_files.then(|| project.included_paths_or_root(self).to_vec()); - let mut ignore_files = ignore_walk_roots.as_deref().and_then(|walk_roots| { + // also what decides what the walk would skip as a build's output, for one changed path at + // a time. a build writes a copy of every source it read, and none of it is the project's: + // the writes are synced like any file's, but they neither join the project nor set off a + // walk + let walk_roots = project.included_paths_or_root(self).to_vec(); + let ignore_walk_roots = respect_ignore_files.then_some(walk_roots.as_slice()); + let mut ignore_files = ignore_walk_roots.and_then(|walk_roots| { Some(create_walker_builder(self, walk_roots)?.incremental_matcher()) }); @@ -151,6 +156,37 @@ impl ProjectDatabase { continue; } + // a build's manifest appearing or going away turns the directory into an output or + // back into part of the project, as an ignore file saying `*` would. walking it + // again settles which: a walk skips an output. a manifest a build only rewrites + // changes nothing + if is_build_manifest(path, &walk_roots) + && matches!( + change, + ChangeEvent::Created { .. } | ChangeEvent::Deleted { .. } + ) + { + File::sync_path(self, path); + file_system_changed = true; + if let Some(directory) = path.parent() + && !is_within_build_output(self.system(), directory, &walk_roots) + && project.is_directory_included(self, directory) + { + tracing::debug!( + manifest = %path, + directory = %directory, + "Queueing project-file reindex for a build output's manifest" + ); + removed_paths.insert(directory.to_path_buf()); + result.changed_files.mark_unknown(); + if self.system().is_directory(directory) { + added_paths.insert(directory.to_path_buf()); + } + } + + continue; + } + if is_ignore_file(path) && project.settings(self).src().respect_ignore_files { File::sync_path(self, path); if let Some(directory) = path.parent() { @@ -166,6 +202,7 @@ impl ProjectDatabase { ); reload_project_files = true; } else if project.is_directory_included(self, directory) + && !is_within_build_output(self.system(), path, &walk_roots) && ignore_files.as_mut().is_none_or(|ignore_files| { !ignore_files.is_ignored(directory, true) }) @@ -242,6 +279,7 @@ impl ProjectDatabase { if !project .is_file_included(self, path) .should_index_file(self.system(), path) + || is_within_build_output(self.system(), path, &walk_roots) { continue; } @@ -271,6 +309,7 @@ impl ProjectDatabase { } } else if change.is_created() && project.is_directory_included(self, path) + && !is_within_build_output(self.system(), path, &walk_roots) && ignore_files .as_mut() .is_none_or(|ignore_files| !ignore_files.is_ignored(path, true)) diff --git a/crates/ty_project/src/lib.rs b/crates/ty_project/src/lib.rs index 3665397971..b22314645b 100644 --- a/crates/ty_project/src/lib.rs +++ b/crates/ty_project/src/lib.rs @@ -37,6 +37,7 @@ use ty_python_semantic::lint::RuleSelection; use uv::DependencyMetadataError; pub use uv::{ScriptEnvironmentAvailability, UseUv, UvEnvironments, UvSyncChanges}; +pub mod build_output; mod db; mod files; pub mod glob; diff --git a/crates/ty_project/src/walk.rs b/crates/ty_project/src/walk.rs index 4b6d224b9e..85e7d6299c 100644 --- a/crates/ty_project/src/walk.rs +++ b/crates/ty_project/src/walk.rs @@ -1,3 +1,4 @@ +use crate::build_output::is_build_output; use crate::files::IndexedFile; use crate::glob::IncludeExcludeFilter; use crate::script::script_tag; @@ -78,8 +79,8 @@ impl<'a> ProjectFilesFilter<'a> { /// ## Note /// /// This method may return `true` for files that don't end up being included when walking the - /// project tree because it doesn't consider `.gitignore` and other ignore files when deciding - /// if a file's included. + /// project tree because it doesn't consider `.gitignore` and other ignore files, or the build + /// outputs the walk skips (see [`crate::build_output`]), when deciding if a file's included. pub(crate) fn is_file_included( &self, path: &SystemPath, @@ -210,6 +211,21 @@ impl ProjectFilesWalker { GlobFilterCheckMode::TopDown, ); return match directory_included { + // a tree a build wrote is a copy of the project, not part + // of it. a walk root is never skipped for it: the project + // root is never an output, and a directory named on the + // command line was asked for. see `crate::build_output` + IncludeResult::Included { .. } + if entry.depth() > 0 + && is_build_output(db.system(), entry.path()) => + { + tracing::debug!( + "Skipping directory `{path}` because a build \ + wrote it", + path = entry.path() + ); + WalkState::Skip + } IncludeResult::Included { .. } => WalkState::Continue, IncludeResult::Excluded => { tracing::debug!( diff --git a/crates/ty_server/tests/e2e/build_trees.rs b/crates/ty_server/tests/e2e/build_trees.rs new file mode 100644 index 0000000000..902f164b51 --- /dev/null +++ b/crates/ty_server/tests/e2e/build_trees.rs @@ -0,0 +1,139 @@ +//! a tree `by build` wrote is not part of the project it was built from +//! +//! the output is a copy of the project (a `.py` for every `.by`, every hand-written `.py` carried +//! over), so checking it with the workspace would check everything twice and report every +//! diagnostic a second time, against a file nobody edits. it is recognised by the manifest the +//! build leaves at its top, wherever the build wrote it, and not by its name + +use anyhow::Result; +use lsp_types::{WorkspaceDiagnosticReport, WorkspaceDocumentDiagnosticReport}; +use ruff_db::system::SystemPath; +use ty_server::{ClientOptions, DiagnosticMode}; + +use crate::{TestServer, TestServerBuilder}; + +const ERROR: &str = "x: int = \"\"\n"; +const MANIFEST: &str = "# written by `by build`; delete it and stale output stays\napp.py\n"; + +/// the paths, relative to the workspace, of the files the report has a diagnostic for +fn reported_paths(server: &TestServer, report: &WorkspaceDiagnosticReport) -> Vec { + let root = server.file_path("project"); + let mut paths: Vec = report + .items + .iter() + .filter_map(|item| match item { + WorkspaceDocumentDiagnosticReport::WorkspaceFullDocumentDiagnosticReport(full) + if !full.full_document_diagnostic_report.items.is_empty() => + { + let path = full.uri.to_file_path().ok()?; + let path = SystemPath::from_std_path(&path)?.to_path_buf(); + Some(path.strip_prefix(&root).ok()?.as_str().replace('\\', "/")) + } + _ => None, + }) + .collect(); + paths.sort(); + paths +} + +fn workspace_server(builder: TestServerBuilder) -> Result { + Ok(builder + .with_workspace(SystemPath::new("project"), None)? + .with_initialization_options( + &ClientOptions::default().with_diagnostic_mode(DiagnosticMode::Workspace), + ) + .build() + .wait_until_workspaces_are_initialized()) +} + +/// the default output, `build/`, holds a copy of the module with the error. only the source is +/// reported +#[test] +fn the_workspace_does_not_include_the_build_output() -> Result<()> { + let mut server = workspace_server( + TestServerBuilder::new()? + .with_file("project/app.py", ERROR)? + .with_file("project/build/.by-manifest", MANIFEST)? + .with_file("project/build/app.py", ERROR)?, + )?; + + let report = server.workspace_diagnostic_request(None, None); + assert_eq!(reported_paths(&server, &report), ["app.py"]); + + Ok(()) +} + +/// a build told to write elsewhere is left out wherever it wrote, and a directory of the author's +/// own called `build`, with no manifest in it, stays part of the project +#[test] +fn the_output_is_recognised_by_its_manifest_not_its_name() -> Result<()> { + let mut server = workspace_server( + TestServerBuilder::new()? + .with_file("project/app.py", ERROR)? + .with_file("project/build/tool.py", ERROR)? + .with_file("project/elsewhere/.by-manifest", MANIFEST)? + .with_file("project/elsewhere/app.py", ERROR)?, + )?; + + let report = server.workspace_diagnostic_request(None, None); + assert_eq!( + reported_paths(&server, &report), + ["app.py", "build/tool.py"] + ); + + Ok(()) +} + +/// pulls the workspace's diagnostics each time the server says something changed, until `done` +fn reported_after_change(server: &mut TestServer, done: impl Fn(&[String]) -> bool) -> Vec { + loop { + server.await_diagnostic_refresh(); + let report = server.workspace_diagnostic_request(None, None); + let paths = reported_paths(server, &report); + if done(&paths) { + return paths; + } + } +} + +/// a build run while the server watches: the tree it writes does not join the project, and a +/// directory that becomes an output when a manifest appears in it leaves the project +/// +/// these read real file system events, like the other watching tests +#[test] +fn a_build_written_while_the_server_watches_does_not_join_the_project() -> Result<()> { + let mut server = workspace_server( + TestServerBuilder::new()? + .with_file("project/app.py", ERROR)? + .with_file("project/elsewhere/app.py", ERROR)? + .enable_pull_diagnostics(true) + .enable_workspace_diagnostic_refresh(true) + .watch_file_system(), + )?; + let report = server.workspace_diagnostic_request(None, None); + assert_eq!( + reported_paths(&server, &report), + ["app.py", "elsewhere/app.py"], + "a directory with no manifest is part of the project" + ); + + // the build marks its output before the first file lands in it + server.write_file("project/build/.by-manifest", MANIFEST)?; + server.write_file("project/build/app.py", ERROR)?; + // a write the server has to see after the build's, so that once it is reported the build's + // writes have been taken in too + server.write_file("project/later.py", ERROR)?; + let paths = reported_after_change(&mut server, |paths| { + paths.iter().any(|path| path == "later.py") + }); + assert_eq!(paths, ["app.py", "elsewhere/app.py", "later.py"]); + + // a directory already in the project leaves it when a build claims it + server.write_file("project/elsewhere/.by-manifest", MANIFEST)?; + let paths = reported_after_change(&mut server, |paths| { + !paths.iter().any(|path| path == "elsewhere/app.py") + }); + assert_eq!(paths, ["app.py", "later.py"]); + + Ok(()) +} diff --git a/crates/ty_server/tests/e2e/main.rs b/crates/ty_server/tests/e2e/main.rs index a4cd5c184c..554a27ccf3 100644 --- a/crates/ty_server/tests/e2e/main.rs +++ b/crates/ty_server/tests/e2e/main.rs @@ -29,6 +29,7 @@ mod alignment_groups; mod build_output; +mod build_trees; mod call_hierarchy; mod code_actions; mod commands; From 58e44df86fd79e015625c78bc5102c7f62bc2748 Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Fri, 25 Sep 2026 22:02:08 +1000 Subject: [PATCH 03/11] refresh the client after a batch of file system changes only when it changed the database, so `by build` writing its output, or any write to a file nothing has read, no longer makes the editor pull every open file's diagnostics and inlay hints again for each batch the watcher settles; django template discovery depends on the directories it walked instead of a counter every create and delete bumped, and skips a build's output, and a sync walks up to the first directory whose entries changed, so a file reported alone in a new directory still reaches a listing of the one above --- crates/ruff_db/src/files.rs | 104 ++++++-- crates/ruff_db/src/files/directory.rs | 48 +++- crates/ty/tests/file_watching.rs | 76 +++++- crates/ty_ide/src/django_template/project.rs | 123 +++++++--- crates/ty_project/src/db/changes.rs | 80 +++--- crates/ty_project/src/files.rs | 11 +- crates/ty_project/src/lib.rs | 79 +++--- crates/ty_server/src/server/api/changes.rs | 25 +- crates/ty_server/src/session.rs | 12 +- .../ty_server/tests/e2e/django_templates.rs | 116 +++++++++ crates/ty_server/tests/e2e/file_watching.rs | 231 +++++++++++++++++- crates/ty_server/tests/e2e/goto_definition.rs | 47 ++-- crates/ty_server/tests/e2e/main.rs | 13 + .../tests/e2e/publish_diagnostics.rs | 54 +++- .../ty_server/tests/e2e/script_preparation.rs | 5 +- 15 files changed, 853 insertions(+), 171 deletions(-) diff --git a/crates/ruff_db/src/files.rs b/crates/ruff_db/src/files.rs index 1f8d3ec542..c643eb0e06 100644 --- a/crates/ruff_db/src/files.rs +++ b/crates/ruff_db/src/files.rs @@ -4,7 +4,9 @@ use std::sync::Arc; use std::sync::atomic::{AtomicBool, Ordering}; use dashmap::mapref::entry::Entry; -pub use directory::{DirectoryListing, DirectoryListingError, directory_listing}; +pub use directory::{ + DirectoryListing, DirectoryListingError, directory_listing, system_path_to_directory, +}; pub use file_root::{FileRoot, FileRootKind}; pub use path::FilePath; use ruff_notebook::{Notebook, NotebookError}; @@ -270,7 +272,9 @@ impl Files { /// Refreshing the state of files recursively is expensive. It requires iterating over all known files /// and making system calls to get the latest status of matching files. /// That's why [`File::sync_path`] is preferred if it is known that the path is a file. - pub fn sync_all_recursive(db: &mut dyn Db, paths: I) + /// + /// Returns whether it changed anything in the database. + pub fn sync_all_recursive(db: &mut dyn Db, paths: I) -> bool where P: AsRef, I: IntoIterator, @@ -284,14 +288,10 @@ impl Files { .collect::>(); if paths.is_empty() { - return; + return false; } - let parents = paths - .iter() - .filter_map(|path| path.parent().map(SystemPath::to_path_buf)) - .collect::>(); - + let mut wrote = false; let inner = Arc::clone(&db.files().inner); for entry in inner.system_by_path.iter_mut() { let path = entry.key(); @@ -299,11 +299,25 @@ impl Files { .range(..=path.to_path_buf()) .next_back() .is_some_and(|candidate| path.starts_with(candidate.as_path())) - || parents.contains(path) { - File::sync_system_path(db, path, Some(*entry.value())); + wrote |= File::sync_system_path(db, path, Some(*entry.value())).wrote; } } + + // each of `paths` appeared or went away, which changed its parent's entries the way a + // file's appearing does + for path in &paths { + wrote |= File::touch_parent_directory_after_sync( + db, + path, + SyncPathResult { + status_changed: true, + wrote: false, + }, + ); + } + + wrote } /// Refreshes the state of all known files. @@ -314,12 +328,16 @@ impl Files { /// # Performance /// Refreshing the state of every file is expensive. It requires iterating over all known files and /// issuing a system call to get the latest status of each file. - pub fn sync_all(db: &mut dyn Db) { + /// + /// Returns whether it changed anything in the database. + pub fn sync_all(db: &mut dyn Db) -> bool { tracing::debug!("Syncing all files"); + let mut wrote = false; let inner = Arc::clone(&db.files().inner); for entry in inner.system_by_path.iter_mut() { - File::sync_system_path(db, entry.key(), Some(*entry.value())); + wrote |= File::sync_system_path(db, entry.key(), Some(*entry.value())).wrote; } + wrote } } @@ -395,6 +413,8 @@ impl get_size2::GetSize for File {} struct SyncPathResult { status_changed: bool, + /// whether the sync set any of the file's fields + wrote: bool, } impl File { @@ -453,25 +473,33 @@ impl File { /// /// Directory listings are invalidated if the path's file status changed, its prior status is /// unknown, or if `path` is itself a directory. - pub fn sync_path(db: &mut dyn Db, path: &SystemPath) { + /// + /// Returns whether it changed anything in the database. + pub fn sync_path(db: &mut dyn Db, path: &SystemPath) -> bool { let absolute = SystemPath::absolute(path, db.system().current_directory()); let result = Self::sync_system_path(db, &absolute, None); - Self::touch_parent_directory_after_sync(db, &absolute, result); + let wrote = result.wrote; + Self::touch_parent_directory_after_sync(db, &absolute, result) || wrote } /// Refreshes *only* the file metadata by querying the file system if needed. /// - /// This specifically does not invalidate any directory listings. - pub fn sync_path_only(db: &mut dyn Db, path: &SystemPath) { + /// This specifically does not invalidate any directory listings. Returns whether it changed + /// anything in the database. + pub fn sync_path_only(db: &mut dyn Db, path: &SystemPath) -> bool { let absolute = SystemPath::absolute(path, db.system().current_directory()); - Self::sync_system_path(db, &absolute, None); + Self::sync_system_path(db, &absolute, None).wrote } /// Increments the revision for the virtual file at `path`. - pub fn sync_virtual_path(db: &mut dyn Db, path: &SystemVirtualPath) { - if let Some(virtual_file) = db.files().try_virtual_file(path) { - virtual_file.sync(db); - } + /// + /// Returns whether there is a virtual file at `path`. + pub fn sync_virtual_path(db: &mut dyn Db, path: &SystemVirtualPath) -> bool { + let Some(virtual_file) = db.files().try_virtual_file(path) else { + return false; + }; + virtual_file.sync(db); + true } /// Syncs the [`File`]'s state with the state of the file on the system. @@ -498,6 +526,7 @@ impl File { let Some(file) = file.or_else(|| db.files().try_system(db, path)) else { return SyncPathResult { status_changed: true, + wrote: false, }; }; @@ -532,28 +561,49 @@ impl File { clear_override = true; } + let mut permissions_changed = false; if file.permissions(db) != permission { tracing::debug!("Updating the permissions of `{}`", file.path(db)); file.set_permissions(db).to(permission); + permissions_changed = true; } if clear_override && file.source_text_override(db).is_some() { file.set_source_text_override(db).to(None); } - SyncPathResult { status_changed } + SyncPathResult { + status_changed, + wrote: clear_override || permissions_changed, + } } + /// syncs the directories whose entries a sync of `path` shows to have changed, and returns + /// whether that changed anything in the database + /// + /// a path that appeared or went away changed its parent's entries. so did the parent, if it + /// appeared or went away with it (a file written into a directory made for it), or if + /// nothing had read it, when nothing can say it did not. up from there, to the first + /// directory that was there before and after: the only one whose entries changed without + /// its own existence changing + /// + /// all the way, and not only to the parent, because a change is often reported for the + /// path alone: a client watching `**/*.py` says nothing of the directories a new file + /// brought with it, and a query that listed the directory above them would never be told fn touch_parent_directory_after_sync( db: &mut dyn Db, - path: &SystemPath, - result: SyncPathResult, - ) { - if result.status_changed + mut path: &SystemPath, + mut result: SyncPathResult, + ) -> bool { + let mut wrote = false; + while result.status_changed && let Some(parent) = path.parent() { - Self::sync_system_path(db, parent, None); + result = Self::sync_system_path(db, parent, None); + wrote |= result.wrote; + path = parent; } + wrote } /// Returns `true` if the file exists. diff --git a/crates/ruff_db/src/files/directory.rs b/crates/ruff_db/src/files/directory.rs index ba2f445a6d..fde306708e 100644 --- a/crates/ruff_db/src/files/directory.rs +++ b/crates/ruff_db/src/files/directory.rs @@ -136,10 +136,54 @@ fn directory_listing_query( #[cfg(test)] mod tests { - use crate::files::directory_listing; - use crate::system::{DbWithWritableSystem as _, SystemPath}; + use crate::files::{File, Files, directory_listing}; + use crate::system::{DbWithWritableSystem as _, SystemPath, WritableSystem as _}; use crate::tests::TestDb; + fn names(db: &TestDb, directory: &str) -> Vec { + directory_listing(db, SystemPath::new(directory)) + .unwrap() + .iter() + .map(|(name, _)| name.to_string()) + .collect() + } + + /// a change is often reported for a file alone, as a client watching `**/*.py` reports it, + /// and not for the directory that was made for it: the directory above the new one is still + /// the one whose entries changed + #[test] + fn a_file_written_into_a_new_directory_changes_the_listing_above_it() -> std::io::Result<()> { + let mut db = TestDb::new(); + db.write_file("src/a.py", "")?; + assert_eq!(names(&db, "src"), ["a.py"]); + + let file = SystemPath::new("src/pkg/module.py"); + db.writable_system() + .create_directory_all(SystemPath::new("src/pkg"))?; + db.writable_system().write_file(file, "")?; + File::sync_path(&mut db, file); + + assert_eq!(names(&db, "src"), ["a.py", "pkg"]); + + Ok(()) + } + + /// the same holds of a directory reported alone, without the new directory it was made in + #[test] + fn a_directory_made_in_a_new_directory_changes_the_listing_above_it() -> std::io::Result<()> { + let mut db = TestDb::new(); + db.write_file("src/a.py", "")?; + assert_eq!(names(&db, "src"), ["a.py"]); + + let directory = SystemPath::new("src/pkg/sub"); + db.writable_system().create_directory_all(directory)?; + Files::sync_all_recursive(&mut db, [directory]); + + assert_eq!(names(&db, "src"), ["a.py", "pkg"]); + + Ok(()) + } + #[test] fn listing_is_sorted() -> std::io::Result<()> { let mut db = TestDb::new(); diff --git a/crates/ty/tests/file_watching.rs b/crates/ty/tests/file_watching.rs index fa3ddc6afb..7df019b28b 100644 --- a/crates/ty/tests/file_watching.rs +++ b/crates/ty/tests/file_watching.rs @@ -1,5 +1,7 @@ use std::collections::HashSet; use std::io::Write; +#[cfg(unix)] +use std::os::unix::fs::PermissionsExt as _; use std::time::{Duration, Instant}; use anyhow::{Context, anyhow}; @@ -18,7 +20,9 @@ use ty_project::metadata::options::{EnvironmentOptions, Options, SrcOptions}; use ty_project::metadata::pyproject::{PyProject, Tool}; use ty_project::metadata::python_version::SupportedPythonVersion; use ty_project::metadata::value::{RelativeGlobPattern, RelativePathBuf}; -use ty_project::watch::{ChangeEvent, CreatedKind, DeletedKind, ProjectWatcher, directory_watcher}; +use ty_project::watch::{ + ChangeEvent, ChangedKind, CreatedKind, DeletedKind, ProjectWatcher, directory_watcher, +}; use ty_project::{ChangeResult, Db, ProjectDatabase, ProjectMetadata}; use ty_python_core::platform::PythonPlatform; use ty_static::EnvVars; @@ -975,6 +979,70 @@ fn changed_file() -> anyhow::Result<()> { Ok(()) } +/// a change reported for a file whose content, metadata and place in the project are all as they +/// were changes nothing, however many times it is reported, and a real change to the file does +#[test] +fn a_change_that_changes_nothing_leaves_the_database_as_it_was() -> anyhow::Result<()> { + let mut case = setup([("foo.py", "x = 1\n")])?; + let foo_path = case.project_path("foo.py"); + let foo = case.system_file(&foo_path)?; + case.assert_indexed_project_files([foo]); + + let unchanged = [ChangeEvent::Changed { + path: foo_path.clone(), + kind: ChangedKind::Any, + }]; + // applying a batch cancels the database's other readers, and salsa counts cancellations in a + // `u8`, starting a new revision with no write in it when the count wraps: enough batches for + // it to wrap several times + for batch in 0..1024 { + let result = case.apply_changes(&unchanged); + assert!( + !result.database_changed(), + "batch {batch} changed the database" + ); + } + + update_file(&foo_path, "x = 2\n")?; + assert!(case.apply_changes(&unchanged).database_changed()); + + Ok(()) +} + +/// a batch whose only effect is to clear a diagnostic the walk reported still changes what a +/// check of the project says +#[cfg(unix)] +#[test] +fn clearing_a_walk_diagnostic_changes_the_database() -> anyhow::Result<()> { + let mut case = setup([("foo.py", ""), ("locked/bar.py", "")])?; + let foo_path = case.project_path("foo.py"); + let locked = case.project_path("locked"); + let unreadable = |case: &TestCase| { + case.db() + .check() + .iter() + .any(|diagnostic| diagnostic.concise_message().to_string().contains("locked")) + }; + + std::fs::set_permissions(locked.as_std_path(), std::fs::Permissions::from_mode(0o000))?; + let foo = case.system_file(&foo_path)?; + case.assert_indexed_project_files([foo]); + std::fs::set_permissions(locked.as_std_path(), std::fs::Permissions::from_mode(0o755))?; + assert!( + unreadable(&case), + "the walk reports the directory it could not read" + ); + + let result = case.apply_changes(&[ChangeEvent::Changed { + path: foo_path, + kind: ChangedKind::Any, + }]); + assert!(result.database_changed()); + assert!(!unreadable(&case)); + + Ok(()) +} + /// a manifest written into a directory of the project makes the directory a build's output, /// which is not part of the project #[test] @@ -986,11 +1054,12 @@ fn a_build_manifest_created_takes_its_directory_out_of_the_project() -> anyhow:: let manifest = case.project_path("build/.by-manifest"); std::fs::write(manifest.as_std_path(), "app.py\n")?; - case.apply_changes(&[ChangeEvent::Created { + let result = case.apply_changes(&[ChangeEvent::Created { path: manifest, kind: CreatedKind::File, }]); + assert!(result.database_changed()); case.assert_indexed_project_files([app]); Ok(()) @@ -1010,11 +1079,12 @@ fn a_build_manifest_deleted_returns_its_directory_to_the_project() -> anyhow::Re let manifest = case.project_path("build/.by-manifest"); std::fs::remove_file(manifest.as_std_path())?; - case.apply_changes(&[ChangeEvent::Deleted { + let result = case.apply_changes(&[ChangeEvent::Deleted { path: manifest, kind: DeletedKind::File, }]); + assert!(result.database_changed()); let built = case.system_file(case.project_path("build/app.py"))?; case.assert_indexed_project_files([app, built]); diff --git a/crates/ty_ide/src/django_template/project.rs b/crates/ty_ide/src/django_template/project.rs index 2c471f40cc..6bccf69ff6 100644 --- a/crates/ty_ide/src/django_template/project.rs +++ b/crates/ty_ide/src/django_template/project.rs @@ -12,10 +12,10 @@ //! type checker has already populated for every project file, the ast work is //! shared rather than duplicated. -use std::sync::Mutex; +use std::sync::{Mutex, PoisonError}; use compact_str::{CompactString, ToCompactString}; -use ruff_db::files::{File, FilePath, system_path_to_file}; +use ruff_db::files::{File, FilePath, system_path_to_directory, system_path_to_file}; use ruff_db::parsed::parsed_module; use ruff_db::source::source_text; use ruff_db::system::walk_directory::WalkState; @@ -28,6 +28,7 @@ use ty_module_resolver::ImportingFile; use ty_module_resolver::{ Module, ModuleName, file_to_module, resolve_module_confident, resolve_real_module, }; +use ty_project::build_output::is_build_output; use ty_project::{Db, Project}; use ty_python_core::definition::DefinitionKind; use ty_python_semantic::ProgramEnvironment; @@ -1634,11 +1635,6 @@ fn discover( directory: &str, order: &SearchOrder, ) -> Box<[DiscoveredFile]> { - // walking the file system is invisible to salsa, so the callers' queries would - // hand back their first answer forever. reading the revision the project bumps - // on every create and delete is what makes a template added mid-session show up - let _ = project.file_system_revision(db); - let mut found = discover_by_convention(db, project, directory, order); // a directory the settings name is under no obligation to be called @@ -1692,13 +1688,18 @@ fn discover_by_convention( ) -> Vec { let root = project.root(db); let found = Mutex::new(Vec::new()); + let listed = Mutex::new(Vec::new()); db.system() .walk_directory(root) .standard_filters(project.settings(db).src().respect_ignore_files) .ignore_hidden(true) .run(|| { - Box::new(|entry| { + // a walk visits from threads of its own, which each need a handle on the database + let db = Db::dyn_clone(db); + let found = &found; + let listed = &listed; + Box::new(move |entry| { let Ok(entry) = entry else { return WalkState::Continue; }; @@ -1715,6 +1716,14 @@ fn discover_by_convention( if !inside && entry.depth() >= DISCOVERY_DEPTH { return WalkState::Skip; } + list(&*db, listed, entry.path()); + // a build's output is a copy of the project, templates and all, and + // not the project's own (see `ty_project::build_output`). its manifest + // is one of the entries of the directory just listed, so writing or + // deleting it is a change to what the walk read + if entry.depth() > 0 && is_build_output(db.system(), entry.path()) { + return WalkState::Skip; + } WalkState::Continue } FileType::File => { @@ -1722,15 +1731,17 @@ fn discover_by_convention( && let Some((under, name)) = relative_to_directory(entry.path(), directory) { - found.lock().unwrap().push(DiscoveredFile { - name, - path: entry.path().to_path_buf(), - precedence: order.rank(under), - // the walk starts at the project root and - // respects its ignore rules, so whatever it - // reaches is the project's own - own: true, - }); + found.lock().unwrap_or_else(PoisonError::into_inner).push( + DiscoveredFile { + name, + path: entry.path().to_path_buf(), + precedence: order.rank(under), + // the walk starts at the project root and + // respects its ignore rules, so whatever it + // reaches is the project's own + own: true, + }, + ); } WalkState::Continue } @@ -1739,9 +1750,45 @@ fn discover_by_convention( }) }); - found - .into_inner() - .unwrap_or_else(std::sync::PoisonError::into_inner) + depend_on_entries( + db, + listed.into_inner().unwrap_or_else(PoisonError::into_inner), + ); + + found.into_inner().unwrap_or_else(PoisonError::into_inner) +} + +/// note that a walk is about to read the entries of `directory`, for [`depend_on_entries`] +/// +/// the database reads a directory's revision the first time it is asked about the directory, +/// and it is asked here, before the walk reads the entries. a revision first read after them +/// could already take in an entry created in between that the walk never saw, and the change +/// the watcher then reports would find the revision as it was and leave the answer missing it +fn list(db: &dyn Db, listed: &Mutex>, directory: &SystemPath) { + let _ = system_path_to_directory(db, directory); + listed + .lock() + .unwrap_or_else(PoisonError::into_inner) + .push(directory.to_path_buf()); +} + +/// make the calling query depend on which entries each of `directories` holds +/// +/// walking the file system is invisible to salsa, so a query that walks would hand back its +/// first answer forever. a directory's revision changes when an entry is created in it, deleted +/// from it or renamed, and the file watcher's changes sync the directory holding each path they +/// name, so depending on the revision of every directory a walk read, and on whether a root it +/// was asked to walk exists, is what makes a template added mid-session show up. and only that: +/// a write anywhere the walk did not go, inside a build's output or an ignored directory, +/// changes nothing the query read +fn depend_on_entries(db: &dyn Db, directories: Vec) { + for directory in directories { + // an `Err` still read the path's status: a root that is not there yet is depended on + // being absent + if let Ok(directory) = system_path_to_directory(db, &directory) { + let _ = directory.revision(db); + } + } } /// add every file under `root` to `found`, named the way django would name it @@ -1754,36 +1801,56 @@ fn collect_under( found: &mut Vec, ) { let collected = Mutex::new(Vec::new()); + let listed = Mutex::new(Vec::new()); + // before the walk, and whether or not it is there: a root that does not exist yet is + // depended on being absent + list(db, &listed, root); db.system() .walk_directory(root) .standard_filters(project.settings(db).src().respect_ignore_files) .ignore_hidden(true) .run(|| { - Box::new(|entry| { + // a walk visits from threads of its own, which each need a handle on the database + let db = Db::dyn_clone(db); + let collected = &collected; + let listed = &listed; + Box::new(move |entry| { let Ok(entry) = entry else { return WalkState::Continue; }; + if entry.file_type().is_directory() && entry.depth() > 0 { + list(&*db, listed, entry.path()); + } + if matches!(entry.file_type(), FileType::File) && let Some(name) = relative_to_root(entry.path(), root) { - collected.lock().unwrap().push(DiscoveredFile { - name, - path: entry.path().to_path_buf(), - precedence, - own, - }); + collected + .lock() + .unwrap_or_else(PoisonError::into_inner) + .push(DiscoveredFile { + name, + path: entry.path().to_path_buf(), + precedence, + own, + }); } WalkState::Continue }) }); + depend_on_entries( + db, + listed.into_inner().unwrap_or_else(PoisonError::into_inner), + ); + found.extend( collected .into_inner() - .unwrap_or_else(std::sync::PoisonError::into_inner), + .unwrap_or_else(PoisonError::into_inner), ); } diff --git a/crates/ty_project/src/db/changes.rs b/crates/ty_project/src/db/changes.rs index dc55a9760b..519c7e31cd 100644 --- a/crates/ty_project/src/db/changes.rs +++ b/crates/ty_project/src/db/changes.rs @@ -11,7 +11,6 @@ use ruff_db::Db as _; use ruff_db::files::{File, Files, system_path_to_file}; use ruff_db::system::{SystemPath, SystemPathBuf}; use rustc_hash::FxHashSet; -use salsa::Setter as _; use ty_python_core::program::FallibleStrategy; /// Represents the result of applying changes to the project database. @@ -20,6 +19,7 @@ pub struct ChangeResult { project_sync_path: Option, custom_stdlib_changed: bool, changed_files: ChangedFiles, + database_changed: bool, } impl ChangeResult { @@ -35,6 +35,16 @@ impl ChangeResult { self.project_sync_path.as_deref() } + /// whether applying the changes changed anything in the database + /// + /// when it did not, every query answers as it did before the changes, so nothing a client + /// has been shown can be out of date. most file system events are like that: a write to a + /// file nothing has read, to a directory nothing has listed, to a build's output or to an + /// ignored directory + pub fn database_changed(&self) -> bool { + self.database_changed + } + /// Returns `true` if the custom stdlib's VERSIONS file has changed. pub fn custom_stdlib_changed(&self) -> bool { self.custom_stdlib_changed @@ -116,6 +126,8 @@ impl ProjectDatabase { } else { ChangedFiles::Known(FxHashSet::default()) }, + // every write below reports whether it set anything, and each one that did sets this + database_changed: false, }; // Paths whose project files should be discovered incrementally. let mut added_paths = BTreeSet::default(); @@ -128,9 +140,6 @@ impl ProjectDatabase { let mut removed_paths = BTreeSet::default(); let mut reload_project = false; let mut reload_project_files = false; - // Whether the shape of the tree changed, as opposed to the contents of a - // file that was already there. See `Project::file_system_revision`. - let mut file_system_changed = false; // TODO: This should be removed once the incremental checker is ported // over to the `ignore` crate, since the `ignore` crate will respect // the settings provided in `create_walker`. ---AG @@ -150,7 +159,7 @@ impl ProjectDatabase { if let Some(path) = change.system_path() { if configuration_paths.is_configuration(path, &project_root) { - File::sync_path(self, path); + result.database_changed |= File::sync_path(self, path); reload_project = true; continue; @@ -166,8 +175,7 @@ impl ProjectDatabase { ChangeEvent::Created { .. } | ChangeEvent::Deleted { .. } ) { - File::sync_path(self, path); - file_system_changed = true; + result.database_changed |= File::sync_path(self, path); if let Some(directory) = path.parent() && !is_within_build_output(self.system(), directory, &walk_roots) && project.is_directory_included(self, directory) @@ -188,7 +196,7 @@ impl ProjectDatabase { } if is_ignore_file(path) && project.settings(self).src().respect_ignore_files { - File::sync_path(self, path); + result.database_changed |= File::sync_path(self, path); if let Some(directory) = path.parent() { if project .included_paths_or_root(self) @@ -240,14 +248,10 @@ impl ProjectDatabase { ChangeEvent::Changed { path, .. } | ChangeEvent::Opened(path) | ChangeEvent::Created { path, .. } => { - if matches!(change, ChangeEvent::Created { .. }) { - file_system_changed = true; - } - match change { ChangeEvent::Changed { .. } => { if synced_files.insert(path.to_path_buf()) { - File::sync_path_only(self, path); + result.database_changed |= File::sync_path_only(self, path); } } ChangeEvent::Opened(_) @@ -256,7 +260,7 @@ impl ProjectDatabase { .. } => { if synced_files.insert(path.to_path_buf()) { - File::sync_path(self, path); + result.database_changed |= File::sync_path(self, path); } } _ => { @@ -296,9 +300,10 @@ impl ProjectDatabase { && !project.is_file_explicitly_included(self, file); if exclude_script { - project.remove_file(self, file); + result.database_changed |= project.remove_file(self, file); } else { - project.add_file(self, file, is_script); + result.database_changed |= + project.add_file(self, file, is_script); } if let ChangedFiles::Known(changed_files) = @@ -323,8 +328,6 @@ impl ProjectDatabase { } ChangeEvent::Deleted { kind, path } => { - file_system_changed = true; - let is_file = match kind { DeletedKind::File => true, DeletedKind::Directory => false, @@ -336,11 +339,11 @@ impl ProjectDatabase { if is_file { if synced_files.insert(path.to_path_buf()) { - File::sync_path(self, path); + result.database_changed |= File::sync_path(self, path); } if let Some(file) = self.files().try_system(self, path) { - project.remove_file(self, file); + result.database_changed |= project.remove_file(self, file); } } else { sync_recursively.insert(path.clone()); @@ -364,20 +367,20 @@ impl ProjectDatabase { } ChangeEvent::CreatedVirtual(path) | ChangeEvent::ChangedVirtual(path) => { - File::sync_virtual_path(self, path); + result.database_changed |= File::sync_virtual_path(self, path); } ChangeEvent::DeletedVirtual(path) => { if let Some(virtual_file) = self.files().try_virtual_file(path) { virtual_file.close(self); + result.database_changed = true; } } ChangeEvent::Rescan => { - file_system_changed = true; reload_project = true; reload_project_files = true; - Files::sync_all(self); + result.database_changed |= Files::sync_all(self); sync_recursively.clear(); removed_paths.clear(); break; @@ -385,15 +388,7 @@ impl ProjectDatabase { } } - Files::sync_all_recursive(self, sync_recursively); - - // Bumped before the reload paths below, some of which return early. - if file_system_changed { - let revision = project.file_system_revision(self); - project - .set_file_system_revision(self) - .to(revision.wrapping_add(1)); - } + result.database_changed |= Files::sync_all_recursive(self, sync_recursively); if reload_project { // The active project root may have been deleted. Start rediscovery from the closest @@ -411,8 +406,12 @@ impl ProjectDatabase { // We're not refreshing uv metadata, so use the existing environment. let environment = metadata.environment().clone(); match project.rediscover(self, path, environment) { - Ok(ProjectReloadResult::Unchanged) => {} + // a rediscovery that leaves the metadata and settings as they were can still + // replace the program settings or the settings diagnostics, and does not say + // whether it did, so it is taken to have + Ok(ProjectReloadResult::Unchanged) => result.database_changed = true, Ok(ProjectReloadResult::Changed { files_changed }) => { + result.database_changed = true; result.project_changed = true; result.changed_files.mark_unknown(); if files_changed { @@ -427,7 +426,7 @@ impl ProjectDatabase { "Failed to load project, keeping old project configuration: {error:#}" ); if reload_project_files { - project.reload_files(self); + result.database_changed |= project.reload_files(self); result.changed_files.mark_unknown(); return result; } @@ -437,7 +436,7 @@ impl ProjectDatabase { } if reload_project_files { - project.reload_files(self); + result.database_changed |= project.reload_files(self); result.changed_files.mark_unknown(); // A full project-file reload supersedes incremental project-file updates. added_paths.clear(); @@ -458,13 +457,14 @@ impl ProjectDatabase { Ok((_, diagnostics)) => diagnostics, Err(error) => vec![error.into_diagnostic()], }; - project.update_program(self, program_settings); + result.database_changed |= project.update_program(self, program_settings); settings_diagnostics.extend( program_settings_diagnostics .into_iter() .map(|diagnostic| diagnostic.into_diagnostic(self)), ); - project.update_settings_diagnostics(self, settings_diagnostics); + result.database_changed |= + project.update_settings_diagnostics(self, settings_diagnostics); } Err(error) => { tracing::error!("Failed to resolve program settings: {error}"); @@ -472,7 +472,7 @@ impl ProjectDatabase { } } - project.remove_files_under(self, removed_paths); + result.database_changed |= project.remove_files_under(self, removed_paths); let diagnostics = if !project.file_set(self).is_lazy() { // Use directory walking to discover newly added files. @@ -480,7 +480,7 @@ impl ProjectDatabase { let (files, diagnostics) = walker.collect_vec(self); for file in files { - project.add_file(self, file.file, file.is_script); + result.database_changed |= project.add_file(self, file.file, file.is_script); } diagnostics @@ -493,7 +493,7 @@ impl ProjectDatabase { // across revisions doesn't feel essential, considering that they're rare. However, we could // implement a `BTreeMap` or similar and only prune the diagnostics from paths that we've // re-scanned (or that were removed etc). - project.replace_index_diagnostics(self, diagnostics); + result.database_changed |= project.replace_index_diagnostics(self, diagnostics); result } diff --git a/crates/ty_project/src/files.rs b/crates/ty_project/src/files.rs index 5638ac0450..8577a92773 100644 --- a/crates/ty_project/src/files.rs +++ b/crates/ty_project/src/files.rs @@ -266,7 +266,16 @@ impl IndexedMut<'_> { } pub(super) fn set_diagnostics(&mut self, diagnostics: Vec) { - self.inner_mut().diagnostics = diagnostics; + if self.indexed.diagnostics != diagnostics { + self.inner_mut().diagnostics = diagnostics; + self.did_change = true; + } + } + + /// whether anything this view was asked to do changed the index, which it then writes back + /// to the database when it is dropped + pub(super) fn did_change(&self) -> bool { + self.did_change } fn inner_mut(&mut self) -> &mut IndexedInner { diff --git a/crates/ty_project/src/lib.rs b/crates/ty_project/src/lib.rs index b22314645b..8f3cd91631 100644 --- a/crates/ty_project/src/lib.rs +++ b/crates/ty_project/src/lib.rs @@ -136,17 +136,6 @@ pub struct Project { #[default] #[returns(copy)] force_exclude_flag: bool, - - /// A counter bumped whenever a file or directory is created or deleted. - /// - /// The project's file set holds Python files only, so nothing in the database - /// changes when a file of any other kind appears or disappears. A query that - /// discovers such files by walking the file system — the template and static - /// asset discovery the Django template services do — has nothing else to - /// depend on, and would otherwise serve its first answer forever. - #[default] - #[returns(copy)] - pub file_system_revision: u64, } /// A checker for the part of a project the type checker does not read. @@ -294,7 +283,6 @@ impl Project { .durability(Durability::MEDIUM) .open_fileset_durability(Durability::LOW) .file_set_durability(Durability::LOW) - .file_system_revision_durability(Durability::LOW) .new(db) } @@ -389,11 +377,14 @@ impl Project { .map(|metadata| Some(Box::new(metadata))) } - pub fn update_program(self, db: &mut dyn Db, settings: ProgramSettings) { - if self.program_settings(db) != &settings { - settings.search_paths.try_register_static_roots(db); - self.set_program_settings(db).to(settings); + /// returns whether the settings differ from the ones the program had + pub fn update_program(self, db: &mut dyn Db, settings: ProgramSettings) -> bool { + if self.program_settings(db) == &settings { + return false; } + settings.search_paths.try_register_static_roots(db); + self.set_program_settings(db).to(settings); + true } pub fn root(self, db: &dyn Db) -> &SystemPath { @@ -539,15 +530,17 @@ impl Project { /// Replace stored settings diagnostics after recomputing program settings. /// /// This is used when a change affects [`ty_python_core::program::ProgramSettings`] without - /// reloading the full project. + /// reloading the full project. Returns whether the diagnostics changed. fn update_settings_diagnostics( self, db: &mut dyn Db, settings_diagnostics: Vec, - ) { - if self.settings_diagnostics(db) != settings_diagnostics { - self.set_settings_diagnostics(db).to(settings_diagnostics); + ) -> bool { + if self.settings_diagnostics(db) == settings_diagnostics { + return false; } + self.set_settings_diagnostics(db).to(settings_diagnostics); + true } /// Checks the project and its dependencies according to the project's check mode. @@ -737,8 +730,9 @@ impl Project { self.set_open_fileset(db).to(FxHashSet::default()) } + /// returns whether the file was one of the project's #[tracing::instrument(level = "debug", skip(self, db))] - pub fn remove_file(self, db: &mut dyn Db, file: File) { + pub fn remove_file(self, db: &mut dyn Db, file: File) -> bool { tracing::debug!( "Removing file `{}` from project `{}`", file.path(db), @@ -746,17 +740,17 @@ impl Project { ); let Some(mut index) = IndexedFiles::indexed_mut(db, self) else { - return; + return false; }; - index.remove(file); + index.remove(file) } - /// Removes all indexed project files under `paths`. + /// Removes all indexed project files under `paths`, and returns whether there were any. /// /// This is a no-op if the project files are still lazily indexed. #[tracing::instrument(level = "debug", skip(self, db, paths))] - fn remove_files_under(self, db: &mut dyn Db, paths: I) + fn remove_files_under(self, db: &mut dyn Db, paths: I) -> bool where I: IntoIterator, P: AsRef, @@ -769,11 +763,11 @@ impl Project { .collect::>(); if paths.is_empty() { - return; + return false; } if self.file_set(db).is_lazy() { - return; + return false; } let files_to_remove = { @@ -792,19 +786,22 @@ impl Project { }; if files_to_remove.is_empty() { - return; + return false; } let Some(mut index) = IndexedFiles::indexed_mut(db, self) else { - return; + return false; }; for file in files_to_remove { index.remove(file); } + index.did_change() } - fn add_file(self, db: &mut dyn Db, file: File, is_script: bool) { + /// returns whether the file was not already one of the project's, as a script if `is_script` + /// and as a module if not + fn add_file(self, db: &mut dyn Db, file: File, is_script: bool) -> bool { tracing::debug!( "Adding file `{}` to project `{}`", file.path(db), @@ -812,21 +809,24 @@ impl Project { ); let Some(mut index) = IndexedFiles::indexed_mut(db, self) else { - return; + return false; }; index.insert(file, is_script); + index.did_change() } - /// Replaces the diagnostics from indexing the project files with `diagnostics`. + /// Replaces the diagnostics from indexing the project files with `diagnostics`, and returns + /// whether they differ from the ones it had. /// /// This is a no-op if the project files haven't been indexed yet. - fn replace_index_diagnostics(self, db: &mut dyn Db, diagnostics: Vec) { + fn replace_index_diagnostics(self, db: &mut dyn Db, diagnostics: Vec) -> bool { let Some(mut index) = IndexedFiles::indexed_mut(db, self) else { - return; + return false; }; index.set_diagnostics(diagnostics); + index.did_change() } /// Returns whether `file` itself is an explicit check path. @@ -875,13 +875,16 @@ impl Project { } } - fn reload_files(self, db: &mut dyn Db) { + /// returns whether the files had been indexed, and so are indexed again when next asked for + fn reload_files(self, db: &mut dyn Db) -> bool { tracing::debug!("Reloading files for project `{}`", self.name(db)); - if !self.file_set(db).is_lazy() { - // Force a re-index of the files in the next revision. - self.set_file_set(db).to(IndexedFiles::lazy()); + if self.file_set(db).is_lazy() { + return false; } + // Force a re-index of the files in the next revision. + self.set_file_set(db).to(IndexedFiles::lazy()); + true } /// Check if the project's settings have any issues diff --git a/crates/ty_server/src/server/api/changes.rs b/crates/ty_server/src/server/api/changes.rs index 7171454f05..c39e78e65a 100644 --- a/crates/ty_server/src/server/api/changes.rs +++ b/crates/ty_server/src/server/api/changes.rs @@ -15,7 +15,13 @@ use crate::session::Session; use crate::session::client::Client; use crate::system::AnySystemPath; -/// Applies `changes` to every project and refreshes what the client is showing. +/// applies `changes` to every project and, if that changed anything, refreshes what the client is +/// showing +/// +/// only then: a batch of changes is often a write to something no answer depends on (a build's +/// output, an ignored directory, a file nothing imports), and a refresh makes the client ask again +/// for the diagnostics and inlay hints of everything it shows, and poll the workspace's +/// diagnostics again pub(crate) fn apply(session: &mut Session, client: &Client, changes: &[ChangeEvent]) { if changes.is_empty() { return; @@ -28,17 +34,30 @@ pub(crate) fn apply(session: &mut Session, client: &Client, changes: &[ChangeEve .map(|(root, _)| root.clone()) .collect(); + let mut changed = false; for root in roots { tracing::debug!("Applying changes to `{root}`"); - session.apply_changes(client, &AnySystemPath::System(root.clone()), changes); - publish_settings_diagnostics(session, client, root); + let result = session.apply_changes(client, &AnySystemPath::System(root.clone()), changes); + if result.database_changed() { + changed = true; + publish_settings_diagnostics(session, client, root); + } } // a change can move what there is to watch: a new virtual environment, a configuration // naming other search paths session.update_file_watcher(client); + if !changed { + tracing::debug!("The changes changed nothing the session answers from"); + return; + } + + // a workspace diagnostic request waiting for the session to change is answered here rather + // than by a notification's handler: most changes come from the session's own watcher + session.resume_suspended_workspace_diagnostic_request(client); + if client_capabilities.supports_workspace_diagnostic_refresh() { client.send_request::(session, (), |_, ()| {}); } else { diff --git a/crates/ty_server/src/session.rs b/crates/ty_server/src/session.rs index 8dc309d207..886f5eee35 100644 --- a/crates/ty_server/src/session.rs +++ b/crates/ty_server/src/session.rs @@ -660,14 +660,17 @@ impl Session { self.projects.values_mut().next().unwrap() } + /// applies `changes` to the project holding `path` + /// + /// the session's revision moves only if the project's database did: a change that changed + /// nothing leaves every answer as it was, and must not wake what waits for the session to + /// change pub(crate) fn apply_changes( &mut self, client: &Client, path: &AnySystemPath, changes: &[ChangeEvent], ) -> ChangeResult { - self.bump_revision(); - let capabilities = self.resolved_client_capabilities; let script_progress = self.script_progress.clone(); let db = self.project_db_mut(path); @@ -689,6 +692,9 @@ impl Session { } let scripts = result.scripts_to_synchronize(db); Self::synchronize_closed_scripts(db, &scripts, client, capabilities, &script_progress); + if result.database_changed() { + self.bump_revision(); + } result } @@ -2444,6 +2450,8 @@ impl DocumentHandle { }; session.apply_changes(client, path, &changes); + // the document's text changed, whether or not its file in the database did + session.bump_revision(); } fn set_version(&mut self, version: DocumentVersion) { diff --git a/crates/ty_server/tests/e2e/django_templates.rs b/crates/ty_server/tests/e2e/django_templates.rs index 30363882ed..bfc968848e 100644 --- a/crates/ty_server/tests/e2e/django_templates.rs +++ b/crates/ty_server/tests/e2e/django_templates.rs @@ -278,6 +278,122 @@ fn a_template_created_during_the_session_is_offered_to_an_extends() -> Result<() Ok(()) } +/// a client watching for files reports the file alone, and not the directory it was +/// written into along with it: the walk that found the templates listed the directory +/// above the new one, and that is the directory whose entries changed +#[test] +fn a_template_created_in_a_new_directory_is_offered_to_an_extends() -> Result<()> { + let mut server = server("{% extends '' %}\n")?; + let uri = template_uri(&server); + + let offered = |server: &mut TestServer| -> Vec { + server + .completion_request(&uri, Position::new(0, 12)) + .iter() + .map(|completion| completion.label.clone()) + .collect() + }; + + assert!(!offered(&mut server).contains(&"blog/cards/card.html".to_string())); + + let created = SystemPath::new("src/blog/templates/blog/cards/card.html"); + server.write_file(created, "

a card

\n")?; + server.did_change_watched_files(vec![FileEvent { + uri: server.file_uri(created), + kind: FileChangeType::Created, + }]); + + assert!( + offered(&mut server).contains(&"blog/cards/card.html".to_string()), + "a template in a directory created after the server started is still a template" + ); + + Ok(()) +} + +const BUILD_MANIFEST: &str = "src/build/.by-manifest"; + +/// a project holding a build's output, with a template of its own under `build/` +fn server_with_a_build_output() -> Result { + let mut server = TestServerBuilder::new()? + .with_workspace(SystemPath::new("src"), None)? + .with_file(SystemPath::new("src/blog/views.py"), VIEWS)? + .with_file(SystemPath::new("src/blog/templates/blog/base.html"), BASE)? + .with_file( + SystemPath::new("src/blog/templates/blog/post.html"), + "{% extends '' %}\n", + )? + .with_file(SystemPath::new(BUILD_MANIFEST), "blog/views.py\n")? + .with_file( + SystemPath::new("src/build/blog/templates/blog/built.html"), + BASE, + )? + .build() + .wait_until_workspaces_are_initialized(); + server.open_text_document_as( + SystemPath::new("src/blog/templates/blog/post.html"), + "{% extends '' %}\n", + 1, + LanguageKind::new("django-html"), + ); + Ok(server) +} + +/// the names an `{% extends '' %}` in the template under test is offered +fn offered_to_the_extends(server: &mut TestServer) -> Vec { + let uri = template_uri(server); + server + .completion_request(&uri, Position::new(0, 12)) + .iter() + .map(|completion| completion.label.clone()) + .collect() +} + +/// a build's output is a copy of the project, templates and all, and none of it is the +/// project's own: a template there is not one the project can load by that name +#[test] +fn a_template_in_a_build_output_is_not_offered_to_an_extends() -> Result<()> { + let mut server = server_with_a_build_output()?; + + let offered = offered_to_the_extends(&mut server); + assert!( + offered.contains(&"blog/base.html".to_string()), + "{offered:#?}" + ); + assert!( + !offered.contains(&"blog/built.html".to_string()), + "{offered:#?}" + ); + + Ok(()) +} + +/// deleting the manifest makes the directory an ordinary one of the project's, and the +/// templates in it the project's own +#[test] +fn a_template_in_a_former_build_output_is_offered_to_an_extends() -> Result<()> { + let mut server = server_with_a_build_output()?; + let offered = offered_to_the_extends(&mut server); + assert!( + !offered.contains(&"blog/built.html".to_string()), + "{offered:#?}" + ); + + std::fs::remove_file(server.file_path(BUILD_MANIFEST).as_std_path())?; + server.did_change_watched_files(vec![FileEvent { + uri: server.file_uri(BUILD_MANIFEST), + kind: FileChangeType::Deleted, + }]); + + let offered = offered_to_the_extends(&mut server); + assert!( + offered.contains(&"blog/built.html".to_string()), + "{offered:#?}" + ); + + Ok(()) +} + #[test] fn a_template_is_never_type_checked_as_python() -> Result<()> { // every line of this would be a python syntax error diff --git a/crates/ty_server/tests/e2e/file_watching.rs b/crates/ty_server/tests/e2e/file_watching.rs index 5a74ce6b3a..af483ab549 100644 --- a/crates/ty_server/tests/e2e/file_watching.rs +++ b/crates/ty_server/tests/e2e/file_watching.rs @@ -3,15 +3,26 @@ //! These tests read real file system events from the operating system. Like the command line's //! file-watching tests, they do not work inside a sandbox that withholds those events. +use std::time::Duration; + use anyhow::Result; -use lsp_types::{DocumentDiagnosticReport, Message, RegistrationRequest}; +use lsp_types::{ + DocumentDiagnosticReport, Message, RegistrationRequest, WorkspaceDiagnosticRequest, +}; use ruff_db::system::SystemPath; +use ty_project::UseUv; +use ty_server::{ClientOptions, DiagnosticMode}; +use crate::pull_diagnostics::{ + assert_workspace_diagnostics_suspends_for_long_polling, send_workspace_diagnostic_request, +}; use crate::{TestServer, TestServerBuilder}; const LIB_INT: &str = "def make() -> int:\n return 1\n"; const LIB_STR: &str = "def make() -> str:\n return ''\n"; const USER: &str = "from lib import make\n\nx: bytes = make()\n"; +const IMPORTS_OTHER: &str = "from other import make\n\nx: bytes = make()\n"; +const IMPORTS_DEP: &str = "from dep import make\n\nx: bytes = make()\n"; /// The messages of the diagnostics the server has for `path` now. fn diagnostic_messages(server: &mut TestServer, path: &SystemPath) -> Vec { @@ -52,6 +63,19 @@ fn diagnostics_after_change( } } +/// how long a change is given to reach the server and be answered before it is taken to have +/// sent nothing: the watcher settles a batch 10 ms after its last event +const SETTLE: Duration = Duration::from_secs(1); + +/// asserts that the server asks for no refresh within [`SETTLE`] +#[track_caller] +fn assert_no_refresh(server: &mut TestServer, after: &str) { + assert!( + !server.try_await_diagnostic_refresh(SETTLE), + "the server asked the client to refresh after {after}" + ); +} + /// A file that is not open, rewritten on disk, reaches the diagnostics of an open file that /// depends on it, with no `workspace/didChangeWatchedFiles` from the client. The server hears of /// the write from the operating system, which reports it only once it has happened. @@ -177,3 +201,208 @@ fn the_client_is_asked_to_watch_when_the_server_does_not() -> Result<()> { Ok(()) } + +/// a refresh makes the client pull the diagnostics and inlay hints of everything it shows again, +/// so a write that no answer depends on is not one: not a build writing its output a file at a +/// time, nor a write into an excluded or an ignored directory. a write the project reads after +/// them is +#[test] +fn writes_the_project_does_not_read_are_not_a_refresh() -> Result<()> { + let lib = SystemPath::new("project/src/lib.py"); + let user = SystemPath::new("project/src/user.py"); + + let mut server = TestServerBuilder::new()? + .with_workspace(SystemPath::new("project"), None)? + .with_file("project/basedpython.toml", "[src]\nexclude = [\"runs\"]\n")? + .with_file("project/.gitignore", "__pycache__/\n")? + .with_file(lib, LIB_INT)? + .with_file(user, USER)? + // the output of an earlier build, which a rebuild writes over + .with_file("project/build/.by-manifest", "src/lib.py\nsrc/user.py\n")? + .with_file("project/build/lib.py", LIB_INT)? + .with_file("project/build/user.py", USER)? + .with_file("project/runs/0.trace", "")? + .with_file("project/src/__pycache__/lib.cpython-314.pyc", "")? + .enable_pull_diagnostics(true) + .enable_workspace_diagnostic_refresh(true) + .watch_file_system() + .build() + .wait_until_workspaces_are_initialized(); + + server.open_text_document(user, USER, 1); + let before = diagnostic_messages(&mut server, user); + assert!( + before.iter().any(|message| message.contains("`int`")), + "{before:#?}" + ); + assert_no_refresh(&mut server, "starting"); + + // far enough apart for each round to be a batch of its own, as a build's writes are + for round in 0..8 { + server.write_file("project/build/lib.py", LIB_STR)?; + server.write_file(format!("project/build/pkg/module_{round}.py"), LIB_STR)?; + server.write_file(format!("project/runs/{round}.trace"), "")?; + server.write_file( + format!("project/src/__pycache__/user.{round}.cpython-314.pyc"), + "", + )?; + std::thread::sleep(Duration::from_millis(50)); + } + assert_no_refresh( + &mut server, + "writes to a build's output, an excluded directory and an ignored one", + ); + + server.write_file(lib, LIB_STR)?; + diagnostics_after_change(&mut server, user, |messages| { + messages.iter().any(|message| message.contains("`str`")) + }); + + Ok(()) +} + +/// a file appearing and going away changes what an import resolves to, and each is a refresh, +/// though no file the server had read was written +#[test] +fn a_new_file_and_a_deleted_one_are_each_a_refresh() -> Result<()> { + let user = SystemPath::new("project/src/user.py"); + let other = SystemPath::new("project/src/other.py"); + let unresolved = |messages: &[String]| { + messages + .iter() + .any(|message| message.contains("Cannot resolve imported module `other`")) + }; + + let mut server = TestServerBuilder::new()? + .with_workspace(SystemPath::new("project"), None)? + .with_file(user, IMPORTS_OTHER)? + .enable_pull_diagnostics(true) + .enable_workspace_diagnostic_refresh(true) + .watch_file_system() + .build() + .wait_until_workspaces_are_initialized(); + + server.open_text_document(user, IMPORTS_OTHER, 1); + let before = diagnostic_messages(&mut server, user); + assert!(unresolved(&before), "{before:#?}"); + + server.write_file(other, LIB_INT)?; + diagnostics_after_change(&mut server, user, |messages| !unresolved(messages)); + + std::fs::remove_file(server.file_path(other).as_std_path())?; + diagnostics_after_change(&mut server, user, unresolved); + + Ok(()) +} + +/// a dependency is read from outside the project, in the environment's `site-packages`, and a +/// change to it is a refresh like a change to the project's own file +#[cfg(unix)] +#[test] +fn a_change_in_site_packages_is_a_refresh() -> Result<()> { + let user = SystemPath::new("project/user.py"); + let dependency = SystemPath::new("venv/lib/python3.14/site-packages/dep.py"); + + let builder = TestServerBuilder::new()?; + let home = builder.file_path("base/bin"); + let mut server = builder + .with_workspace(SystemPath::new("project"), None)? + .with_file( + "project/basedpython.toml", + "[environment]\npython = \"../venv\"\n", + )? + .with_file(user, IMPORTS_DEP)? + .with_file("base/bin/python", "")? + .with_file("venv/bin/python", "")? + .with_file( + "venv/pyvenv.cfg", + format!("home = {home}\nversion_info = 3.14.0\n"), + )? + .with_file(dependency, LIB_INT)? + .enable_pull_diagnostics(true) + .enable_workspace_diagnostic_refresh(true) + .watch_file_system() + .build() + .wait_until_workspaces_are_initialized(); + + server.open_text_document(user, IMPORTS_DEP, 1); + let before = diagnostic_messages(&mut server, user); + assert!( + before.iter().any(|message| message.contains("`int`")), + "{before:#?}" + ); + + server.write_file(dependency, LIB_STR)?; + diagnostics_after_change(&mut server, user, |messages| { + messages.iter().any(|message| message.contains("`str`")) + }); + + Ok(()) +} + +/// a configuration file changes every answer without changing a source file +#[test] +fn a_configuration_change_is_a_refresh() -> Result<()> { + let lib = SystemPath::new("project/lib.py"); + let user = SystemPath::new("project/user.py"); + let configuration = SystemPath::new("project/basedpython.toml"); + + let mut server = TestServerBuilder::new()? + .with_workspace(SystemPath::new("project"), None)? + .with_file(configuration, "")? + .with_file(lib, LIB_INT)? + .with_file(user, USER)? + .with_use_uv(UseUv::Off) + .enable_pull_diagnostics(true) + .enable_workspace_diagnostic_refresh(true) + .watch_file_system() + .build() + .wait_until_workspaces_are_initialized(); + + server.open_text_document(user, USER, 1); + let before = diagnostic_messages(&mut server, user); + assert!(!before.is_empty(), "{before:#?}"); + + server.write_file(configuration, "[rules]\ninvalid-assignment = \"ignore\"\n")?; + diagnostics_after_change(&mut server, user, <[String]>::is_empty); + + Ok(()) +} + +/// a workspace diagnostic request held open until the workspace's diagnostics change is answered +/// when the server's own watcher sees the change, with nothing from the client +#[test] +fn a_request_held_open_is_answered_after_a_change_the_server_saw() -> Result<()> { + let broken = SystemPath::new("project/broken.py"); + + let mut server = TestServerBuilder::new()? + .with_workspace(SystemPath::new("project"), None)? + .with_file("project/clean.py", LIB_INT)? + .with_initialization_options( + &ClientOptions::default().with_diagnostic_mode(DiagnosticMode::Workspace), + ) + .watch_file_system() + .build() + .wait_until_workspaces_are_initialized(); + + // checking the one file first leaves the workspace check with nothing slow to do, so the long + // poll is held open well within the time `assert_workspace_diagnostics_suspends_for_long_polling` + // gives it, rather than still being checked when the file is written + let clean = SystemPath::new("project/clean.py"); + server.open_text_document(clean, LIB_INT, 1); + assert_eq!( + diagnostic_messages(&mut server, clean), + Vec::::new() + ); + + let long_poll = send_workspace_diagnostic_request(&mut server); + assert_workspace_diagnostics_suspends_for_long_polling(&mut server, &long_poll); + + server.write_file(broken, "x: int = ''\n")?; + let report = server + .try_await_response::(&long_poll, Some(Duration::from_secs(10))) + .unwrap_or_else(|error| panic!("the request held open was not answered: {error}")); + assert_eq!(report.items.len(), 1, "{report:#?}"); + + Ok(()) +} diff --git a/crates/ty_server/tests/e2e/goto_definition.rs b/crates/ty_server/tests/e2e/goto_definition.rs index f457a84835..0fb9c8fead 100644 --- a/crates/ty_server/tests/e2e/goto_definition.rs +++ b/crates/ty_server/tests/e2e/goto_definition.rs @@ -251,12 +251,13 @@ from idna import encode server.write_file(script, updated)?; // A watcher notification can arrive after the file is written but before `didSave`. - // Synchronization must still wait for the save notification. + // Synchronization must still wait for the save notification. the event changes nothing + // the server reads, as an open document's text comes from the client, so it asks for no + // refresh; the request below is answered after the event is handled server.did_change_watched_files(vec![FileEvent { uri: server.file_uri(script), kind: FileChangeType::Changed, }]); - server.await_diagnostic_refresh(); assert!( server @@ -267,14 +268,15 @@ from idna import encode server.save_text_document(script); + // the finished synchronization refreshes diagnostics, and the save itself may refresh + // them before it server.await_diagnostic_refresh(); - - assert!( - server - .goto_definition_request(script, Position::new(4, 19)) - .is_some(), - "saving must synchronize newly declared script dependencies" - ); + while server + .goto_definition_request(script, Position::new(4, 19)) + .is_none() + { + server.await_diagnostic_refresh(); + } Ok(()) } @@ -493,17 +495,24 @@ from idna import encode kind: FileChangeType::Created, }]); - // Watched-file changes refresh diagnostics immediately and again after uv finishes. - server.await_diagnostic_refresh(); - server.await_diagnostic_refresh(); - - server.open_text_document(script, updated, 2); - assert!( - server + // the finished synchronization refreshes diagnostics. whether the event refreshes them + // before it depends on whether the server read the rewritten file when the document + // closed, which races the write, so each refresh is checked until the new dependency + // resolves: a closed script is looked at by opening it, and closed again to go on + // waiting + let mut version = 2; + loop { + server.await_diagnostic_refresh(); + server.open_text_document(script, updated, version); + if server .goto_definition_request(script, Position::new(4, 19)) - .is_some(), - "watched changes must update environments even while scripts are closed" - ); + .is_some() + { + break; + } + server.close_text_document(script); + version += 1; + } Ok(()) } diff --git a/crates/ty_server/tests/e2e/main.rs b/crates/ty_server/tests/e2e/main.rs index 554a27ccf3..c1579475bf 100644 --- a/crates/ty_server/tests/e2e/main.rs +++ b/crates/ty_server/tests/e2e/main.rs @@ -642,6 +642,19 @@ impl TestServer { self.send(Message::Response(Response::new_ok(id, ()))); } + /// Acknowledges a diagnostic refresh if the server requests one within `timeout`, and says + /// whether it did. + pub(crate) fn try_await_diagnostic_refresh(&mut self, timeout: Duration) -> bool { + match self.try_await_request::(Some(timeout)) { + Ok((id, ())) => { + self.send(Message::Response(Response::new_ok(id, ()))); + true + } + Err(ServerMessageError::Timeout) => false, + Err(error) => panic!("Failed to receive a diagnostic refresh: {error}"), + } + } + /// Checks server-created progress with matching begin, report, and end notifications. #[cfg(feature = "test-uv")] #[track_caller] diff --git a/crates/ty_server/tests/e2e/publish_diagnostics.rs b/crates/ty_server/tests/e2e/publish_diagnostics.rs index f68a2a384a..18959bcdb4 100644 --- a/crates/ty_server/tests/e2e/publish_diagnostics.rs +++ b/crates/ty_server/tests/e2e/publish_diagnostics.rs @@ -718,6 +718,7 @@ assert_type("test", list[str]) fn on_did_change_watched_files() -> Result<()> { let workspace_root = SystemPath::new("src"); let foo = SystemPath::new("src/foo.py"); + let bar = SystemPath::new("src/bar.py"); let foo_content = "\ def foo() -> str: print(a) @@ -726,14 +727,16 @@ def foo() -> str: let mut server = TestServerBuilder::new()? .with_workspace(workspace_root, None)? .with_file(foo, "")? + .with_file(bar, "")? .enable_pull_diagnostics(false) .build() .wait_until_workspaces_are_initialized(); let foo = server.file_path(foo); let foo_uri = server.file_uri(&foo); + let bar = server.file_path(bar); - server.open_text_document(&foo, "", 1); + server.open_text_document(&foo, "import bar\n", 1); let _open_diagnostics = server.await_notification::(); @@ -744,11 +747,19 @@ def foo() -> str: server.collect_publish_diagnostic_notifications(2); std::fs::write(&foo, foo_content)?; + // a module the open document imports, which the editor does not hold + std::fs::write(&bar, "x = 1\n")?; - server.did_change_watched_files(vec![FileEvent { - uri: foo_uri.clone(), - kind: FileChangeType::Changed, - }]); + server.did_change_watched_files(vec![ + FileEvent { + uri: foo_uri.clone(), + kind: FileChangeType::Changed, + }, + FileEvent { + uri: server.file_uri(&bar), + kind: FileChangeType::Changed, + }, + ]); let mut diagnostics = collect_publish_diagnostic_notifications_with_versions(&mut server, 3); assert_eq!(diagnostics[&first_cell].version, Some(0)); @@ -776,6 +787,39 @@ def foo() -> str: Ok(()) } +/// A change on disk to a document the editor holds changes nothing the server answers from: the +/// editor's text is what it checks. Publishing every open document's diagnostics again would +/// tell the client nothing it did not already have. +#[test] +fn on_did_change_watched_files_for_an_open_document_publishes_nothing() -> Result<()> { + let foo = SystemPath::new("src/foo.py"); + + let mut server = TestServerBuilder::new()? + .with_workspace(SystemPath::new("src"), None)? + .with_file(foo, "")? + .enable_pull_diagnostics(false) + .build() + .wait_until_workspaces_are_initialized(); + + server.open_text_document(foo, "", 1); + let _open_diagnostics = server.await_notification::(); + + server.write_file(foo, "def foo() -> str:\n print(a)\n")?; + server.did_change_watched_files(vec![FileEvent { + uri: server.file_uri(foo), + kind: FileChangeType::Changed, + }]); + + let diagnostics = server + .try_await_notification::(Some(Duration::from_millis(500))); + assert!( + diagnostics.is_err(), + "a change to the disk copy of an open document is not a change: {diagnostics:?}" + ); + + Ok(()) +} + #[test] fn on_did_change_watched_files_pull_diagnostics() -> Result<()> { let workspace_root = SystemPath::new("src"); diff --git a/crates/ty_server/tests/e2e/script_preparation.rs b/crates/ty_server/tests/e2e/script_preparation.rs index e185bcb5d4..c9f3d7208d 100644 --- a/crates/ty_server/tests/e2e/script_preparation.rs +++ b/crates/ty_server/tests/e2e/script_preparation.rs @@ -74,8 +74,9 @@ fn created_closed_script_is_prepared_after_a_file_event() -> Result<()> { kind: lsp_types::FileChangeType::Created, }]); - // The file event and completed synchronization each refresh diagnostics. - server.await_diagnostic_refresh(); + // the finished synchronization refreshes diagnostics. the file event may not: until + // something has listed the project's files, a new one changes nothing the server has read. + // the workspace request waits for the script's environment either way server.await_diagnostic_refresh(); let report = server.workspace_diagnostic_request(None, None); assert_snapshot!(condensed_workspace_diagnostic_snapshot(report), @" From 5b6973a68683573f2b0854521b82b9002be29ebf Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Fri, 25 Sep 2026 19:27:17 +1000 Subject: [PATCH 04/11] tag every inlay hint with the name of the inlayHints setting that switches it, as data.kind, splitting the hint kinds so each is switched by exactly one setting --- .../ty_ide/src/django_template/inlay_hints.rs | 13 +- crates/ty_ide/src/inlay_hints.rs | 414 +++++++++++++++--- ...rguments_unnamed_with_the_setting_off.snap | 74 ++++ .../src/server/api/requests/inlay_hints.rs | 46 +- crates/ty_server/src/session/options.rs | 45 ++ .../ty_server/tests/e2e/django_templates.rs | 11 + crates/ty_server/tests/e2e/inlay_hints.rs | 82 +++- crates/ty_wasm/src/lib.rs | 23 +- docs/basedpython/features/editor.md | 12 + 9 files changed, 605 insertions(+), 115 deletions(-) create mode 100644 crates/ty_ide/src/snapshots/ty_ide__inlay_hints__tests__basedpython_type_arguments_unnamed_with_the_setting_off.snap diff --git a/crates/ty_ide/src/django_template/inlay_hints.rs b/crates/ty_ide/src/django_template/inlay_hints.rs index 91b65ac178..232f48b9ef 100644 --- a/crates/ty_ide/src/django_template/inlay_hints.rs +++ b/crates/ty_ide/src/django_template/inlay_hints.rs @@ -11,6 +11,8 @@ use ruff_text_size::{TextRange, TextSize}; use ty_project::Db; use crate::InlayHintSettings; +use crate::inlay_hints::hint_switches; +use strum_macros::EnumIter; use super::index::{BindingOrigin, TemplateIndex, TemplateReference}; use super::lexer::TokenKind; @@ -30,7 +32,7 @@ pub struct TemplateInlayHint { } /// what a template hint says -#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, EnumIter)] pub enum TemplateInlayHintKind { /// the type of a name the template binds Type, @@ -38,6 +40,11 @@ pub enum TemplateInlayHintKind { Template, } +hint_switches!(TemplateInlayHintKind, template_enabled, set_template_enabled, { + Type => template_binding_types = "templateBindingTypes", + Template => resolved_templates = "resolvedTemplates", +}); + /// every hint `range` of the template `file` shows pub(crate) fn inlay_hints( db: &dyn Db, @@ -50,11 +57,11 @@ pub(crate) fn inlay_hints( ) -> Vec { let mut hints = Vec::new(); - if settings.template_binding_types { + if settings.template_enabled(TemplateInlayHintKind::Type) { binding_types(db, env, file, index, source, range, &mut hints); } - if settings.resolved_templates { + if settings.template_enabled(TemplateInlayHintKind::Template) { for reference in index.extends().into_iter().chain(index.includes()) { resolved_template(db, index, range, reference, &mut hints); } diff --git a/crates/ty_ide/src/inlay_hints.rs b/crates/ty_ide/src/inlay_hints.rs index 2d653371b4..f4e0a6db48 100644 --- a/crates/ty_ide/src/inlay_hints.rs +++ b/crates/ty_ide/src/inlay_hints.rs @@ -3,6 +3,7 @@ use ty_python_semantic::ProgramEnvironment; use itertools::{Either, Itertools}; use rustc_hash::FxHashMap; +use strum_macros::EnumIter; use crate::importer::{ImportAction, ImportRequest, Importer, MembersInScope}; use crate::{Db, HasNavigationTargets, NavigationTarget}; @@ -87,7 +88,7 @@ impl InlayHint { Some(Self { position, - kind: InlayHintKind::Type, + kind: InlayHintKind::VariableType, label: InlayHintLabel { parts: label_parts }, padding_left: false, padding_right: false, @@ -285,7 +286,7 @@ impl InlayHint { Self { position, - kind: InlayHintKind::TypeArgument, + kind: InlayHintKind::CallTypeArguments, label: InlayHintLabel { parts }, padding_left: false, padding_right: false, @@ -324,7 +325,7 @@ impl InlayHint { fn type_argument_name(position: TextSize, name: &str) -> Self { Self { position, - kind: InlayHintKind::TypeArgument, + kind: InlayHintKind::TypeArgumentName, label: InlayHintLabel { parts: vec![InlayHintLabelPart::new(name), "=".into()], }, @@ -436,6 +437,7 @@ impl InlayHint { fn implicit_parameters( db: &dyn Db, env: &ProgramEnvironment<'_>, + kind: InlayHintKind, position: TextSize, parameters: &[(&str, Option)], leading_space: bool, @@ -459,7 +461,7 @@ impl InlayHint { Self { position, - kind: InlayHintKind::ImplicitParameter, + kind, label: InlayHintLabel { parts }, padding_left: leading_space, padding_right: parameter_follows, @@ -477,7 +479,7 @@ impl InlayHint { ) -> Self { Self { position, - kind: InlayHintKind::Type, + kind: InlayHintKind::InferredReturnType, label: InlayHintLabel { parts: vec![format!("-> {}", returned.display(db, env)).into()], }, @@ -493,12 +495,13 @@ impl InlayHint { fn inferred_annotation( db: &dyn Db, env: &ProgramEnvironment<'_>, + kind: InlayHintKind, position: TextSize, ty: Type, ) -> Self { Self { position, - kind: InlayHintKind::Type, + kind, label: InlayHintLabel { parts: vec![format!(": {}", ty.display(db, env)).into()], }, @@ -513,9 +516,25 @@ impl InlayHint { } } -#[derive(Debug, Clone)] +/// what a hint in a python or basedpython file is +/// +/// each kind is switched by a setting of [`InlayHintSettings`] of its own, so a +/// client can tell every kind apart by [`InlayHintKind::setting`] rather than +/// by reading its label. the django template hints have kinds of their own, in +/// `TemplateInlayHintKind` +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, EnumIter)] pub enum InlayHintKind { - Type, + /// The type of an unannotated binding + VariableType, + /// The type of an unannotated lambda parameter + LambdaParameterType, + /// basedpython: the type a parameter takes from the method it overrides or + /// the overloads it implements + InheritedParameterType, + /// basedpython: the type a property declaration leaves to its accessors + PropertyType, + /// basedpython: the return type of a `def` that leaves its annotation out + InferredReturnType, CallArgumentName, /// basedpython: a function's inferred exception set Raises, @@ -523,9 +542,10 @@ pub enum InlayHintKind { Variance, /// basedpython: a type parameter reified without saying so Reification, - /// The type arguments inferred for a generic call, or the name of the type - /// parameter a positional type argument fills - TypeArgument, + /// The type arguments inferred for a generic call + CallTypeArguments, + /// The name of the type parameter a positional type argument fills + TypeArgumentName, /// basedpython: a method that overrides a superclass member without saying so Override, /// basedpython-ui: the observables a function reads while composing @@ -542,14 +562,123 @@ pub enum InlayHintKind { RevealedType, /// The value an enum member takes without the source writing one EnumValue, - /// basedpython: a parameter the source never spells (`it`, `self`) + /// basedpython: the parameters a trailing lambda binds without spelling them: + /// `it`, and the receiver named `self` when the callback declares one ImplicitParameter, + /// basedpython: the `self` an `init(...)` binds without spelling it + ImplicitSelf, /// basedpython: an argument a call site fills from a `context` declaration ImplicitArgument, /// basedpython: the default a parameter takes from the method its `def` overrides InheritedDefault, } +impl InlayHintKind { + /// whether a client shows this kind of hint as a parameter rather than as a + /// type, which are the only two kinds LSP has + pub const fn is_parameter(self) -> bool { + match self { + InlayHintKind::VariableType + | InlayHintKind::LambdaParameterType + | InlayHintKind::InheritedParameterType + | InlayHintKind::PropertyType + | InlayHintKind::InferredReturnType + // basedpython: an inferred exception set is a type, like a return type + | InlayHintKind::Raises + | InlayHintKind::Variance + | InlayHintKind::Reification + | InlayHintKind::CallTypeArguments + | InlayHintKind::TypeArgumentName + | InlayHintKind::Override + // basedpython-ui: a read set, a dependency set and an invalidation set + // are typing facts about the function, and `unstable` is a modifier + // like `override` + | InlayHintKind::Reads + | InlayHintKind::Stability + | InlayHintKind::DerivedDeps + | InlayHintKind::Invalidates + | InlayHintKind::NumericPromotion + | InlayHintKind::RevealedType + | InlayHintKind::EnumValue => false, + InlayHintKind::CallArgumentName + | InlayHintKind::ImplicitParameter + | InlayHintKind::ImplicitSelf + | InlayHintKind::ImplicitArgument + | InlayHintKind::InheritedDefault => true, + } + } +} + +/// the one table saying which setting switches each kind of hint +/// +/// from it come the kind's `setting()`, the `inlayHints` option a client knows +/// the setting by, and the `InlayHintSettings` methods that read and write the +/// field holding it, so a kind cannot be named by one setting and switched by +/// another. a client that lets its user treat each kind differently needs to +/// know which kind a hint is, and the label is what a hint says rather than +/// what it is: a revealed `list[int]` and a call's `[int]` differ by a bracket +macro_rules! hint_switches { + ( + $kind:ident, + $enabled:ident, + $set_enabled:ident, + { $($variant:ident => $field:ident = $setting:literal,)* } + ) => { + impl $kind { + /// the name of the `inlayHints` option that switches this kind of hint + pub const fn setting(self) -> &'static str { + match self { + $($kind::$variant => $setting,)* + } + } + } + + impl $crate::InlayHintSettings { + /// whether hints of `kind` are shown + pub const fn $enabled(&self, kind: $kind) -> bool { + match kind { + $($kind::$variant => self.$field,)* + } + } + + /// show or hide hints of `kind` + pub const fn $set_enabled(&mut self, kind: $kind, enabled: bool) { + match kind { + $($kind::$variant => self.$field = enabled,)* + } + } + } + }; +} + +pub(crate) use hint_switches; + +hint_switches!(InlayHintKind, enabled, set_enabled, { + VariableType => variable_types = "variableTypes", + LambdaParameterType => lambda_parameter_types = "lambdaParameterTypes", + InheritedParameterType => inherited_parameter_types = "inheritedParameterTypes", + PropertyType => property_types = "propertyTypes", + InferredReturnType => inferred_return_types = "inferredReturnTypes", + CallArgumentName => call_argument_names = "callArgumentNames", + Raises => inferred_raises = "inferredRaises", + Variance => inferred_variance = "inferredVariance", + Reification => inferred_reification = "inferredReification", + CallTypeArguments => call_type_arguments = "callTypeArguments", + TypeArgumentName => type_argument_names = "typeArgumentNames", + Override => inferred_override = "inferredOverride", + Reads => inferred_reads = "inferredReads", + Stability => parameter_stability = "parameterStability", + DerivedDeps => derived_dependencies = "derivedDependencies", + Invalidates => inferred_invalidations = "inferredInvalidations", + NumericPromotion => numeric_promotions = "numericPromotions", + RevealedType => revealed_types = "revealedTypes", + EnumValue => enum_values = "enumValues", + ImplicitParameter => implicit_parameters = "implicitParameters", + ImplicitSelf => implicit_self = "implicitSelf", + ImplicitArgument => implicit_arguments = "implicitArguments", + InheritedDefault => inherited_parameter_defaults = "inheritedParameterDefaults", +}); + #[derive(Debug, Clone)] pub struct InlayHintLabel { parts: Vec, @@ -767,6 +896,14 @@ pub struct InlayHintSettings { /// ```python /// x: dict["_KT="str, "_VT="int] /// ``` + /// + /// in a basedpython file it also names the type arguments of the types the + /// variable type and call type argument hints show, which are still shown + /// and hidden by their own settings + /// + /// ```by + /// x": dict[Key=str, Value=int]" = identity({"a": 1}) + /// ``` pub type_argument_names: bool, /// basedpython: whether to show `override` on a method that overrides a @@ -1330,8 +1467,21 @@ impl<'a, 'db> InlayHintVisitor<'a, 'db> { self.source_type.is_basedpython() } + /// add `hint` if the setting of its kind is on, answering whether it was added + /// + /// every hint is added here, so a hint is shown exactly when the one setting + /// its kind names is on. the same check made before a hint is worked out + /// only saves the work of one that would be dropped + fn push(&mut self, hint: InlayHint) -> bool { + if !self.settings.enabled(hint.kind) { + return false; + } + self.hints.push(hint); + true + } + fn add_type_hint(&mut self, expr: &Expr, rhs: &Expr, ty: Type<'db>, allow_edits: bool) { - if !self.settings.variable_types { + if !self.settings.enabled(InlayHintKind::VariableType) { return; } @@ -1357,7 +1507,7 @@ impl<'a, 'db> InlayHintVisitor<'a, 'db> { if let Some(inlay_hint) = InlayHint::variable_type(context, expr, rhs, ty, allow_edits, named_type_arguments) { - self.hints.push(inlay_hint); + self.push(inlay_hint); } } @@ -1367,7 +1517,7 @@ impl<'a, 'db> InlayHintVisitor<'a, 'db> { /// annotation, before the `:` — so accepting it reads as ordinary source. fn add_inferred_raises(&mut self, function: &ast::StmtFunctionDef) { let env = &self.model.program_environment(); - if !self.settings.inferred_raises || function.raises.is_some() { + if !self.settings.enabled(InlayHintKind::Raises) || function.raises.is_some() { return; } @@ -1383,8 +1533,7 @@ impl<'a, 'db> InlayHintVisitor<'a, 'db> { .as_deref() .map_or_else(|| function.parameters.end(), Ranged::end); - self.hints - .push(InlayHint::inferred_raises(self.db, env, position, raised)); + self.push(InlayHint::inferred_raises(self.db, env, position, raised)); } /// basedpython: hint the variance ty infers for each type parameter of @@ -1394,7 +1543,7 @@ impl<'a, 'db> InlayHintVisitor<'a, 'db> { type_params: Option<&ast::TypeParams>, owner: impl FnOnce(&SemanticModel<'db>) -> Option>, ) { - if !self.settings.inferred_variance || !self.is_basedpython() { + if !self.settings.enabled(InlayHintKind::Variance) || !self.is_basedpython() { return; } @@ -1427,7 +1576,7 @@ impl<'a, 'db> InlayHintVisitor<'a, 'db> { continue; }; - self.hints.push(InlayHint::inferred_variance( + self.push(InlayHint::inferred_variance( type_param.range().start(), variance, )); @@ -1455,7 +1604,7 @@ impl<'a, 'db> InlayHintVisitor<'a, 'db> { type_params: Option<&ast::TypeParams>, inferred: impl FnOnce(&mut Self) -> Vec, ) { - if !self.settings.inferred_reification || !self.is_basedpython() { + if !self.settings.enabled(InlayHintKind::Reification) || !self.is_basedpython() { return; } @@ -1469,8 +1618,7 @@ impl<'a, 'db> InlayHintVisitor<'a, 'db> { // everything the parameter's own declaration spells for type_param in type_params { if inferred.contains(&type_param.name().id) { - self.hints - .push(InlayHint::inferred_reification(type_param.range().start())); + self.push(InlayHint::inferred_reification(type_param.range().start())); } } } @@ -1478,7 +1626,7 @@ impl<'a, 'db> InlayHintVisitor<'a, 'db> { /// Hint the value an enum member takes when its declaration does not write /// one, at `position` — after `auto()`, or after a `case` variant's name. fn add_enum_member_value(&mut self, name: &str, position: TextSize) { - if !self.settings.enum_values { + if !self.settings.enabled(InlayHintKind::EnumValue) { return; } @@ -1497,7 +1645,7 @@ impl<'a, 'db> InlayHintVisitor<'a, 'db> { return; }; - self.hints.push(InlayHint::enum_member_value( + self.push(InlayHint::enum_member_value( position, &rendered.to_string(), )); @@ -1507,7 +1655,7 @@ impl<'a, 'db> InlayHintVisitor<'a, 'db> { /// member without saying so. fn add_inferred_override(&mut self, function: &ast::StmtFunctionDef) { let env = &self.model.program_environment(); - if !self.settings.inferred_override || !self.is_basedpython() { + if !self.settings.enabled(InlayHintKind::Override) || !self.is_basedpython() { return; } @@ -1524,7 +1672,7 @@ impl<'a, 'db> InlayHintVisitor<'a, 'db> { // the range of a `def` excludes its decorators, so this is the modifier // position even on a decorated method - self.hints.push(InlayHint::inferred_override( + self.push(InlayHint::inferred_override( function.range().start(), superclass .navigation_targets(self.db, env) @@ -1539,7 +1687,7 @@ impl<'a, 'db> InlayHintVisitor<'a, 'db> { /// annotation and the `raises` clause — and before the `:`, so it reads /// as one more clause of the header. fn add_inferred_reads(&mut self, function: &ast::StmtFunctionDef) { - if !self.settings.inferred_reads || !self.is_basedpython() { + if !self.settings.enabled(InlayHintKind::Reads) || !self.is_basedpython() { return; } @@ -1556,13 +1704,13 @@ impl<'a, 'db> InlayHintVisitor<'a, 'db> { .or(function.returns.as_deref()) .map_or_else(|| function.parameters.end(), Ranged::end); - self.hints.push(InlayHint::inferred_reads(position, &reads)); + self.push(InlayHint::inferred_reads(position, &reads)); } /// basedpython-ui: hint `unstable` before each parameter of the composable /// `function` whose declared type the runtime cannot compare. fn add_parameter_stability(&mut self, function: &ast::StmtFunctionDef) { - if !self.settings.parameter_stability || !self.is_basedpython() { + if !self.settings.enabled(InlayHintKind::Stability) || !self.is_basedpython() { return; } if !function @@ -1584,7 +1732,7 @@ impl<'a, 'db> InlayHintVisitor<'a, 'db> { continue; }; if parameter_stability(self.db, env, ty) == Some(false) { - self.hints.push(InlayHint::unstable_parameter( + self.push(InlayHint::unstable_parameter( parameter.name.range().start(), )); } @@ -1594,7 +1742,7 @@ impl<'a, 'db> InlayHintVisitor<'a, 'db> { /// basedpython-ui: hint what a `derived(...)` / `remember(...)` call's /// computation depends on, at the end of its line. fn add_derived_dependencies(&mut self, call: &ast::ExprCall) { - if !self.settings.derived_dependencies || !self.is_basedpython() { + if !self.settings.enabled(InlayHintKind::DerivedDeps) || !self.is_basedpython() { return; } @@ -1602,7 +1750,7 @@ impl<'a, 'db> InlayHintVisitor<'a, 'db> { return; }; - self.hints.push(InlayHint::derived_dependencies( + self.push(InlayHint::derived_dependencies( self.source.line_end(call.range().end()), &reads, )); @@ -1619,7 +1767,7 @@ impl<'a, 'db> InlayHintVisitor<'a, 'db> { /// after, so a write in one (`if todos.pop():`) is hinted at the end of /// the line it is spelled on. fn add_invalidations(&mut self, site: WriteSite<'_>) { - if !self.settings.inferred_invalidations || !self.is_basedpython() { + if !self.settings.enabled(InlayHintKind::Invalidates) || !self.is_basedpython() { return; } @@ -1634,14 +1782,13 @@ impl<'a, 'db> InlayHintVisitor<'a, 'db> { WriteSite::Statement(stmt) => stmt.end(), WriteSite::Lambda(lambda) => lambda.end(), }; - self.hints - .push(InlayHint::invalidations(position, &invalidations)); + self.push(InlayHint::invalidations(position, &invalidations)); } /// Hint the type arguments inferred for a generic call. fn add_call_type_arguments(&mut self, call: &ast::ExprCall, arguments: &[(Name, Type<'db>)]) { let env = &self.model.program_environment(); - if !self.settings.call_type_arguments || arguments.is_empty() { + if !self.settings.enabled(InlayHintKind::CallTypeArguments) || arguments.is_empty() { return; } @@ -1653,7 +1800,7 @@ impl<'a, 'db> InlayHintVisitor<'a, 'db> { return; } - self.hints.push(InlayHint::call_type_arguments( + self.push(InlayHint::call_type_arguments( self.db, env, call.func.range().end(), @@ -1676,7 +1823,7 @@ impl<'a, 'db> InlayHintVisitor<'a, 'db> { /// — after the explicit arguments, by keyword. fn add_implicit_context_arguments(&mut self, call: &ast::ExprCall, callee: Option>) { let env = &self.model.program_environment(); - if !self.settings.implicit_arguments || !self.is_basedpython() { + if !self.settings.enabled(InlayHintKind::ImplicitArgument) || !self.is_basedpython() { return; } @@ -1709,7 +1856,7 @@ impl<'a, 'db> InlayHintVisitor<'a, 'db> { |argument| argument.range().end(), ); - self.hints.push(InlayHint::implicit_context_arguments( + self.push(InlayHint::implicit_context_arguments( position, last_explicit.is_some(), &labelled, @@ -1719,7 +1866,7 @@ impl<'a, 'db> InlayHintVisitor<'a, 'db> { /// Hint the name of the type parameter each positional argument of a /// subscripted generic fills. fn add_type_argument_names(&mut self, subscript: &ast::ExprSubscript) { - if !self.settings.type_argument_names { + if !self.settings.enabled(InlayHintKind::TypeArgumentName) { return; } @@ -1738,7 +1885,7 @@ impl<'a, 'db> InlayHintVisitor<'a, 'db> { } for (name, argument) in names.iter().zip(subscript_arguments(&subscript.slice)) { - self.hints.push(InlayHint::type_argument_name( + self.push(InlayHint::type_argument_name( argument.range().start(), name, )); @@ -1749,7 +1896,7 @@ impl<'a, 'db> InlayHintVisitor<'a, 'db> { /// `float` / `complex` type expression. fn add_numeric_promotion(&mut self, expr: &Expr) { let env = &self.model.program_environment(); - if !self.settings.numeric_promotions || !self.in_type_expression { + if !self.settings.enabled(InlayHintKind::NumericPromotion) || !self.in_type_expression { return; } @@ -1769,8 +1916,7 @@ impl<'a, 'db> InlayHintVisitor<'a, 'db> { return; }; - self.hints - .push(InlayHint::numeric_promotion(expr.range().end(), arms)); + self.push(InlayHint::numeric_promotion(expr.range().end(), arms)); } /// Visit the operands of a union written in a type expression, each knowing what the @@ -1804,7 +1950,7 @@ impl<'a, 'db> InlayHintVisitor<'a, 'db> { /// Hint the type a `reveal_type` call reveals, at the end of its line. fn add_revealed_type(&mut self, call: &ast::ExprCall) { let env = &self.model.program_environment(); - if !self.settings.revealed_types { + if !self.settings.enabled(InlayHintKind::RevealedType) { return; } @@ -1815,7 +1961,7 @@ impl<'a, 'db> InlayHintVisitor<'a, 'db> { return; }; - self.hints.push(InlayHint::revealed_type( + self.push(InlayHint::revealed_type( self.db, env, self.source.line_end(call.range().end()), @@ -1834,16 +1980,20 @@ impl<'a, 'db> InlayHintVisitor<'a, 'db> { /// head instead. fn add_implicit_self(&mut self, parameter: &ast::Parameter) { let env = &self.model.program_environment(); - if !self.settings.implicit_self || !self.is_basedpython() || !parameter.range().is_empty() { + if !self.settings.enabled(InlayHintKind::ImplicitSelf) + || !self.is_basedpython() + || !parameter.range().is_empty() + { return; } let position = parameter.range().start(); let ty = hintable_parameter_type(&self.model, parameter); - self.hints.push(InlayHint::implicit_parameters( + self.push(InlayHint::implicit_parameters( self.db, env, + InlayHintKind::ImplicitSelf, position, &[(parameter.name.as_str(), ty)], false, @@ -1859,7 +2009,7 @@ impl<'a, 'db> InlayHintVisitor<'a, 'db> { /// the colon rather than between the callee and it. fn add_trailing_lambda_parameter(&mut self, function: &ast::StmtFunctionDef) { let env = &self.model.program_environment(); - if !self.settings.implicit_parameters { + if !self.settings.enabled(InlayHintKind::ImplicitParameter) { return; } @@ -1877,9 +2027,10 @@ impl<'a, 'db> InlayHintVisitor<'a, 'db> { return; } - self.hints.push(InlayHint::implicit_parameters( + self.push(InlayHint::implicit_parameters( self.db, env, + InlayHintKind::ImplicitParameter, colon + TextSize::from(1), ¶meters, true, @@ -1898,7 +2049,7 @@ impl<'a, 'db> InlayHintVisitor<'a, 'db> { /// Hint the inferred type of an unannotated lambda parameter. fn add_lambda_parameter_type(&mut self, parameter: &ast::Parameter) { let env = &self.model.program_environment(); - if !self.settings.lambda_parameter_types + if !self.settings.enabled(InlayHintKind::LambdaParameterType) || !self.in_lambda || parameter.annotation.is_some() || parameter.range().is_empty() @@ -1910,9 +2061,10 @@ impl<'a, 'db> InlayHintVisitor<'a, 'db> { return; }; - self.hints.push(InlayHint::inferred_annotation( + self.push(InlayHint::inferred_annotation( self.db, env, + InlayHintKind::LambdaParameterType, parameter.name.range().end(), ty, )); @@ -1925,7 +2077,7 @@ impl<'a, 'db> InlayHintVisitor<'a, 'db> { /// under their own setting. fn add_inherited_parameter_type(&mut self, parameter: &ast::Parameter) { let env = &self.model.program_environment(); - if !self.settings.inherited_parameter_types + if !self.settings.enabled(InlayHintKind::InheritedParameterType) || self.in_lambda || parameter.annotation.is_some() || parameter.range().is_empty() @@ -1937,9 +2089,10 @@ impl<'a, 'db> InlayHintVisitor<'a, 'db> { return; }; - self.hints.push(InlayHint::inferred_annotation( + self.push(InlayHint::inferred_annotation( self.db, env, + InlayHintKind::InheritedParameterType, parameter.name.range().end(), ty, )); @@ -1948,7 +2101,7 @@ impl<'a, 'db> InlayHintVisitor<'a, 'db> { /// basedpython: hint the default a parameter takes from the method its `def` /// overrides, where the parameter's own default would be written. fn add_inherited_parameter_default(&mut self, parameter: &ast::ParameterWithDefault) { - if !self.settings.inherited_parameter_defaults + if !self.settings.enabled(InlayHintKind::InheritedDefault) || self.in_lambda || parameter.parameter.range().is_empty() { @@ -1959,7 +2112,7 @@ impl<'a, 'db> InlayHintVisitor<'a, 'db> { return; }; - self.hints.push(InlayHint::inherited_default( + self.push(InlayHint::inherited_default( parameter.parameter.range().end(), &value, parameter.parameter.annotation.is_some(), @@ -1974,7 +2127,7 @@ impl<'a, 'db> InlayHintVisitor<'a, 'db> { /// `def f() -> int raises TypeError`. fn add_inferred_return(&mut self, function: &ast::StmtFunctionDef) { let env = &self.model.program_environment(); - if !self.settings.inferred_return_types + if !self.settings.enabled(InlayHintKind::InferredReturnType) || function.returns.is_some() || function.is_asserts_return { @@ -1988,7 +2141,7 @@ impl<'a, 'db> InlayHintVisitor<'a, 'db> { return; }; - self.hints.push(InlayHint::inferred_return( + self.push(InlayHint::inferred_return( self.db, env, function.parameters.end(), @@ -2002,7 +2155,7 @@ impl<'a, 'db> InlayHintVisitor<'a, 'db> { /// so the type goes where the declaration would have written it — after the /// name, exactly as it does for an unannotated variable fn add_property_type(&mut self, getter: &ast::StmtFunctionDef) { - if !self.settings.property_types + if !self.settings.enabled(InlayHintKind::PropertyType) || !self.is_basedpython() || getter.property_construct_range().is_none() // the declaration named a type; the getter carries it as its return @@ -2020,9 +2173,10 @@ impl<'a, 'db> InlayHintVisitor<'a, 'db> { }; let env = &self.model.program_environment(); - self.hints.push(InlayHint::inferred_annotation( + self.push(InlayHint::inferred_annotation( self.db, env, + InlayHintKind::PropertyType, getter.name.range().end(), ty, )); @@ -2041,7 +2195,7 @@ impl<'a, 'db> InlayHintVisitor<'a, 'db> { name: &str, navigation_target: Option, ) -> bool { - if !self.settings.call_argument_names { + if !self.settings.enabled(InlayHintKind::CallArgumentName) { return false; } @@ -2051,8 +2205,7 @@ impl<'a, 'db> InlayHintVisitor<'a, 'db> { let inlay_hint = InlayHint::call_argument_name(position, name, navigation_target); - self.hints.push(inlay_hint); - true + self.push(inlay_hint) } } @@ -2722,6 +2875,8 @@ mod tests { use ruff_python_parser::parse_unchecked_source; use ruff_python_trivia::textwrap::dedent; use ruff_text_size::{TextLen, TextSize}; + use rustc_hash::FxHashSet; + use strum::IntoEnumIterator; use ruff_db::system::{DbWithWritableSystem, SystemPathBuf}; use ty_project::ProjectMetadata; @@ -3043,6 +3198,119 @@ Source with applied edits: "); } + /// every kind of hint is shown with only its own setting on, and nothing + /// else is. a hint is only ever added when its kind's setting is on, so this + /// is what catches a hint whose work is skipped by the wrong setting before + /// it is ever made + #[test] + fn every_kind_is_switched_by_the_setting_it_names() { + fn only(kind: InlayHintKind) -> InlayHintSettings { + let mut settings = InlayHintSettings::none(); + settings.set_enabled(kind, true); + settings + } + + let mut by = basedpython_ui_inlay_hint_test( + r#" + from basedpython_ui import composable, derived, state, Button, Text + + def identity[T](x: T) -> T: + return x + + def two(a: int, b: str) -> None: ... + + x = identity(1) + two(1, 'b') + y = map(lambda v: v + 1, [1]) + z: dict[str, int] = {} + + def raiser(): + raise TypeError + + class Box[T]: + def get(self) -> T: ... + + def reify[T](): + print(T) + + class Base: + def f(self, a: int = 1) -> None: ... + + class Child(Base): + def f(self, a) -> None: ... + + reveal_type(x) + + def apply(fn: (int) -> None) -> None: + fn(1) + + apply: + print(it) + + class Init: + init(a: int) + + class Prop: + let a + get() = 1 + + def ctx(context c: int) -> None: ... + context cc = 1 + ctx() + + enum class Color: + case Red, Green + + @composable + def Counter(items: list[int]): + let count = state(0) + let doubled = derived(lambda: count.value * 2) + Text(f"{doubled.value}") + Button("+"): + count.value += 1 + "#, + ); + let mut python = inlay_hint_test( + " + def f(x: float | None) -> None: + reveal_type(x) + ", + ); + + let mut produced = FxHashSet::default(); + for kind in InlayHintKind::iter() { + let settings = only(kind); + for test in [&mut by, &mut python] { + let hints = inlay_hints( + &test.db, + ProgramFile::new( + &test.db, + test.file, + test.db.program_environment().program(&test.db), + ), + test.range, + &settings, + ); + for hint in hints { + assert_eq!( + hint.kind, + kind, + "`{}` produced a {:?} hint", + kind.setting(), + hint.kind, + ); + produced.insert(kind); + } + } + } + + let missing: Vec<_> = InlayHintKind::iter() + .filter(|kind| !produced.contains(kind)) + .map(InlayHintKind::setting) + .collect(); + assert!(missing.is_empty(), "no source produced {missing:?}"); + } + /// The same, for the hints a `.py` file gets: one shown at the end of a line /// and one shown between the operands of a union. #[test] @@ -12095,6 +12363,26 @@ def each(fn: (int) -> None) -> None: })); } + /// with `typeArgumentNames` off, the types other hints show leave their type + /// arguments unnamed, while those hints are still shown + #[test] + fn basedpython_type_arguments_unnamed_with_the_setting_off() { + let mut test = basedpython_inlay_hint_test( + " + class Pair[Key, Value]: + init(key: Key, value: Value) + + a = Pair(1, 'x') + ", + ); + + assert_snapshot!(test.inlay_hints_with_settings(&InlayHintSettings { + variable_types: true, + call_type_arguments: true, + ..InlayHintSettings::none() + })); + } + /// A `some` parameter opens an anonymous hole, which is not a position a call /// site can supply, so there is no specialization to offer for it. #[test] diff --git a/crates/ty_ide/src/snapshots/ty_ide__inlay_hints__tests__basedpython_type_arguments_unnamed_with_the_setting_off.snap b/crates/ty_ide/src/snapshots/ty_ide__inlay_hints__tests__basedpython_type_arguments_unnamed_with_the_setting_off.snap new file mode 100644 index 0000000000..011f4556d6 --- /dev/null +++ b/crates/ty_ide/src/snapshots/ty_ide__inlay_hints__tests__basedpython_type_arguments_unnamed_with_the_setting_off.snap @@ -0,0 +1,74 @@ +--- +source: crates/ty_ide/src/inlay_hints.rs +expression: "test.inlay_hints_with_settings(&InlayHintSettings\n{\n variable_types: true, call_type_arguments: true,\n ..InlayHintSettings::none()\n})" +--- + +class Pair[Key, Value]: + init(key: Key, value: Value) + +a[: Pair[1, "x"]] = Pair[[1, "x"]](1, 'x') + +--------------------------------------------- +info[inlay-hint-location]: Inlay Hint Target + --> main.by:2:7 + | +2 | class Pair[Key, Value]: + | ^^^^ +info: Source + --> main2.py:5:5 + | +5 | a[: Pair[1, "x"]] = Pair[[1, "x"]](1, 'x') + | ^^^^ + +info[inlay-hint-location]: Inlay Hint Target + --> stdlib/builtins.byi:LL:7 + | +LL | class int: + | ^^^ +info: Source + --> main2.py:LL:10 + | +LL | a[: Pair[1, "x"]] = Pair[[1, "x"]](1, 'x') + | ^ + +info[inlay-hint-location]: Inlay Hint Target + --> stdlib/builtins.byi:LL:7 + | +LL | class str(Sequence[str]): + | ^^^ +info: Source + --> main2.py:LL:13 + | +LL | a[: Pair[1, "x"]] = Pair[[1, "x"]](1, 'x') + | ^^^ + +info[inlay-hint-location]: Inlay Hint Target + --> stdlib/builtins.byi:LL:7 + | +LL | class int: + | ^^^ +info: Source + --> main2.py:LL:27 + | +LL | a[: Pair[1, "x"]] = Pair[[1, "x"]](1, 'x') + | ^ + +info[inlay-hint-location]: Inlay Hint Target + --> stdlib/builtins.byi:LL:7 + | +LL | class str(Sequence[str]): + | ^^^ +info: Source + --> main2.py:LL:30 + | +LL | a[: Pair[1, "x"]] = Pair[[1, "x"]](1, 'x') + | ^^^ + +--------------------------------------------- +info[inlay-hint-edit]: Inlay hint edits +--> main.by:1:1 + | +4 | + - a = Pair(1, 'x') +5 + a: Pair[1, "x"] = Pair(1, 'x') + | diff --git a/crates/ty_server/src/server/api/requests/inlay_hints.rs b/crates/ty_server/src/server/api/requests/inlay_hints.rs index 53101670aa..569ca91c92 100644 --- a/crates/ty_server/src/server/api/requests/inlay_hints.rs +++ b/crates/ty_server/src/server/api/requests/inlay_hints.rs @@ -86,11 +86,11 @@ impl BackgroundDocumentRequestHandler for InlayHintRequestHandler { .to_lsp_position(db, file, snapshot.encoding())? .local_position(), label: inlay_hint_label(&hint.label, db, snapshot.encoding()), - kind: Some(inlay_hint_kind(&hint.kind)), + kind: Some(inlay_hint_kind(hint.kind)), tooltip: None, padding_left: hint.padding_left.then_some(true), padding_right: hint.padding_right.then_some(true), - data: None, + data: Some(kind_tag(hint.kind.setting())), text_edits: Some( hint.text_edits .into_iter() @@ -139,34 +139,28 @@ fn template_inlay_hint( tooltip: None, padding_left: hint.padding_left.then_some(true), padding_right: None, - data: None, + data: Some(kind_tag(hint.kind.setting())), text_edits: None, }) } -fn inlay_hint_kind(inlay_hint_kind: &InlayHintKind) -> lsp_types::InlayHintKind { - match inlay_hint_kind { - InlayHintKind::Type - // basedpython: an inferred exception set is a type, like a return type - | InlayHintKind::Raises - | InlayHintKind::Variance - | InlayHintKind::Reification - | InlayHintKind::TypeArgument - | InlayHintKind::Override - // basedpython-ui: a read set, a dependency set and an invalidation set - // are typing facts about the function, and `unstable` is a modifier - // like `override` - | InlayHintKind::Reads - | InlayHintKind::Stability - | InlayHintKind::DerivedDeps - | InlayHintKind::Invalidates - | InlayHintKind::NumericPromotion - | InlayHintKind::RevealedType - | InlayHintKind::EnumValue => lsp_types::InlayHintKind::Type, - InlayHintKind::CallArgumentName - | InlayHintKind::ImplicitParameter - | InlayHintKind::ImplicitArgument - | InlayHintKind::InheritedDefault => lsp_types::InlayHintKind::Parameter, +/// which kind of hint this is, named by the `inlayHints` option that switches +/// it, as `data.kind` +/// +/// LSP's own `kind` has two values for the two dozen things these hints are, +/// and a client that lets its user treat each kind differently has nothing else +/// to go on but the label, which says what a hint shows rather than what it is. +/// `data` is otherwise only carried back on `inlayHint/resolve`, which this +/// server does not offer +fn kind_tag(setting: &str) -> serde_json::Value { + serde_json::json!({ "kind": setting }) +} + +fn inlay_hint_kind(inlay_hint_kind: InlayHintKind) -> lsp_types::InlayHintKind { + if inlay_hint_kind.is_parameter() { + lsp_types::InlayHintKind::Parameter + } else { + lsp_types::InlayHintKind::Type } } diff --git a/crates/ty_server/src/session/options.rs b/crates/ty_server/src/session/options.rs index 11d545de86..11448811cd 100644 --- a/crates/ty_server/src/session/options.rs +++ b/crates/ty_server/src/session/options.rs @@ -620,3 +620,48 @@ pub(crate) struct PythonExecutable { uri: Option, sys_prefix: SystemPathBuf, } + +#[cfg(test)] +mod tests { + use strum::IntoEnumIterator; + use ty_ide::{InlayHintKind, InlayHintSettings, TemplateInlayHintKind}; + + use super::InlayHintOptions; + + fn settings_with_off(setting: &str) -> InlayHintSettings { + serde_json::from_value::(serde_json::json!({ setting: false })) + .unwrap_or_else(|error| panic!("`{setting}` is not an `inlayHints` option: {error}")) + .into_settings() + } + + /// the name a hint is tagged with is the option a client turns it off by, and + /// turning it off hides that kind of hint and no other + #[test] + fn each_hint_kind_is_turned_off_by_the_option_it_names() { + for kind in InlayHintKind::iter() { + let settings = settings_with_off(kind.setting()); + for other in InlayHintKind::iter() { + assert_eq!( + settings.enabled(other), + other != kind, + "`{}: false` switched {other:?}", + kind.setting(), + ); + } + assert!(TemplateInlayHintKind::iter().all(|other| settings.template_enabled(other))); + } + + for kind in TemplateInlayHintKind::iter() { + let settings = settings_with_off(kind.setting()); + for other in TemplateInlayHintKind::iter() { + assert_eq!( + settings.template_enabled(other), + other != kind, + "`{}: false` switched {other:?}", + kind.setting(), + ); + } + assert!(InlayHintKind::iter().all(|other| settings.enabled(other))); + } + } +} diff --git a/crates/ty_server/tests/e2e/django_templates.rs b/crates/ty_server/tests/e2e/django_templates.rs index bfc968848e..7cf65f0801 100644 --- a/crates/ty_server/tests/e2e/django_templates.rs +++ b/crates/ty_server/tests/e2e/django_templates.rs @@ -1115,6 +1115,17 @@ fn a_template_hints_a_loop_binding_and_the_file_an_include_resolves_to() -> Resu ["11 : Book", "50 → blog/templates/blog/card.html"] ); + // each is tagged with the `inlayHints` option that switches it, as a python + // hint is + let kinds: Vec<_> = hints.iter().map(|hint| hint.data.clone()).collect(); + assert_eq!( + kinds, + [ + Some(serde_json::json!({ "kind": "templateBindingTypes" })), + Some(serde_json::json!({ "kind": "resolvedTemplates" })), + ] + ); + Ok(()) } diff --git a/crates/ty_server/tests/e2e/inlay_hints.rs b/crates/ty_server/tests/e2e/inlay_hints.rs index 6216106ffe..9ecca40b8c 100644 --- a/crates/ty_server/tests/e2e/inlay_hints.rs +++ b/crates/ty_server/tests/e2e/inlay_hints.rs @@ -83,7 +83,10 @@ y = foo(Thing()) }, "newText": ": Thing" } - ] + ], + "data": { + "kind": "variableTypes" + } }, { "position": { @@ -126,7 +129,10 @@ y = foo(Thing()) }, "newText": "a=" } - ] + ], + "data": { + "kind": "callArgumentNames" + } } ] "#); @@ -174,7 +180,10 @@ def f(x: int) -> None: ], "kind": 1, "textEdits": [], - "paddingLeft": true + "paddingLeft": true, + "data": { + "kind": "revealedTypes" + } } ] "#); @@ -182,6 +191,68 @@ def f(x: int) -> None: Ok(()) } +/// the basedpython hints are each tagged, as `data.kind`, with the `inlayHints` +/// option that switches them, though LSP's own `kind` cannot tell them apart: +/// an implicit `self` is a `Parameter` like a call's argument name, and an +/// inferred return type is a `Type` like a variable's +#[test] +fn a_basedpython_hint_names_the_setting_that_switches_it() -> Result<()> { + let workspace_root = SystemPath::new("src"); + let foo = SystemPath::new("src/foo.by"); + let foo_content = "\ +class C: + init(a: int) + +def f(): + return 1 +"; + + let mut server = TestServerBuilder::new()? + .with_initialization_options(&ClientOptions::default()) + .with_workspace(workspace_root, None)? + .with_file(foo, foo_content)? + .enable_inlay_hints(true) + .build() + .wait_until_workspaces_are_initialized(); + + server.open_text_document(foo, foo_content, 1); + + let hints = server + .inlay_hints_request(foo, Range::new(Position::new(0, 0), Position::new(5, 0))) + .unwrap(); + + let kinds: Vec<_> = hints + .iter() + .map(|hint| { + let label = match &hint.label { + lsp_types::Label::String(label) => label.clone(), + lsp_types::Label::InlayHintLabelPartList(parts) => { + parts.iter().map(|part| part.value.as_str()).collect() + } + }; + (label, hint.kind, hint.data.clone()) + }) + .collect(); + + assert_eq!( + kinds, + vec![ + ( + "self,".to_string(), + Some(lsp_types::InlayHintKind::Parameter), + Some(serde_json::json!({ "kind": "implicitSelf" })) + ), + ( + "-> 1".to_string(), + Some(lsp_types::InlayHintKind::Type), + Some(serde_json::json!({ "kind": "inferredReturnTypes" })) + ), + ] + ); + + Ok(()) +} + /// Tests that disabling variable types inlay hints works correctly. #[test] fn variable_inlay_hints_disabled() -> Result<()> { @@ -347,7 +418,10 @@ def get_a() -> A: }, "newText": ", A" } - ] + ], + "data": { + "kind": "variableTypes" + } } ] "#); diff --git a/crates/ty_wasm/src/lib.rs b/crates/ty_wasm/src/lib.rs index c801570226..751daa7092 100644 --- a/crates/ty_wasm/src/lib.rs +++ b/crates/ty_wasm/src/lib.rs @@ -1469,25 +1469,10 @@ pub enum InlayHintKind { impl From for InlayHintKind { fn from(kind: ty_ide::InlayHintKind) -> Self { - match kind { - ty_ide::InlayHintKind::Type - // basedpython: an inferred exception set is a type, like a return type - | ty_ide::InlayHintKind::Raises - | ty_ide::InlayHintKind::Variance - | ty_ide::InlayHintKind::Reification - | ty_ide::InlayHintKind::TypeArgument - | ty_ide::InlayHintKind::Override - | ty_ide::InlayHintKind::Reads - | ty_ide::InlayHintKind::Stability - | ty_ide::InlayHintKind::DerivedDeps - | ty_ide::InlayHintKind::Invalidates - | ty_ide::InlayHintKind::NumericPromotion - | ty_ide::InlayHintKind::RevealedType - | ty_ide::InlayHintKind::EnumValue => Self::Type, - ty_ide::InlayHintKind::CallArgumentName - | ty_ide::InlayHintKind::ImplicitParameter - | ty_ide::InlayHintKind::ImplicitArgument - | ty_ide::InlayHintKind::InheritedDefault => Self::Parameter, + if kind.is_parameter() { + Self::Parameter + } else { + Self::Type } } } diff --git a/docs/basedpython/features/editor.md b/docs/basedpython/features/editor.md index fa5c16de26..6162afef2b 100644 --- a/docs/basedpython/features/editor.md +++ b/docs/basedpython/features/editor.md @@ -229,6 +229,18 @@ on | `templateBindingTypes` | a django template `{% for %}` binding's element type | | `resolvedTemplates` | the file a django `{% extends %}` name resolves to | +every hint also carries its setting's name as `data.kind` — +`{"kind": "revealedTypes"}` — so an editor can show each kind in its own way, +or only while a key is held, without telling kinds apart by their text. lsp's +own `kind` is only `Type` or `Parameter`, and a revealed `list[int]` and a +call's `[int]` differ by a bracket + +a hint is shown or hidden by its own setting alone, but one setting also +changes what other hints say: in a `.by` file, `typeArgumentNames` names the +type arguments of the types that `variableTypes` and `callTypeArguments` hints +show, so `x = identity({"a": 1})` is hinted `: dict[Key=str, Value=int]` with it +on and `: dict[str, int]` with it off + ### keeping a hand-aligned block aligned a hint takes up room. drawn after the name, it pushes the rest of the line along From 1be82a07e26fb1a97d7a3fa13c0fb5aaee9446fe Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Fri, 25 Sep 2026 15:39:07 +1000 Subject: [PATCH 05/11] server: answer by/superMembers, the superclass members a class member overrides, from the override checks' own walk up the MRO, which they now share with it --- crates/ty_ide/src/lib.rs | 2 + crates/ty_ide/src/super_members.rs | 367 ++++++++++ .../src/types/ide_support.rs | 106 ++- .../ty_python_semantic/src/types/overrides.rs | 693 ++++++++++-------- crates/ty_server/src/server/api.rs | 3 + crates/ty_server/src/server/api/requests.rs | 2 + .../src/server/api/requests/super_members.rs | 125 ++++ crates/ty_server/tests/e2e/main.rs | 1 + crates/ty_server/tests/e2e/super_members.rs | 109 +++ docs/basedpython/features/editor.md | 19 + 10 files changed, 1110 insertions(+), 317 deletions(-) create mode 100644 crates/ty_ide/src/super_members.rs create mode 100644 crates/ty_server/src/server/api/requests/super_members.rs create mode 100644 crates/ty_server/tests/e2e/super_members.rs diff --git a/crates/ty_ide/src/lib.rs b/crates/ty_ide/src/lib.rs index 27b2adc4be..17c0c42601 100644 --- a/crates/ty_ide/src/lib.rs +++ b/crates/ty_ide/src/lib.rs @@ -35,6 +35,7 @@ mod selection_range; mod semantic_tokens; mod signature_help; mod stub_mapping; +mod super_members; mod symbols; mod syntax_outline; mod type_hierarchy; @@ -87,6 +88,7 @@ pub use semantic_tokens::{ semantic_tokens, }; pub use signature_help::{ParameterDetails, SignatureDetails, SignatureHelpInfo, signature_help}; +pub use super_members::{SuperMember, super_members}; pub use symbols::{FlatSymbols, HierarchicalSymbols, SymbolId, SymbolInfo, SymbolKind}; pub use syntax_outline::{ OutlineCall, OutlineClause, OutlineModifier, OutlineStatement, OutlineString, SyntaxOutline, diff --git a/crates/ty_ide/src/super_members.rs b/crates/ty_ide/src/super_members.rs new file mode 100644 index 0000000000..bd9efd1747 --- /dev/null +++ b/crates/ty_ide/src/super_members.rs @@ -0,0 +1,367 @@ +//! Finds the superclass members a class member overrides — the direction +//! [`crate::goto_implementation()`] does not go, which finds the members that +//! override one. +//! +//! What counts as an override is the override checks' own answer, through +//! [`overridden_members`], so a member goes to exactly what +//! `invalid-method-override` compares it with and `missing-override-decorator` +//! asks it to be marked as overriding. + +use ruff_db::parsed::parsed_module; +use ruff_python_ast::ExprRef; +use ruff_python_ast::name::Name; +use ruff_text_size::TextSize; +use ty_python_core::ProgramFile; +use ty_python_semantic::SemanticModel; +use ty_python_semantic::types::Type; +use ty_python_semantic::types::ide_support::{ClassMemberNode, overridden_members}; + +use crate::goto::{Definitions, GotoTarget, find_goto_target}; +use crate::{Db, HasNavigationTargets, NavigationTarget}; + +/// A superclass member that the member at a position overrides. +#[derive(Debug, Clone)] +pub struct SuperMember { + /// The member's name. + pub name: Name, + /// The superclass that declares it. + pub superclass: Name, + /// Where the superclass declares it — or, for a member the superclass + /// synthesizes rather than writes, the superclass itself. + pub target: NavigationTarget, + /// Whether the superclass synthesizes the member, as a dataclass does its + /// `__init__`, so that [`Self::target`] is the class. + pub synthesized: bool, +} + +/// The superclass members the class member declared at `offset` overrides +/// directly, in MRO order. +/// +/// `offset` is on the member's name where it is declared: the name of a `def`, +/// or a name a class body assigns or annotates. `None` when there is no such +/// member there; empty when there is one and it overrides nothing. +pub fn super_members( + db: &dyn Db, + file: ProgramFile<'_>, + offset: TextSize, +) -> Option> { + let module = parsed_module(db, file.python_file(db)).load(db); + let model = SemanticModel::new(db, file); + let (member, name) = match find_goto_target(&model, &module, offset)? { + GotoTarget::FunctionDef(function) => ( + ClassMemberNode::Function(function), + function.name.id.clone(), + ), + GotoTarget::Expression(ExprRef::Name(name)) if name.ctx.is_store() => { + (ClassMemberNode::Name(name), name.id.clone()) + } + _ => return None, + }; + + let env = model.program_environment(); + let members = overridden_members(&model, member)?; + Some( + members + .into_iter() + .filter_map(|member| { + let declared = Definitions::new(member.definitions) + .into_navigation_targets(db) + .into_iter() + .next(); + let synthesized = declared.is_none(); + let target = declared.or_else(|| { + Type::ClassLiteral(member.superclass) + .navigation_targets(db, &env) + .into_iter() + .next() + })?; + Some(SuperMember { + name: name.clone(), + superclass: member.superclass_name, + target, + synthesized, + }) + }) + .collect(), + ) +} + +#[cfg(test)] +mod tests { + use crate::tests::{CursorTest, cursor_test}; + use insta::assert_snapshot; + use ruff_db::source::source_text; + use ruff_source_file::LineIndex; + + impl CursorTest { + /// One line per member: `Superclass.name` and the line it is declared + /// on, `(synthesized)` when the target is the class. + fn super_members(&self) -> String { + let Some(members) = super::super_members( + &self.db, + self.program_file(self.cursor.file), + self.cursor.offset, + ) else { + return "None".to_string(); + }; + if members.is_empty() { + return "[]".to_string(); + } + members + .iter() + .map(|member| { + let file = member.target.file(); + let source = source_text(&self.db, file); + let line = LineIndex::from_source_text(&source) + .line_index(member.target.focus_range().start()); + format!( + "{}.{} {}:{}{}", + member.superclass, + member.name, + file.path(&self.db).as_str(), + line, + if member.synthesized { + " (synthesized)" + } else { + "" + }, + ) + }) + .collect::>() + .join("\n") + } + } + + #[test] + fn a_method_goes_to_the_method_it_overrides() { + let test = cursor_test( + " +class A: + def f(self) -> None: ... + +class B(A): + def f(self) -> None: ... +", + ); + assert_snapshot!(test.super_members(), @"A.f /main.py:3"); + } + + #[test] + fn two_levels_up_goes_to_the_nearer() { + let test = cursor_test( + " +class A: + def f(self) -> None: ... + +class B(A): + def f(self) -> None: ... + +class C(B): + def f(self) -> None: ... +", + ); + assert_snapshot!(test.super_members(), @"B.f /main.py:6"); + } + + #[test] + fn two_levels_up_past_a_class_that_does_not_declare_it() { + let test = cursor_test( + " +class A: + def f(self) -> None: ... + +class B(A): + pass + +class C(B): + def f(self) -> None: ... +", + ); + assert_snapshot!(test.super_members(), @"A.f /main.py:3"); + } + + #[test] + fn several_bases_that_each_declare_it_are_each_overridden_in_mro_order() { + let test = cursor_test( + " +class Root: + def f(self) -> None: ... + +class Left(Root): + def f(self) -> None: ... + +class Right(Root): + def f(self) -> None: ... + +class Both(Left, Right): + def f(self) -> None: ... +", + ); + assert_snapshot!(test.super_members(), @" + Left.f /main.py:6 + Right.f /main.py:9 + "); + } + + #[test] + fn a_diamond_whose_sides_do_not_declare_it_goes_to_the_root_once() { + let test = cursor_test( + " +class Root: + def f(self) -> None: ... + +class Left(Root): ... + +class Right(Root): ... + +class Both(Left, Right): + def f(self) -> None: ... +", + ); + assert_snapshot!(test.super_members(), @"Root.f /main.py:3"); + } + + #[test] + fn a_protocol_member() { + let test = cursor_test( + " +from typing import Protocol + +class Greeter(Protocol): + def greet(self) -> str: ... + +class English(Greeter): + def greet(self) -> str: + return 'hello' +", + ); + assert_snapshot!(test.super_members(), @"Greeter.greet /main.py:5"); + } + + #[test] + fn a_property_getter_goes_to_the_getter() { + let test = cursor_test( + " +class A: + @property + def x(self) -> int: + return 1 + + @x.setter + def x(self, value: int) -> None: ... + +class B(A): + @property + def x(self) -> int: + return 2 +", + ); + assert_snapshot!(test.super_members(), @"A.x /main.py:4"); + } + + #[test] + fn a_class_attribute() { + let test = cursor_test( + " +class A: + size: int = 1 + +class B(A): + size = 2 +", + ); + assert_snapshot!(test.super_members(), @"A.size /main.py:3"); + } + + #[test] + fn a_member_the_superclass_synthesizes_goes_to_the_superclass() { + let test = cursor_test( + " +from dataclasses import dataclass + +@dataclass +class A: + x: int + +class B(A): + def __init__(self) -> None: + super().__init__(1) +", + ); + assert_snapshot!(test.super_members(), @"A.__init__ /main.py:5 (synthesized)"); + } + + #[test] + fn a_method_that_overrides_nothing() { + let test = cursor_test( + " +class A: + def f(self) -> None: ... + +class B(A): + def g(self) -> None: ... +", + ); + assert_snapshot!(test.super_members(), @"[]"); + } + + #[test] + fn a_function_that_is_not_a_member() { + let test = cursor_test( + " +def f() -> None: ... +", + ); + assert_snapshot!(test.super_members(), @"None"); + } + + #[test] + fn basedpython_override_and_protocol_class() { + let test = CursorTest::builder() + .source( + "main.by", + " +protocol class Shape: + def area(self) -> float: ... + +class Square(Shape): + def __init__(self, side: float) -> None: + self.side = side + + override def area(self) -> float: + return self.side * self.side +", + ) + .build(); + assert_snapshot!(test.super_members(), @"Shape.area /main.by:3"); + } + + #[test] + fn a_private_member_overrides_nothing() { + let test = CursorTest::builder() + .source( + "main.by", + " +class A: + private def f(self) -> None: ... + +class B(A): + private def f(self) -> None: ... +", + ) + .build(); + assert_snapshot!(test.super_members(), @"[]"); + } + + #[test] + fn a_member_of_object() { + let test = cursor_test( + " +class A: + def __repr__(self) -> str: + return 'A' +", + ); + assert_snapshot!(test.super_members(), @"object.__repr__ stdlib/builtins.byi:93"); + } +} diff --git a/crates/ty_python_semantic/src/types/ide_support.rs b/crates/ty_python_semantic/src/types/ide_support.rs index 6c5afb02f4..a4ab09566b 100644 --- a/crates/ty_python_semantic/src/types/ide_support.rs +++ b/crates/ty_python_semantic/src/types/ide_support.rs @@ -22,16 +22,18 @@ use crate::types::implicit_names::{ImplicitNamePosition, implicit_name}; use crate::types::infer::{infer_definition_types, nearest_enclosing_function}; use crate::types::list_members::all_end_of_scope_members; use crate::types::literal::LiteralValueTypeKind; -use crate::types::overrides::is_constructor_like_method; +use crate::types::overrides::{ + OverriddenDeclaration, OverrideStep, is_constructor_like_method, overridden_declarations, +}; use crate::types::receivers; use crate::types::repeated_underscore::{UnderscoreLowering, UnderscoreRefusal}; use crate::types::signatures::{ Parameter, ParameterKind, ParametersKind, ReturnCallableTypeVarScope, Signature, }; use crate::types::{ - CallDunderError, CallableTypes, ClassBase, ClassLiteral, KnownClass, KnownFunction, KnownUnion, - PropertyAccessorRole, SpecialFormType, Type, TypeContext, TypeVarBoundOrConstraints, - TypeVarVariance, binding_type, + CallDunderError, CallableTypes, ClassBase, ClassLiteral, ClassType, KnownClass, KnownFunction, + KnownUnion, PropertyAccessorRole, SpecialFormType, Type, TypeContext, + TypeVarBoundOrConstraints, TypeVarVariance, binding_type, }; use crate::{Db, HasDefinition, HasType, ProgramEnvironment, SemanticModel}; use ruff_db::files::{File, FileRange}; @@ -2950,6 +2952,102 @@ pub fn inferred_override<'db>( .map(Type::from) } +/// A class-body member, as the tree spells it: a `def`, or a name a class body +/// assigns or declares. +#[derive(Debug, Clone, Copy)] +pub enum ClassMemberNode<'a> { + Function(&'a ast::StmtFunctionDef), + Name(&'a ast::ExprName), +} + +/// A superclass member that a class member overrides. +#[derive(Debug, Clone)] +pub struct OverriddenMember<'db> { + /// The superclass that declares the member. + pub superclass: ClassLiteral<'db>, + /// The superclass's name. + pub superclass_name: Name, + /// Where the superclass declares it. For a property, the accessors that play + /// the overriding `def`'s part when it plays one — a setter's is the + /// setter. Empty when the superclass synthesizes the member rather than + /// writing it, as a dataclass does its `__init__`. + pub definitions: Vec>, +} + +/// basedpython: the superclass members that the class member `member` +/// overrides directly, in MRO order, or `None` when `member` is not a member of +/// a class body. +/// +/// What counts as an override is the override checks' own walk up the MRO, +/// [`overridden_declarations`], so a member is said to override exactly what +/// `invalid-method-override` and `missing-override-decorator` hold it to. +/// "Directly" is what that walk finds less what is reached through what it +/// found first: a member overriding `B.f`, where `B.f` overrides `A.f`, +/// overrides `B.f` — `A.f` is `B.f`'s to override. A class with several bases +/// that each declare the member overrides each of them. +/// +/// Empty for a member that overrides nothing — including a `private` one, which +/// python mangles into a name of its own. +pub fn overridden_members<'db>( + model: &SemanticModel<'db>, + member: ClassMemberNode<'_>, +) -> Option>> { + let db = model.db(); + let env = model.program_environment(); + let (definition, name) = match member { + ClassMemberNode::Function(function) => (function.definition(model), &function.name.id), + ClassMemberNode::Name(name) => ( + semantic_index(db, db.program_file(model.file())).try_definition(name)?, + &name.id, + ), + }; + + let scope = definition.scope(db); + let class_node = scope.node(db).as_class()?; + let class_definition = + semantic_index(db, scope.program_file(db)).expect_single_definition(class_node); + let class = extract_class_literal(db, &env, binding_type(db, class_definition))? + .default_specialization(db); + + // a getter overrides a getter and a setter a setter + let accessor_role = matches!(definition.kind(db), DefinitionKind::Function(_)) + .then(|| { + binding_type(db, definition) + .as_property_instance() + .and_then(|property| property.accessor_role(db, definition)) + }) + .flatten(); + + let mut members: Vec> = Vec::new(); + let mut found: Vec> = Vec::new(); + for step in overridden_declarations(db, &env, class, name) { + let OverrideStep::Overrides(OverriddenDeclaration { superclass, .. }) = step else { + continue; + }; + // reached through a member already found, which overrides it in turn + if found + .iter() + .any(|nearer| nearer.is_subclass_of(db, &env, superclass)) + { + continue; + } + found.push(superclass); + let superclass = superclass.class_literal(db); + let definitions = own_member_definitions(db, superclass, name, accessor_role) + .filter(|definitions| !definitions.is_empty()) + // a setter overriding a property whose superclass has only a getter + // overrides that property all the same + .or_else(|| own_member_definitions(db, superclass, name, None)) + .unwrap_or_default(); + members.push(OverriddenMember { + superclass, + superclass_name: superclass.name(db).clone(), + definitions, + }); + } + Some(members) +} + /// The enum members `target` admits under their bare name, each as its name /// and type. /// diff --git a/crates/ty_python_semantic/src/types/overrides.rs b/crates/ty_python_semantic/src/types/overrides.rs index 1cb093a5de..2a8245ff8c 100644 --- a/crates/ty_python_semantic/src/types/overrides.rs +++ b/crates/ty_python_semantic/src/types/overrides.rs @@ -437,6 +437,134 @@ fn conflicting_named_tuple_field_in_mro<'db>( None } +/// a superclass's declaration of a member that a class's own member of the same +/// name overrides +pub(crate) struct OverriddenDeclaration<'db> { + /// the superclass that declares it + pub(crate) superclass: ClassType<'db>, + /// the superclass's body scope + pub(crate) scope: ScopeId<'db>, + /// the member's symbol in [`Self::scope`], or `None` when the superclass does + /// not write the member in its body but synthesizes it, as a dataclass does + /// its `__init__` + pub(crate) symbol: Option, + /// whether the superclass synthesizes the member, and as what kind of class + pub(crate) method_kind: MethodKind<'db>, + /// the member's type on an instance of the superclass + pub(crate) ty: Type<'db>, +} + +/// one step of the walk [`overridden_declarations`] makes up a class's MRO +pub(crate) enum OverrideStep<'db> { + /// a base whose members cannot be known, `Any`, an unknown or a divergent + /// base, so anything may be declared there + Dynamic, + /// a `TypedDict` base, whose keys are not members a class body overrides + TypedDict, + /// a superclass that declares the member + Overrides(OverriddenDeclaration<'db>), +} + +/// every superclass declaration the member `name` of `class` overrides, in MRO +/// order, with the dynamic and `TypedDict` bases met along the way +/// +/// this is what the override checks mean by one member overriding another, and +/// the one place that says it: the checks walk it, and so do the IDE's answers +/// about overrides, where the members a member overrides are and the `override` +/// inlay hint +/// +/// a superclass counts when its body binds or declares `name`, or when it +/// synthesizes it. the walk ends at the first superclass that does so without an +/// instance of it having the member, and `Protocol` and `Generic` are passed +/// over +pub(crate) fn overridden_declarations<'db>( + db: &'db dyn Db, + env: &ProgramEnvironment<'db>, + class: ClassType<'db>, + name: &Name, +) -> Vec> { + let mut steps = Vec::new(); + + // basedpython: a `private` member is emitted under a name python mangles per + // class, so it is a member of its own rather than an override of anything. a + // `protected` one keeps one name across the hierarchy, so it overrides like + // any other member + if is_private_to(db, class, name) { + return steps; + } + + for class_base in class.iter_mro(db).skip(1) { + let superclass = match class_base { + ClassBase::Protocol | ClassBase::Generic => continue, + ClassBase::Any | ClassBase::Dynamic(_) | ClassBase::Divergent(_) => { + steps.push(OverrideStep::Dynamic); + continue; + } + ClassBase::TypedDict(_) => { + steps.push(OverrideStep::TypedDict); + continue; + } + ClassBase::Class(class) => class, + }; + + // basedpython: a `private` member of a superclass is emitted under a + // name mangled with that superclass's own, so nothing a subclass + // declares can override it. it is not a contract for this member to + // meet, and the subclass's member is not missing an `@override` + if is_private_to(db, superclass, name) { + continue; + } + + let Some((superclass_literal, superclass_specialization)) = + superclass.static_class_literal(db) + else { + continue; + }; + let superclass_scope = superclass_literal.body_scope(db); + let superclass_symbol_table = place_table(db, superclass_scope); + let superclass_symbol_id = superclass_symbol_table.symbol_id(name); + + let mut method_kind = MethodKind::default(); + + // If the member is not defined on the class itself, skip it + if let Some(id) = superclass_symbol_id { + let superclass_symbol = superclass_symbol_table.symbol(id); + if !(superclass_symbol.is_bound() || superclass_symbol.is_declared()) { + continue; + } + } else { + if superclass_literal + .own_synthesized_member(db, env, superclass_specialization, None, name) + .is_none() + { + continue; + } + method_kind = CodeGeneratorKind::from_class(db, superclass_literal.into()) + .map(MethodKind::Synthesized) + .unwrap_or_default(); + } + + let superclass_instance_member = Type::instance(db, env, superclass).member(db, env, name); + let Place::Defined(DefinedPlace { + ty: superclass_type, + .. + }) = superclass_instance_member.place + else { + // If not defined on any superclass, no point in continuing to walk up the MRO + break; + }; + + steps.push(OverrideStep::Overrides(OverriddenDeclaration { + superclass, + scope: superclass_scope, + symbol: superclass_symbol_id, + method_kind, + ty: superclass_type, + })); + } + steps +} + fn check_class_declaration<'db>( context: &InferContext<'db, '_>, configuration: OverrideRulesConfig, @@ -645,12 +773,6 @@ fn check_class_declaration<'db>( let mut missing_override_target: Option> = None; let mut overridden_final_method = None; let mut overridden_final_variable: Option<(ClassType<'db>, Option>)> = None; - // basedpython: a `private` member is emitted under a name python mangles per - // class, so it is a member of its own rather than an override of anything. a - // `protected` one keeps one name across the hierarchy, so it overrides like - // any other member - let is_private_member = is_private_to(db, class, &member.name); - // basedpython: a member emitted under a different name from the member it // inherits under the same written name does not override it — it sits // beside it, and the inherited one still answers. that is a keyword declaring @@ -685,353 +807,298 @@ fn check_class_declaration<'db>( let mut immediate_parent_method: Option<(ClassType<'db>, Type<'db>)> = None; let mut immediate_parent_variable_kind: Option<(ClassType<'db>, VariableKind)> = None; - if !is_private_member { - for class_base in class.iter_mro(db).skip(1) { - let superclass = match class_base { - ClassBase::Protocol | ClassBase::Generic => continue, - ClassBase::Any | ClassBase::Dynamic(_) => { - has_dynamic_superclass = true; - continue; - } - ClassBase::Divergent(_) => { - has_dynamic_superclass = true; - continue; - } - ClassBase::TypedDict(_) => { - has_typeddict_in_mro = true; - continue; - } - ClassBase::Class(class) => class, - }; - - // basedpython: a `private` member of a superclass is emitted under a - // name mangled with that superclass's own, so nothing a subclass - // declares can override it. it is not a contract for this member to - // meet, and the subclass's member is not missing an `@override` - if is_private_to(db, superclass, &member.name) { + for step in overridden_declarations(db, env, class, &member.name) { + let OverriddenDeclaration { + superclass, + scope: superclass_scope, + symbol: superclass_symbol_id, + method_kind, + ty: superclass_type, + } = match step { + OverrideStep::Dynamic => { + has_dynamic_superclass = true; continue; } - - let Some((superclass_literal, superclass_specialization)) = - superclass.static_class_literal(db) - else { + OverrideStep::TypedDict => { + has_typeddict_in_mro = true; continue; - }; - let superclass_scope = superclass_literal.body_scope(db); - let superclass_symbol_table = place_table(db, superclass_scope); - let superclass_symbol_id = superclass_symbol_table.symbol_id(&member.name); - - let mut method_kind = MethodKind::default(); - - // If the member is not defined on the class itself, skip it - if let Some(id) = superclass_symbol_id { - let superclass_symbol = superclass_symbol_table.symbol(id); - if !(superclass_symbol.is_bound() || superclass_symbol.is_declared()) { - continue; - } - } else { - if superclass_literal - .own_synthesized_member(db, env, superclass_specialization, None, &member.name) - .is_none() - { - continue; - } - method_kind = CodeGeneratorKind::from_class(db, superclass_literal.into()) - .map(MethodKind::Synthesized) - .unwrap_or_default(); } + OverrideStep::Overrides(declaration) => declaration, + }; - let superclass_instance_member = - Type::instance(db, env, superclass).member(db, env, &member.name); - let Place::Defined(DefinedPlace { - ty: superclass_type, - .. - }) = superclass_instance_member.place - else { - // If not defined on any superclass, no point in continuing to walk up the MRO - break; - }; + subclass_overrides_superclass_declaration = true; - subclass_overrides_superclass_declaration = true; + // Record the first overridden superclass member that is subject to the missing override + // decorator check so that we can later confirm that the overriding definition is indeed + // marked with the decorator. + if configuration.check_missing_overrides() + && missing_override_target.is_none() + && !is_constructor_like_method(&member.name) + { + missing_override_target = Some(MissingOverrideTarget::for_superclass( + db, + superclass, + superclass_scope, + superclass_symbol_id, + )); + } - // Record the first overridden superclass member that is subject to the missing override - // decorator check so that we can later confirm that the overriding definition is indeed - // marked with the decorator. - if configuration.check_missing_overrides() - && missing_override_target.is_none() - && !is_constructor_like_method(&member.name) - { - missing_override_target = Some(MissingOverrideTarget::for_superclass( - db, - superclass, - superclass_scope, - superclass_symbol_id, - )); - } + // Record the first superclass that defines this method as the "immediate parent method" + if immediate_parent_method.is_none() { + immediate_parent_method = Some((superclass, superclass_type)); + } - // Record the first superclass that defines this method as the "immediate parent method" - if immediate_parent_method.is_none() { - immediate_parent_method = Some((superclass, superclass_type)); - } + if (configuration.check_final_method_overridden() && overridden_final_method.is_none()) + || (configuration.check_final_variable_overridden() + && overridden_final_variable.is_none()) + { + let own_class_member = superclass.own_class_member(db, env, None, &member.name); - if (configuration.check_final_method_overridden() && overridden_final_method.is_none()) - || (configuration.check_final_variable_overridden() - && overridden_final_variable.is_none()) - { - let own_class_member = superclass.own_class_member(db, env, None, &member.name); - - if configuration.check_final_method_overridden() { - overridden_final_method = overridden_final_method.or_else(|| { - let superclass_symbol_id = superclass_symbol_id?; - - // TODO: `@final` should be more like a type qualifier: - // we should also recognise `@final`-decorated methods that don't end up - // as being function- or property-types (because they're wrapped by other - // decorators that transform the type into something else). - let underlying_functions = extract_underlying_functions( - db, - own_class_member.ignore_possibly_undefined()?, - ); + if configuration.check_final_method_overridden() { + overridden_final_method = overridden_final_method.or_else(|| { + let superclass_symbol_id = superclass_symbol_id?; - if underlying_functions.iter().any(|function| { - function.has_known_decorator(db, FunctionDecorators::FINAL) - }) && is_function_definition(db, superclass_scope, superclass_symbol_id) - { - Some((superclass, underlying_functions)) - } else { - None - } - }); - } + // TODO: `@final` should be more like a type qualifier: + // we should also recognise `@final`-decorated methods that don't end up + // as being function- or property-types (because they're wrapped by other + // decorators that transform the type into something else). + let underlying_functions = extract_underlying_functions( + db, + own_class_member.ignore_possibly_undefined()?, + ); - if configuration.check_final_variable_overridden() { - overridden_final_variable = overridden_final_variable.or_else(|| { - if !own_class_member - .qualifiers() - .contains(TypeQualifiers::FINAL) - { - return None; - } + if underlying_functions + .iter() + .any(|function| function.has_known_decorator(db, FunctionDecorators::FINAL)) + && is_function_definition(db, superclass_scope, superclass_symbol_id) + { + Some((superclass, underlying_functions)) + } else { + None + } + }); + } - // `let` is read-only, not `final`. but models a - // read-only property, so overriding it is allowed. - if superclass_symbol_id - .is_some_and(|id| *is_let_declaration(db, superclass_scope, id)) - { - return None; - } + if configuration.check_final_variable_overridden() { + overridden_final_variable = overridden_final_variable.or_else(|| { + if !own_class_member + .qualifiers() + .contains(TypeQualifiers::FINAL) + { + return None; + } - // Find the declaration definition in the superclass for the secondary - // annotation. - let superclass_definition = superclass_symbol_id.and_then(|id| { - use_def_map(db, superclass_scope) - .end_of_scope_symbol_declarations(id) - .find_map(|decl| decl.declaration.definition()) - }); + // `let` is read-only, not `final`. but models a + // read-only property, so overriding it is allowed. + if superclass_symbol_id + .is_some_and(|id| *is_let_declaration(db, superclass_scope, id)) + { + return None; + } - Some((superclass, superclass_definition)) + // Find the declaration definition in the superclass for the secondary + // annotation. + let superclass_definition = superclass_symbol_id.and_then(|id| { + use_def_map(db, superclass_scope) + .end_of_scope_symbol_declarations(id) + .find_map(|decl| decl.declaration.definition()) }); - } - } - // ********************************************************** - // Everything below this point in the loop - // is about Liskov Substitution Principle checks - // ********************************************************** - - // Only one Liskov diagnostic should be emitted per each invalid override, - // even if it overrides multiple superclasses incorrectly! - if liskov_diagnostic_emitted { - continue; + Some((superclass, superclass_definition)) + }); } + } - if !configuration.check_liskov_violations() { - continue; - } + // ********************************************************** + // Everything below this point in the loop + // is about Liskov Substitution Principle checks + // ********************************************************** - if configuration.check_attribute_liskov_violations() { - if let Some(superclass_variable_kind) = - effective_superclass_variable_kind(db, superclass, member.name.clone()) - { - if immediate_parent_variable_kind.is_none() { - immediate_parent_variable_kind = - Some((superclass, superclass_variable_kind)); - } + // Only one Liskov diagnostic should be emitted per each invalid override, + // even if it overrides multiple superclasses incorrectly! + if liskov_diagnostic_emitted { + continue; + } - let subclass_kind = *subclass_variable_kind.get_or_insert_with(|| { - variable_kind( - db, - env, - class.own_class_member(db, env, None, &member.name).inner, - subclass_instance_member, - ) - }); + if !configuration.check_liskov_violations() { + continue; + } - if let Some(subclass_kind) = subclass_kind - && subclass_kind != superclass_variable_kind - { - // An unannotated class-body assignment can inherit an overridden `ClassVar` - // declaration instead of introducing a conflicting instance variable. This - // also applies to augmented assignments after the initial class-body - // assignment, e.g. `epilog = "..."; epilog += "..."`. - if subclass_kind == VariableKind::Instance - && superclass_variable_kind == VariableKind::Class - && matches!( - first_reachable_definition.kind(db), - DefinitionKind::Assignment(_) - | DefinitionKind::AugmentedAssignment(_) - ) - { - continue; - } + if configuration.check_attribute_liskov_violations() { + if let Some(superclass_variable_kind) = + effective_superclass_variable_kind(db, superclass, member.name.clone()) + { + if immediate_parent_variable_kind.is_none() { + immediate_parent_variable_kind = Some((superclass, superclass_variable_kind)); + } - if let Some((immediate_parent, immediate_parent_kind)) = - immediate_parent_variable_kind - && immediate_parent != superclass - && immediate_parent.is_subclass_of(db, env, superclass) - && immediate_parent_kind != superclass_variable_kind - { - continue; - } + let subclass_kind = *subclass_variable_kind.get_or_insert_with(|| { + variable_kind( + db, + env, + class.own_class_member(db, env, None, &member.name).inner, + subclass_instance_member, + ) + }); - let superclass_definition = superclass_symbol_id - .and_then(|id| symbol_definition(db, superclass_scope, id)); - report_invalid_attribute_override( - context, - &member.name, - *first_reachable_definition, - superclass, - superclass_definition, - subclass_kind, - superclass_variable_kind, - ); - liskov_diagnostic_emitted = true; + if let Some(subclass_kind) = subclass_kind + && subclass_kind != superclass_variable_kind + { + // An unannotated class-body assignment can inherit an overridden `ClassVar` + // declaration instead of introducing a conflicting instance variable. This + // also applies to augmented assignments after the initial class-body + // assignment, e.g. `epilog = "..."; epilog += "..."`. + if subclass_kind == VariableKind::Instance + && superclass_variable_kind == VariableKind::Class + && matches!( + first_reachable_definition.kind(db), + DefinitionKind::Assignment(_) | DefinitionKind::AugmentedAssignment(_) + ) + { continue; } - } - } - - if !configuration.check_method_liskov_violations() { - continue; - } - let Type::FunctionLiteral(subclass_function) = member.ty else { - continue; - }; - - // Constructor methods are not checked for Liskov compliance - if is_constructor_like_method(&member.name) { - continue; - } - - // Synthesized `__replace__` methods on dataclasses are not checked - if &member.name == "__replace__" - && class_kind.is_some_and(CodeGeneratorKind::is_dataclass_like) - { - continue; - } + if let Some((immediate_parent, immediate_parent_kind)) = + immediate_parent_variable_kind + && immediate_parent != superclass + && immediate_parent.is_subclass_of(db, env, superclass) + && immediate_parent_kind != superclass_variable_kind + { + continue; + } - // basedpython: a reified type-parameter list is part of the - // method's runtime interface — `a.f[int]()` through the base - // dispatches to the override's wrapper. the value-level callable - // check below cannot see it (type parameters are phantom there), - // so check list compatibility first - let superclass_function = match superclass_type { - Type::FunctionLiteral(function) => Some(function), - Type::BoundMethod(bound_method) => Some(bound_method.function(db)), - _ => None, - }; - if let Some(superclass_function) = superclass_function - && let Some(error) = crate::types::reified_infer::reified_override_error( - db, - env, - superclass_function, - subclass_function, - ) - { - report_invalid_reified_override( - context, - &member.name, - *first_reachable_definition, - subclass_function, - superclass, - error, - ); - liskov_diagnostic_emitted = true; - continue; + let superclass_definition = superclass_symbol_id + .and_then(|id| symbol_definition(db, superclass_scope, id)); + report_invalid_attribute_override( + context, + &member.name, + *first_reachable_definition, + superclass, + superclass_definition, + subclass_kind, + superclass_variable_kind, + ); + liskov_diagnostic_emitted = true; + continue; + } } + } - // basedpython: an override may not raise more than what it - // overrides. orthogonal to Liskov compatibility below, so it runs - // whether or not the signatures line up - // only against the nearest superclass defining the method: a more - // distant ancestor's bound is already enforced on that superclass, - // and reporting it again here blames the wrong class - if let Some(superclass_function) = superclass_function - && immediate_parent_method.is_some_and(|(parent, _)| parent == superclass) - { - crate::types::exceptions::check_override_raises( - context, - &member.name, - subclass_function, - superclass_function, - superclass, - ); - } + if !configuration.check_method_liskov_violations() { + continue; + } - let Some((subclass_override_type, superclass_override_type)) = - method_override_types(db, env, type_on_subclass_instance, superclass_type) - else { - continue; - }; + let Type::FunctionLiteral(subclass_function) = member.ty else { + continue; + }; - if subclass_override_type.is_assignable_to(db, env, superclass_override_type) { - continue; - } + // Constructor methods are not checked for Liskov compliance + if is_constructor_like_method(&member.name) { + continue; + } - // If this superclass is not the immediate parent for this method, - // check if the immediate parent itself already has an LSP violation with this ancestor. - // If so, don't report the same violation for the child class -- it would be a false positive - // since the child cannot fix the violation without contradicting its immediate parent's contract. - // See: https://github.com/astral-sh/ty/issues/2000 - if let Some((immediate_parent, immediate_parent_type)) = immediate_parent_method { - if immediate_parent != superclass { - // The immediate parent already defines this method and is different from the - // current ancestor we're checking. Check if the immediate parent's method - // is also incompatible with this ancestor. - if !is_assignable_method_override( - db, - env, - immediate_parent_type, - superclass_type, - ) { - // The immediate parent already has an LSP violation with this ancestor. - // Don't report the same violation for the child. - continue; - } - } - } + // Synthesized `__replace__` methods on dataclasses are not checked + if &member.name == "__replace__" + && class_kind.is_some_and(CodeGeneratorKind::is_dataclass_like) + { + continue; + } - report_invalid_method_override( + // basedpython: a reified type-parameter list is part of the + // method's runtime interface — `a.f[int]()` through the base + // dispatches to the override's wrapper. the value-level callable + // check below cannot see it (type parameters are phantom there), + // so check list compatibility first + let superclass_function = match superclass_type { + Type::FunctionLiteral(function) => Some(function), + Type::BoundMethod(bound_method) => Some(bound_method.function(db)), + _ => None, + }; + if let Some(superclass_function) = superclass_function + && let Some(error) = crate::types::reified_infer::reified_override_error( + db, + env, + superclass_function, + subclass_function, + ) + { + report_invalid_reified_override( context, &member.name, - class, *first_reachable_definition, subclass_function, superclass, - superclass_type, - method_kind, - || { - subclass_override_type.assignability_error_context( - db, - env, - superclass_override_type, - ) - }, + error, ); - liskov_diagnostic_emitted = true; + continue; } + + // basedpython: an override may not raise more than what it + // overrides. orthogonal to Liskov compatibility below, so it runs + // whether or not the signatures line up + // only against the nearest superclass defining the method: a more + // distant ancestor's bound is already enforced on that superclass, + // and reporting it again here blames the wrong class + if let Some(superclass_function) = superclass_function + && immediate_parent_method.is_some_and(|(parent, _)| parent == superclass) + { + crate::types::exceptions::check_override_raises( + context, + &member.name, + subclass_function, + superclass_function, + superclass, + ); + } + + let Some((subclass_override_type, superclass_override_type)) = + method_override_types(db, env, type_on_subclass_instance, superclass_type) + else { + continue; + }; + + if subclass_override_type.is_assignable_to(db, env, superclass_override_type) { + continue; + } + + // If this superclass is not the immediate parent for this method, + // check if the immediate parent itself already has an LSP violation with this ancestor. + // If so, don't report the same violation for the child class -- it would be a false positive + // since the child cannot fix the violation without contradicting its immediate parent's contract. + // See: https://github.com/astral-sh/ty/issues/2000 + if let Some((immediate_parent, immediate_parent_type)) = immediate_parent_method { + if immediate_parent != superclass { + // The immediate parent already defines this method and is different from the + // current ancestor we're checking. Check if the immediate parent's method + // is also incompatible with this ancestor. + if !is_assignable_method_override(db, env, immediate_parent_type, superclass_type) { + // The immediate parent already has an LSP violation with this ancestor. + // Don't report the same violation for the child. + continue; + } + } + } + + report_invalid_method_override( + context, + &member.name, + class, + *first_reachable_definition, + subclass_function, + superclass, + superclass_type, + method_kind, + || { + subclass_override_type.assignability_error_context( + db, + env, + superclass_override_type, + ) + }, + ); + + liskov_diagnostic_emitted = true; } if !subclass_overrides_superclass_declaration && !has_dynamic_superclass { @@ -1508,7 +1575,7 @@ fn report_invalid_attribute_override<'db>( } #[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] -pub(super) enum MethodKind<'db> { +pub(crate) enum MethodKind<'db> { Synthesized(CodeGeneratorKind<'db>), #[default] NotSynthesized, diff --git a/crates/ty_server/src/server/api.rs b/crates/ty_server/src/server/api.rs index af51453fae..dfdb193e6c 100644 --- a/crates/ty_server/src/server/api.rs +++ b/crates/ty_server/src/server/api.rs @@ -204,6 +204,9 @@ pub(super) fn request(req: server::Request) -> Task { BackgroundSchedule::LatencySensitive, ) } + requests::SuperMembersRequestHandler::METHOD => background_document_request_task::< + requests::SuperMembersRequestHandler, + >(req, BackgroundSchedule::Worker), requests::PrepareTypeHierarchyRequestHandler::METHOD => background_document_request_task::< requests::PrepareTypeHierarchyRequestHandler, >( diff --git a/crates/ty_server/src/server/api/requests.rs b/crates/ty_server/src/server/api/requests.rs index 3d8f17a186..8aba1eca2c 100644 --- a/crates/ty_server/src/server/api/requests.rs +++ b/crates/ty_server/src/server/api/requests.rs @@ -48,6 +48,7 @@ mod semantic_tokens; mod semantic_tokens_range; mod shutdown; mod signature_help; +mod super_members; mod syntax_outline; mod test_items; mod transpile; @@ -93,6 +94,7 @@ pub(super) use semantic_tokens::SemanticTokensRequestHandler; pub(super) use semantic_tokens_range::SemanticTokensRangeRequestHandler; pub(super) use shutdown::ShutdownHandler; pub(super) use signature_help::SignatureHelpRequestHandler; +pub(super) use super_members::SuperMembersRequestHandler; pub(super) use syntax_outline::SyntaxOutlineRequestHandler; pub(super) use test_items::TestItemsHandler; pub(super) use transpile::TranspileRequestHandler; diff --git a/crates/ty_server/src/server/api/requests/super_members.rs b/crates/ty_server/src/server/api/requests/super_members.rs new file mode 100644 index 0000000000..ad24e6681b --- /dev/null +++ b/crates/ty_server/src/server/api/requests/super_members.rs @@ -0,0 +1,125 @@ +//! `by/superMembers` — the superclass members a class member overrides. +//! +//! What an editor's *Go to Super* asks of a method or an attribute, and what an "overrides" gutter +//! marker points at. LSP has no request for it: `textDocument/implementation` goes the other way, +//! from a member to the members that override it, and `typeHierarchy/supertypes` answers about +//! classes, not their members. +//! +//! The answer is the override checks' own: a member is said to override exactly the superclass +//! members `invalid-method-override` compares it with — the nearest along each branch of the MRO, +//! in MRO order, so a member of a class with several bases can override several. + +use std::borrow::Cow; + +use lsp_types::{ + LspRequestMethod, MessageDirection, Position, Range, Request, TextDocumentIdentifier, Uri, +}; +use ty_ide::super_members; +use ty_project::{ProjectDatabase, SemanticDb as _}; + +use crate::document::{PositionExt, ToLink}; +use crate::server::api::traits::{ + BackgroundDocumentRequestHandler, RequestHandler, RetriableRequestHandler, +}; +use crate::session::DocumentSnapshot; +use crate::session::client::Client; + +pub(crate) enum SuperMembersRequest {} + +impl Request for SuperMembersRequest { + type Params = SuperMembersParams; + type Result = Option>; + const METHOD: LspRequestMethod<'static> = LspRequestMethod::Custom("by/superMembers"); + const MESSAGE_DIRECTION: MessageDirection = MessageDirection::ClientToServer; +} + +/// Which member to ask about: the document, and a position on the member's name where it is +/// declared — the name of a `def`, or a name a class body assigns or annotates. +#[derive(Debug, serde::Deserialize, serde::Serialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub(crate) struct SuperMembersParams { + text_document: TextDocumentIdentifier, + position: Position, +} + +/// One superclass member the member overrides. +#[derive(Debug, serde::Deserialize, serde::Serialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct SuperMember { + /// The member's name. + name: String, + /// The superclass that declares it. + container_name: String, + /// The file the superclass declares it in. + uri: Uri, + /// The whole declaration. + range: Range, + /// The declared name, which is where to go. + selection_range: Range, + /// Whether the superclass synthesizes the member rather than writing it, as a dataclass does + /// its `__init__`. The ranges are then the superclass's own. + synthesized: bool, +} + +pub(crate) struct SuperMembersRequestHandler; + +impl RequestHandler for SuperMembersRequestHandler { + type RequestType = SuperMembersRequest; +} + +impl BackgroundDocumentRequestHandler for SuperMembersRequestHandler { + fn document_uri(params: &SuperMembersParams) -> Cow<'_, Uri> { + Cow::Borrowed(¶ms.text_document.uri) + } + + /// `null` when there is no class member declared at the position; an empty list when there is + /// one and it overrides nothing. + fn run_with_snapshot( + db: &ProjectDatabase, + snapshot: &DocumentSnapshot, + _client: &Client, + params: SuperMembersParams, + ) -> crate::server::Result>> { + if snapshot + .workspace_settings() + .is_language_services_disabled() + { + return Ok(None); + } + + let Some(file) = snapshot.to_notebook_or_file(db) else { + return Ok(None); + }; + + let Some(offset) = + params + .position + .to_text_size(db, file, snapshot.uri(), snapshot.encoding()) + else { + return Ok(None); + }; + + let Some(members) = super_members(db, db.program_file(file), offset) else { + return Ok(None); + }; + + Ok(Some( + members + .into_iter() + .filter_map(|member| { + let link = member.target.to_link(db, None, snapshot.encoding())?; + Some(SuperMember { + name: member.name.to_string(), + container_name: member.superclass.to_string(), + uri: link.target_uri, + range: link.target_range, + selection_range: link.target_selection_range, + synthesized: member.synthesized, + }) + }) + .collect(), + )) + } +} + +impl RetriableRequestHandler for SuperMembersRequestHandler {} diff --git a/crates/ty_server/tests/e2e/main.rs b/crates/ty_server/tests/e2e/main.rs index c1579475bf..b921c90cf6 100644 --- a/crates/ty_server/tests/e2e/main.rs +++ b/crates/ty_server/tests/e2e/main.rs @@ -56,6 +56,7 @@ mod rename; mod script_preparation; mod semantic_tokens; mod signature_help; +mod super_members; mod syntax_outline; mod transpile; mod type_hierarchy; diff --git a/crates/ty_server/tests/e2e/super_members.rs b/crates/ty_server/tests/e2e/super_members.rs new file mode 100644 index 0000000000..1040fc5f24 --- /dev/null +++ b/crates/ty_server/tests/e2e/super_members.rs @@ -0,0 +1,109 @@ +//! `by/superMembers` over the wire +//! +//! which members a member overrides is `ty_ide`'s and its unit tests cover it. what these cover is +//! the contract a client reads: a position on a declared name in, and out the members in MRO order, +//! each in the file that declares it with the name to land on — and `null` kept apart from an empty +//! list, which a client tells a user two different things about + +use anyhow::Result; +use lsp_types::{LspRequestMethod, MessageDirection, Request}; +use ruff_db::system::SystemPath; +use ty_server::ClientOptions; + +use crate::{TestServer, TestServerBuilder}; + +/// the request as a client sends it, in json throughout +enum SuperMembers {} + +impl Request for SuperMembers { + type Params = serde_json::Value; + type Result = Option; + const METHOD: LspRequestMethod<'static> = LspRequestMethod::Custom("by/superMembers"); + const MESSAGE_DIRECTION: MessageDirection = MessageDirection::ClientToServer; +} + +const BASES: &str = "\ +class Root: + def speak(self) -> str: + return '' + +class Loud(Root): + def speak(self) -> str: + return 'HEY' + +class Quiet(Root): + def speak(self) -> str: + return 'hey' +"; + +const MAIN: &str = "\ +from bases import Loud, Quiet + +class Both(Loud, Quiet): + override def speak(self) -> str: + return 'hi' + + def listen(self) -> None: ... + +def free() -> None: ... +"; + +fn server() -> Result { + let mut server = TestServerBuilder::new()? + .with_initialization_options(&ClientOptions::default()) + .with_workspace(SystemPath::new("src"), None)? + .with_file(SystemPath::new("src/bases.by"), BASES)? + .with_file(SystemPath::new("src/main.by"), MAIN)? + .build() + .wait_until_workspaces_are_initialized(); + server.open_text_document(SystemPath::new("src/main.by"), MAIN, 1); + Ok(server) +} + +fn ask(server: &mut TestServer, line: u32, character: u32) -> Option { + let uri = server.file_uri(SystemPath::new("src/main.by")); + server.send_request_await::(serde_json::json!({ + "textDocument": { "uri": uri }, + "position": { "line": line, "character": character }, + })) +} + +#[test] +fn a_method_of_a_class_with_two_bases_overrides_each_in_mro_order() -> Result<()> { + let mut server = server()?; + let bases = server.file_uri(SystemPath::new("src/bases.by")); + + // on `speak` in `override def speak` + let answer = ask(&mut server, 3, 17).expect("a member is declared there"); + let members = answer.as_array().expect("a list of members"); + assert_eq!(members.len(), 2, "{answer:#}"); + + for (member, (container, line)) in members.iter().zip([("Loud", 5), ("Quiet", 9)]) { + assert_eq!(member["name"], "speak"); + assert_eq!(member["containerName"], container); + assert_eq!(member["uri"], bases.as_str()); + assert_eq!(member["synthesized"], false); + assert_eq!( + member["selectionRange"], + serde_json::json!({ + "start": { "line": line, "character": 8 }, + "end": { "line": line, "character": 13 }, + }), + ); + assert_eq!(member["range"]["start"]["line"], line); + } + Ok(()) +} + +#[test] +fn a_member_that_overrides_nothing_is_an_empty_list_and_no_member_is_null() -> Result<()> { + let mut server = server()?; + + // on `listen` + assert_eq!(ask(&mut server, 6, 9), Some(serde_json::json!([]))); + // on `free`, a function no class declares + assert_eq!(ask(&mut server, 8, 5), None); + // inside a method body rather than on a declared name + assert_eq!(ask(&mut server, 4, 9), None); + Ok(()) +} diff --git a/docs/basedpython/features/editor.md b/docs/basedpython/features/editor.md index 6162afef2b..b36803c31c 100644 --- a/docs/basedpython/features/editor.md +++ b/docs/basedpython/features/editor.md @@ -387,6 +387,25 @@ the pairing is read off the parse tree, so `match = 1` is an assignment and nothing lights up on it. a position that is not a keyword is answered the way it always was +## go to super + +`textDocument/implementation` on a method goes down, to the methods that +override it. `by/superMembers` goes up: given a position on the name a class +member is declared with — a `def`, or a name the class body assigns or +annotates — it answers the superclass members that member overrides, each with +its class, its file and the name to land on. a class goes up with +`typeHierarchy/supertypes` + +what counts as an override is the override checks' own walk up the MRO, so a +member goes to exactly what `invalid-method-override` compares it with: the +nearest declaration along each branch, in MRO order. a method overriding `B.f`, +where `B.f` overrides `A.f`, goes to `B.f`; a class with two bases that each +declare the method goes to both. a `private` member overrides nothing, since +python mangles it into a name of its own, and a member a superclass synthesizes +rather than writes — a dataclass's `__init__` — goes to that superclass, marked +`synthesized`. the answer is `null` when no class member is declared at the +position, and an empty list for one that overrides nothing + ## the program model a run configuration, a test list and a "go to generated file" are all questions From 7f8768836d1dda0ac83efa0a78e14714552d95a4 Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Fri, 25 Sep 2026 20:11:45 +1000 Subject: [PATCH 06/11] server: answer by/documentSuperMembers, every class member of a document that overrides something with what by/superMembers says it overrides, from the same code in one request per document, so that an editor can draw its overrides markers on every pass; say of each overridden member, and of each overriding one, whether it is abstract, so that a member implementing an abstract one can be told apart; and correct a test that meant a protocol but wrote a syntax error --- crates/ty_ide/src/inlay_hints.rs | 27 +- crates/ty_ide/src/lib.rs | 2 +- ...inferred_override_of_a_private_member.snap | 10 + crates/ty_ide/src/super_members.rs | 646 ++++++++++++++++-- crates/ty_python_semantic/src/types/class.rs | 64 +- .../src/types/ide_support.rs | 318 ++++++--- .../ty_python_semantic/src/types/overrides.rs | 8 +- crates/ty_server/src/server/api.rs | 5 + crates/ty_server/src/server/api/requests.rs | 2 + .../api/requests/document_super_members.rs | 137 ++++ .../src/server/api/requests/super_members.rs | 41 +- crates/ty_server/tests/e2e/super_members.rs | 187 ++++- docs/basedpython/features/editor.md | 21 +- 13 files changed, 1242 insertions(+), 226 deletions(-) create mode 100644 crates/ty_ide/src/snapshots/ty_ide__inlay_hints__tests__basedpython_inferred_override_of_a_private_member.snap create mode 100644 crates/ty_server/src/server/api/requests/document_super_members.rs diff --git a/crates/ty_ide/src/inlay_hints.rs b/crates/ty_ide/src/inlay_hints.rs index f4e0a6db48..ebd6bf52c5 100644 --- a/crates/ty_ide/src/inlay_hints.rs +++ b/crates/ty_ide/src/inlay_hints.rs @@ -1659,14 +1659,7 @@ impl<'a, 'db> InlayHintVisitor<'a, 'db> { return; } - let Some(class_ty) = self.enclosing_class else { - return; - }; - - let Some(superclass) = function - .inferred_type(&self.model) - .and_then(|ty| inferred_override(self.db, env, class_ty, ty, &function.name)) - else { + let Some(superclass) = inferred_override(&self.model, function) else { return; }; @@ -11251,6 +11244,24 @@ Source with applied edits: })); } + #[test] + fn basedpython_inferred_override_of_a_private_member() { + let mut test = basedpython_inlay_hint_test( + " + class A: + private def f(self) -> None: ... + + class B(A): + private def f(self) -> None: ... + ", + ); + + assert_snapshot!(test.inlay_hints_with_settings(&InlayHintSettings { + inferred_override: true, + ..InlayHintSettings::none() + })); + } + /// The mock of the `basedpython_ui` framework the ui hints are tested /// against: its observables, slot functions and the builders the examples /// use, developed in place as a first-party package — which is how the diff --git a/crates/ty_ide/src/lib.rs b/crates/ty_ide/src/lib.rs index 17c0c42601..c56ecf5245 100644 --- a/crates/ty_ide/src/lib.rs +++ b/crates/ty_ide/src/lib.rs @@ -88,7 +88,7 @@ pub use semantic_tokens::{ semantic_tokens, }; pub use signature_help::{ParameterDetails, SignatureDetails, SignatureHelpInfo, signature_help}; -pub use super_members::{SuperMember, super_members}; +pub use super_members::{OverridingMember, SuperMember, document_super_members, super_members}; pub use symbols::{FlatSymbols, HierarchicalSymbols, SymbolId, SymbolInfo, SymbolKind}; pub use syntax_outline::{ OutlineCall, OutlineClause, OutlineModifier, OutlineStatement, OutlineString, SyntaxOutline, diff --git a/crates/ty_ide/src/snapshots/ty_ide__inlay_hints__tests__basedpython_inferred_override_of_a_private_member.snap b/crates/ty_ide/src/snapshots/ty_ide__inlay_hints__tests__basedpython_inferred_override_of_a_private_member.snap new file mode 100644 index 0000000000..c4e63c82d3 --- /dev/null +++ b/crates/ty_ide/src/snapshots/ty_ide__inlay_hints__tests__basedpython_inferred_override_of_a_private_member.snap @@ -0,0 +1,10 @@ +--- +source: crates/ty_ide/src/inlay_hints.rs +expression: "test.inlay_hints_with_settings(&InlayHintSettings\n{ inferred_override: true, ..InlayHintSettings::none() })" +--- + +class A: + private def f(self) -> None: ... + +class B(A): + private def f(self) -> None: ... diff --git a/crates/ty_ide/src/super_members.rs b/crates/ty_ide/src/super_members.rs index bd9efd1747..30324f17c4 100644 --- a/crates/ty_ide/src/super_members.rs +++ b/crates/ty_ide/src/super_members.rs @@ -1,89 +1,186 @@ -//! Finds the superclass members a class member overrides — the direction +//! finds the superclass members a class member overrides: the direction //! [`crate::goto_implementation()`] does not go, which finds the members that -//! override one. +//! override one //! -//! What counts as an override is the override checks' own answer, through -//! [`overridden_members`], so a member goes to exactly what -//! `invalid-method-override` compares it with and `missing-override-decorator` -//! asks it to be marked as overriding. +//! what counts as an override is the override checks' own walk up the MRO, +//! through [`overridden_members`]. of the declarations that walk finds, a member +//! goes to the nearest along each branch of the MRO: the first of them is what +//! `missing-override-decorator` names, and `invalid-method-override` compares +//! the member with each declaration the walk finds, these and the ones they +//! override in turn +//! +//! asked two ways, from the same code: of one member, at a position on its name +//! ([`super_members`]), which is what *Go to Super* asks; and of every member a +//! document declares at once ([`document_super_members`]), which is what an +//! editor drawing an "overrides" marker beside each one asks, on every pass over +//! the document -use ruff_db::parsed::parsed_module; -use ruff_python_ast::ExprRef; use ruff_python_ast::name::Name; -use ruff_text_size::TextSize; +use ruff_text_size::{TextRange, TextSize}; use ty_python_core::ProgramFile; -use ty_python_semantic::SemanticModel; use ty_python_semantic::types::Type; -use ty_python_semantic::types::ide_support::{ClassMemberNode, overridden_members}; +use ty_python_semantic::types::ide_support::{ + ClassMemberDefinition, OverriddenMember, class_member_definitions, is_constructor_like_method, + overridden_members, +}; +use ty_python_semantic::{ProgramEnvironment, SemanticModel}; -use crate::goto::{Definitions, GotoTarget, find_goto_target}; +use crate::goto::Definitions; use crate::{Db, HasNavigationTargets, NavigationTarget}; -/// A superclass member that the member at a position overrides. +/// a superclass member that a class member overrides #[derive(Debug, Clone)] pub struct SuperMember { - /// The member's name. + /// the member's name pub name: Name, - /// The superclass that declares it. + /// the superclass that declares it pub superclass: Name, - /// Where the superclass declares it — or, for a member the superclass - /// synthesizes rather than writes, the superclass itself. + /// where the superclass declares it, or, for a member the superclass + /// synthesizes rather than writes, the superclass itself pub target: NavigationTarget, - /// Whether the superclass synthesizes the member, as a dataclass does its - /// `__init__`, so that [`Self::target`] is the class. + /// whether the superclass synthesizes the member, as a dataclass does its + /// `__init__`, so that [`Self::target`] is the class pub synthesized: bool, + /// whether the member is abstract where the superclass declares it: an + /// `@abstractmethod`, or a protocol method with no implementation + pub is_abstract: bool, +} + +/// a class member a document declares that overrides something, and what it +/// overrides +#[derive(Debug, Clone)] +pub struct OverridingMember { + /// the member's name + pub name: Name, + /// the class whose body declares it + pub class: Name, + /// its name where it is declared: the name of a `def` or a nested class, a + /// name a class body assigns, annotates or captures, or an import's alias + pub name_range: TextRange, + /// whether the member is itself abstract, in the sense of + /// [`SuperMember::is_abstract`] + pub is_abstract: bool, + /// what it overrides directly, in MRO order. never empty + pub super_members: Vec, } -/// The superclass members the class member declared at `offset` overrides -/// directly, in MRO order. +/// the superclass members the class member declared at `offset` overrides +/// directly, in MRO order /// -/// `offset` is on the member's name where it is declared: the name of a `def`, -/// or a name a class body assigns or annotates. `None` when there is no such -/// member there; empty when there is one and it overrides nothing. +/// `offset` is on the member's name where it is declared, as +/// [`OverridingMember::name_range`] has it. `None` when there is no such member +/// there; empty when there is one and it overrides nothing pub fn super_members( db: &dyn Db, file: ProgramFile<'_>, offset: TextSize, ) -> Option> { - let module = parsed_module(db, file.python_file(db)).load(db); - let model = SemanticModel::new(db, file); - let (member, name) = match find_goto_target(&model, &module, offset)? { - GotoTarget::FunctionDef(function) => ( - ClassMemberNode::Function(function), - function.name.id.clone(), - ), - GotoTarget::Expression(ExprRef::Name(name)) if name.ctx.is_store() => { - (ClassMemberNode::Name(name), name.id.clone()) + let declared: Vec<_> = class_member_definitions(db, file) + .into_iter() + .filter(|member| { + !member.name_range.is_empty() && member.name_range.contains_inclusive(offset) + }) + .collect(); + if declared.is_empty() { + return None; + } + let env = SemanticModel::new(db, file).program_environment(); + Some(overriding_member(db, &env, &declared)?.super_members) +} + +/// every class member `file` declares that overrides something, with what it +/// overrides, in source order: [`super_members`] of each, from one pass over the +/// document +/// +/// a member declared more than once is listed at each declaration that names it +/// in the source, as each is answered: a python property's getter and its setter +/// each override their own accessor. a declaration that stands for several, as a +/// basedpython property with `get` and `set` blocks stands for its getter and its +/// setter under the one name, is listed once, with what any of them overrides +/// +/// constructors and the methods they call (`__init__`, `__new__`, +/// `__post_init__`, `__init_subclass__`) are left out: every class's `__init__` +/// overrides `object.__init__`, which says nothing about any of them, and +/// neither `invalid-method-override` nor `missing-override-decorator` holds one +/// to what it overrides. [`super_members`] still answers one when asked +pub fn document_super_members(db: &dyn Db, file: ProgramFile<'_>) -> Vec { + let env = SemanticModel::new(db, file).program_environment(); + class_member_definitions(db, file) + // the definitions come in source order, so the ones naming the same text + // are next to each other + .chunk_by(|left, right| left.name_range == right.name_range) + .filter(|declared| { + declared.first().is_some_and(|member| { + !member.name_range.is_empty() && !is_constructor_like_method(&member.name) + }) + }) + .filter_map(|declared| overriding_member(db, &env, declared)) + .filter(|member| !member.super_members.is_empty()) + .collect() +} + +/// what the definitions `declared`, which all name the same member at the same +/// place in the source, override between them. `None` when `declared` is empty +fn overriding_member( + db: &dyn Db, + env: &ProgramEnvironment<'_>, + declared: &[ClassMemberDefinition<'_>], +) -> Option { + let first = declared.first()?; + let mut is_abstract = false; + let mut overridden: Vec> = Vec::new(); + for member in declared { + let overrides = overridden_members(db, member); + is_abstract |= overrides.is_abstract; + for member in overrides.overridden { + if !overridden + .iter() + .any(|found| found.superclass == member.superclass) + { + overridden.push(member); + } } - _ => return None, - }; - - let env = model.program_environment(); - let members = overridden_members(&model, member)?; - Some( - members - .into_iter() - .filter_map(|member| { - let declared = Definitions::new(member.definitions) - .into_navigation_targets(db) - .into_iter() - .next(); - let synthesized = declared.is_none(); - let target = declared.or_else(|| { + } + Some(OverridingMember { + name: first.name.clone(), + class: first.class_name.clone(), + name_range: first.name_range, + is_abstract, + super_members: targets(db, env, &first.name, overridden), + }) +} + +/// where each of `overridden`, the members named `name` a member overrides, is +/// declared, or the superclass itself for one with no declaration to go to. one +/// with neither is left out +fn targets( + db: &dyn Db, + env: &ProgramEnvironment<'_>, + name: &Name, + overridden: Vec>, +) -> Vec { + overridden + .into_iter() + .filter_map(|member| { + let target = Definitions::new(member.definitions) + .into_navigation_targets(db) + .into_iter() + .next() + .or_else(|| { Type::ClassLiteral(member.superclass) - .navigation_targets(db, &env) + .navigation_targets(db, env) .into_iter() .next() })?; - Some(SuperMember { - name: name.clone(), - superclass: member.superclass_name, - target, - synthesized, - }) + Some(SuperMember { + name: name.clone(), + superclass: member.superclass_name, + target, + synthesized: member.synthesized, + is_abstract: member.is_abstract, }) - .collect(), - ) + }) + .collect() } #[cfg(test)] @@ -94,8 +191,9 @@ mod tests { use ruff_source_file::LineIndex; impl CursorTest { - /// One line per member: `Superclass.name` and the line it is declared - /// on, `(synthesized)` when the target is the class. + /// one line per member: `Superclass.name` and the line it is declared + /// on, `(synthesized)` when the superclass synthesizes it, `(abstract)` + /// when the member is abstract there fn super_members(&self) -> String { let Some(members) = super::super_members( &self.db, @@ -107,24 +205,73 @@ mod tests { if members.is_empty() { return "[]".to_string(); } + members + .iter() + .map(|member| self.super_member(member)) + .collect::>() + .join("\n") + } + + fn super_member(&self, member: &super::SuperMember) -> String { + let file = member.target.file(); + let source = source_text(&self.db, file); + let line = LineIndex::from_source_text(&source) + .line_index(member.target.focus_range().start()); + format!( + "{}.{} {}:{}{}{}", + member.superclass, + member.name, + file.path(&self.db).as_str(), + line, + if member.synthesized { + " (synthesized)" + } else { + "" + }, + if member.is_abstract { + " (abstract)" + } else { + "" + }, + ) + } + + /// one line per overriding member of the cursor's file: its name, the + /// line and column its name is on, `(abstract)` when it is abstract + /// itself, and after `->` what it overrides, as [`Self::super_members`] + /// writes each + fn document_super_members(&self) -> String { + let file = self.cursor.file; + let source = source_text(&self.db, file); + let index = LineIndex::from_source_text(&source); + let members = + super::document_super_members(&self.db, self.program_file(self.cursor.file)); + if members.is_empty() { + return "[]".to_string(); + } members .iter() .map(|member| { - let file = member.target.file(); - let source = source_text(&self.db, file); - let line = LineIndex::from_source_text(&source) - .line_index(member.target.focus_range().start()); + let start = member.name_range.start(); + let line = index.line_index(start); + let column = start - index.line_start(line, &source); format!( - "{}.{} {}:{}{}", - member.superclass, + "{}.{} {}:{}{} -> {}", + member.class, member.name, - file.path(&self.db).as_str(), line, - if member.synthesized { - " (synthesized)" + column.to_u32(), + if member.is_abstract { + " (abstract)" } else { "" }, + member + .super_members + .iter() + .map(|member| self.super_member(member)) + .collect::>() + .join(", "), ) }) .collect::>() @@ -132,6 +279,142 @@ mod tests { } } + #[test] + fn an_abstract_method_is_said_to_be_abstract() { + let test = cursor_test( + " +from abc import ABC, abstractmethod + +class A(ABC): + @abstractmethod + def f(self) -> None: ... + +class B(A): + def f(self) -> None: + pass +", + ); + assert_snapshot!(test.super_members(), @"A.f /main.py:6 (abstract)"); + } + + #[test] + fn a_document_lists_each_member_that_overrides_something() { + let test = cursor_test( + " +from typing import Protocol + +class Greeter(Protocol): + def greet(self) -> str: ... + +class A: + size: int = 1 + + def f(self) -> None: ... + + @property + def x(self) -> int: + return 1 + + @x.setter + def x(self, value: int) -> None: ... + +class B(A): + size = 2 + + def __init__(self) -> None: ... + + def f(self) -> None: ... + + def g(self) -> None: ... + + @property + def x(self) -> int: + return 2 + + @x.setter + def x(self, value: int) -> None: ... + +class C(B, Greeter): + def f(self) -> None: + f = 1 + class Inner(A): + def f(self) -> None: ... + + def greet(self) -> str: + return 'hi' + + names = [f for f in 'ab'] +", + ); + assert_snapshot!(test.document_super_members(), @" + B.size 20:4 -> A.size /main.py:8 + B.f 24:8 -> A.f /main.py:10 + B.x 29:8 -> A.x /main.py:13 + B.x 33:8 -> A.x /main.py:17 + C.f 36:8 -> B.f /main.py:24 + Inner.f 39:16 -> A.f /main.py:10 + C.greet 41:8 -> Greeter.greet /main.py:5 (abstract) + "); + } + + #[test] + fn a_document_member_that_overrides_through_two_bases_lists_both() { + let test = cursor_test( + " +class Root: + def f(self) -> None: ... + +class Left(Root): + def f(self) -> None: ... + +class Right(Root): + def f(self) -> None: ... + +class Both(Left, Right): + def f(self) -> None: ... +", + ); + assert_snapshot!(test.document_super_members(), @" + Left.f 6:8 -> Root.f /main.py:3 + Right.f 9:8 -> Root.f /main.py:3 + Both.f 12:8 -> Left.f /main.py:6, Right.f /main.py:9 + "); + } + + #[test] + fn a_protocol_member_redeclared_in_a_protocol_is_abstract_itself() { + let test = cursor_test( + " +from typing import Protocol + +class Shape(Protocol): + def area(self) -> float: ... + +class Polygon(Shape, Protocol): + def area(self) -> float: ... +", + ); + assert_snapshot!( + test.document_super_members(), + @"Polygon.area 8:8 (abstract) -> Shape.area /main.py:5 (abstract)" + ); + } + + #[test] + fn a_document_with_no_overrides_is_empty() { + let test = cursor_test( + " +class A: + def __init__(self) -> None: ... + + def f(self) -> None: ... + +def g() -> None: ... +", + ); + assert_snapshot!(test.document_super_members(), @"[]"); + } + #[test] fn a_method_goes_to_the_method_it_overrides() { let test = cursor_test( @@ -235,7 +518,7 @@ class English(Greeter): return 'hello' ", ); - assert_snapshot!(test.super_members(), @"Greeter.greet /main.py:5"); + assert_snapshot!(test.super_members(), @"Greeter.greet /main.py:5 (abstract)"); } #[test] @@ -316,12 +599,12 @@ def f() -> None: ... } #[test] - fn basedpython_override_and_protocol_class() { + fn basedpython_override_and_protocol() { let test = CursorTest::builder() .source( "main.by", " -protocol class Shape: +protocol Shape: def area(self) -> float: ... class Square(Shape): @@ -333,7 +616,7 @@ class Square(Shape): ", ) .build(); - assert_snapshot!(test.super_members(), @"Shape.area /main.by:3"); + assert_snapshot!(test.super_members(), @"Shape.area /main.by:3 (abstract)"); } #[test] @@ -353,6 +636,225 @@ class B(A): assert_snapshot!(test.super_members(), @"[]"); } + /// a decorator can replace a class with anything, and the members are the + /// class's own all the same: those of the class as written, not of what the + /// decorator returns + #[test] + fn a_member_of_a_decorated_class() { + let test = cursor_test( + " +class Base: + def f(self) -> None: ... + +class Other(Base): ... + +def to_int(cls: type) -> int: + return 1 + +def to_other(cls: type) -> Other: + return Other() + +class A: + def f(self) -> None: ... + +@to_int +class C(A): + def f(self) -> None: ... + +@to_other +class D(A): + def f(self) -> None: ... +", + ); + assert_snapshot!(test.document_super_members(), @" + C.f 18:8 -> A.f /main.py:14 + D.f 22:8 -> A.f /main.py:14 + "); + } + + #[test] + fn an_accessor_block_is_listed_once() { + let test = CursorTest::builder() + .source( + "main.by", + " +class A: + var age: int = 0 + get() = field + set(value): + field = value + +class B(A): + override var age: int = 0 + get() = field + set(value): + field = value +", + ) + .build(); + assert_snapshot!(test.document_super_members(), @"B.age 9:17 -> A.age /main.by:3"); + } + + #[test] + fn an_accessor_block_is_asked_about_once() { + let test = CursorTest::builder() + .source( + "main.by", + " +class A: + var age: int = 0 + get() = field + set(value): + field = value + +class B(A): + override var age: int = 0 + get() = field + set(value): + field = value +", + ) + .build(); + assert_snapshot!(test.super_members(), @"A.age /main.by:3"); + } + + #[test] + fn a_setter_overriding_a_property_with_only_a_getter_goes_to_the_getter() { + let test = cursor_test( + " +class A: + @property + def x(self) -> int: + return 1 + +class B(A): + @property + def x(self) -> int: + return 2 + + @x.setter + def x(self, value: int) -> None: ... +", + ); + assert_snapshot!(test.document_super_members(), @" + B.x 9:8 -> A.x /main.py:4 + B.x 13:8 -> A.x /main.py:4 + "); + } + + #[test] + fn a_property_overriding_an_attribute() { + let test = cursor_test( + " +class A: + x: int = 1 + +class B(A): + @property + def x(self) -> int: + return 2 +", + ); + assert_snapshot!(test.document_super_members(), @"B.x 7:8 -> A.x /main.py:3"); + } + + /// a base whose members cannot be known is passed over, and the walk goes + /// on to the bases after it + #[test] + fn a_dynamic_base_is_passed_over() { + let test = cursor_test( + " +from typing import Any + +class A: + def f(self) -> None: ... + +class B(Any, A): + def f(self) -> None: ... +", + ); + assert_snapshot!(test.document_super_members(), @"B.f 8:8 -> A.f /main.py:5"); + } + + /// the keys of a typed dict are not members a class body overrides + #[test] + fn a_typed_dict_key_overrides_nothing() { + let test = cursor_test( + " +from typing import TypedDict + +class A(TypedDict): + x: int + +class B(A): + x: int +", + ); + assert_snapshot!(test.document_super_members(), @"[]"); + } + + #[test] + fn a_member_of_a_generic_class() { + let test = cursor_test( + " +class A[T]: + def f(self, x: T) -> T: + return x + +class B(A[int]): + def f(self, x: int) -> int: + return x + +class C[T](A[T]): + def f(self, x: T) -> T: + return x +", + ); + assert_snapshot!(test.document_super_members(), @" + B.f 7:8 -> A.f /main.py:3 + C.f 11:8 -> A.f /main.py:3 + "); + } + + #[test] + fn a_nested_class_an_import_and_a_walrus_in_a_decorator_are_members() { + let test = cursor_test( + " +import os + +def deco(f: object) -> object: + return f + +class A: + import os + class Meta: ... + d = deco + match 1: + case captured: ... + + @deco + def f(self) -> None: ... + +class B(A): + import os + class Meta: ... + + @(d := deco) + def f(self) -> None: ... + + match 1: + case captured: ... +", + ); + assert_snapshot!(test.document_super_members(), @" + B.os 18:11 -> A.os /main.py:8 + B.Meta 19:10 -> A.Meta /main.py:9 + B.d 21:6 -> A.d /main.py:10 + B.f 22:8 -> A.f /main.py:15 + B.captured 25:13 -> A.captured /main.py:12 + "); + } + #[test] fn a_member_of_object() { let test = cursor_test( diff --git a/crates/ty_python_semantic/src/types/class.rs b/crates/ty_python_semantic/src/types/class.rs index 0de48e08cd..6f28c12346 100644 --- a/crates/ty_python_semantic/src/types/class.rs +++ b/crates/ty_python_semantic/src/types/class.rs @@ -1478,36 +1478,6 @@ impl<'db> ClassType<'db> { self, db: &'db dyn Db, ) -> FxIndexMap> { - fn type_as_abstract_method<'db>( - db: &'db dyn Db, - ty: Type<'db>, - defining_class: ClassType<'db>, - ) -> Option { - match ty { - Type::FunctionLiteral(function) => function.as_abstract_method(db, defining_class), - Type::BoundMethod(method) => { - method.function(db).as_abstract_method(db, defining_class) - } - Type::PropertyInstance(property) => { - // A property is abstract if any of its accessors is abstract. - property - .getter(db) - .and_then(|getter| type_as_abstract_method(db, getter, defining_class)) - .or_else(|| { - property.setter(db).and_then(|setter| { - type_as_abstract_method(db, setter, defining_class) - }) - }) - .or_else(|| { - property.deleter(db).and_then(|deleter| { - type_as_abstract_method(db, deleter, defining_class) - }) - }) - } - _ => None, - } - } - let mut abstract_methods: FxIndexMap = FxIndexMap::default(); let env = &ProgramEnvironment::from_file(self.class_literal(db).program_file(db)); @@ -3577,3 +3547,37 @@ pub(super) enum MetaclassErrorKind<'db> { /// A cycle was encountered attempting to determine the metaclass Cycle, } + +/// whether `ty`, the type a class body binds a member to in `defining_class`, is an abstract +/// method, and why: an `@abstractmethod`, or a protocol method with no implementation. a property +/// is abstract when any of its accessors is +/// +/// what `abstract-instantiation` counts as abstract, and what an editor asks when it says whether a +/// member implements what it overrides or overrides an implementation +pub(in crate::types) fn type_as_abstract_method<'db>( + db: &'db dyn Db, + ty: Type<'db>, + defining_class: ClassType<'db>, +) -> Option { + match ty { + Type::FunctionLiteral(function) => function.as_abstract_method(db, defining_class), + Type::BoundMethod(method) => method.function(db).as_abstract_method(db, defining_class), + Type::PropertyInstance(property) => { + // A property is abstract if any of its accessors is abstract. + property + .getter(db) + .and_then(|getter| type_as_abstract_method(db, getter, defining_class)) + .or_else(|| { + property + .setter(db) + .and_then(|setter| type_as_abstract_method(db, setter, defining_class)) + }) + .or_else(|| { + property + .deleter(db) + .and_then(|deleter| type_as_abstract_method(db, deleter, defining_class)) + }) + } + _ => None, + } +} diff --git a/crates/ty_python_semantic/src/types/ide_support.rs b/crates/ty_python_semantic/src/types/ide_support.rs index a4ab09566b..936e09032b 100644 --- a/crates/ty_python_semantic/src/types/ide_support.rs +++ b/crates/ty_python_semantic/src/types/ide_support.rs @@ -8,7 +8,9 @@ use crate::reachability::is_range_reachable; use crate::types::EnumLiteralType; use crate::types::call::bind::CheckTypesMode; use crate::types::call::{CallArguments, CallError, MatchedArgument}; -use crate::types::class::{DynamicClassAnchor, DynamicEnumAnchor, DynamicNamedTupleAnchor}; +use crate::types::class::{ + DynamicClassAnchor, DynamicEnumAnchor, DynamicNamedTupleAnchor, type_as_abstract_method, +}; use crate::types::constraints::ConstraintSetBuilder; use crate::types::context_sensitive::for_each_candidate; use crate::types::dedicated::django; @@ -19,12 +21,12 @@ use crate::types::extensions::{ use crate::types::function::FunctionDecorators; use crate::types::generics::GenericContext; use crate::types::implicit_names::{ImplicitNamePosition, implicit_name}; -use crate::types::infer::{infer_definition_types, nearest_enclosing_function}; +use crate::types::infer::{ + infer_definition_types, nearest_enclosing_function, original_class_type, +}; use crate::types::list_members::all_end_of_scope_members; use crate::types::literal::LiteralValueTypeKind; -use crate::types::overrides::{ - OverriddenDeclaration, OverrideStep, is_constructor_like_method, overridden_declarations, -}; +use crate::types::overrides::{OverriddenDeclaration, OverrideStep, overridden_declarations}; use crate::types::receivers; use crate::types::repeated_underscore::{UnderscoreLowering, UnderscoreRefusal}; use crate::types::signatures::{ @@ -47,7 +49,8 @@ use ty_module_resolver::{ ImportingFile, Module, ModuleName, ResolverFile, resolve_module_confident, }; use ty_python_core::definition::{Definition, DefinitionKind}; -use ty_python_core::scope::FileScopeId; +use ty_python_core::scope::{FileScopeId, ScopeId}; +use ty_python_core::symbol::ScopedSymbolId; use ty_python_core::{ProgramFile, attribute_scopes, semantic_index, use_def_map}; mod data_flow; @@ -59,6 +62,7 @@ use crate::types::definition_resolution::{ self, find_symbol_in_scope, resolve_definition, user_visible_definitions, }; pub use crate::types::definition_resolution::{ImportAliasResolution, ResolvedDefinition}; +pub use crate::types::overrides::is_constructor_like_method; pub use data_flow::{ConditionVerdict, DataFlow, ValueVerdict, data_flow}; pub use stub_mapping::map_stub_definition; pub use unreachable_code::{UnreachableKind, UnreachableRange, unreachable_ranges}; @@ -2911,117 +2915,175 @@ pub fn implicit_enum_member_value<'db>( metadata.value_type(db, env, &name) } -/// basedpython: the superclass whose member `name` the class member `member` -/// overrides, when `member` is a method that is not already marked `override`. +/// basedpython: the superclass whose member the method `function` overrides, +/// when it is a method that is not already marked `override` /// -/// Mirrors the `missing-override-decorator` lint's notion of an override, so the -/// hint appears exactly where writing `override` would silence that lint. +/// the first superclass the override checks' walk up the MRO, +/// `overridden_declarations`, finds, which is the one `missing-override-decorator` +/// names, so the hint appears exactly where writing `override` would silence +/// that lint pub fn inferred_override<'db>( - db: &'db dyn Db, - env: &ProgramEnvironment<'db>, - class: Type<'db>, - member: Type<'db>, - name: &str, + model: &SemanticModel<'db>, + function: &ast::StmtFunctionDef, ) -> Option> { - let Type::FunctionLiteral(function) = member else { + let db = model.db(); + let env = model.program_environment(); + let definition = function.definition(model); + let Type::FunctionLiteral(function_type) = binding_type(db, definition) else { return None; }; - if function.has_known_decorator(db, FunctionDecorators::OVERRIDE) - || is_constructor_like_method(name) - || is_mangled_private(name) + if function_type.has_known_decorator(db, FunctionDecorators::OVERRIDE) + || is_constructor_like_method(&function.name) { return None; } - let Type::ClassLiteral(class) = class else { - return None; - }; - - class - .default_specialization(db) - .iter_mro(db) - .skip(1) - .filter_map(ClassBase::into_class) - .find(|superclass| { - !superclass - .own_class_member(db, env, None, name) - .inner - .place - .is_undefined() + let class = declaring_class(db, definition)?; + overridden_declarations(db, &env, class, &function.name.id) + .into_iter() + .find_map(|step| match step { + OverrideStep::Overrides(OverriddenDeclaration { superclass, .. }) => { + Some(Type::from(superclass)) + } + OverrideStep::Dynamic | OverrideStep::TypedDict => None, }) - .map(Type::from) } -/// A class-body member, as the tree spells it: a `def`, or a name a class body -/// assigns or declares. -#[derive(Debug, Clone, Copy)] -pub enum ClassMemberNode<'a> { - Function(&'a ast::StmtFunctionDef), - Name(&'a ast::ExprName), +/// basedpython: a binding or declaration a class body makes of one of the +/// class's members: a `def`, a nested class, an import, or any other name the +/// body binds or declares +#[derive(Debug, Clone)] +pub struct ClassMemberDefinition<'db> { + definition: Definition<'db>, + /// the member's name + pub name: Name, + /// the name of the class whose body makes the definition + pub class_name: Name, + /// where the definition names the member + pub name_range: TextRange, +} + +/// basedpython: every binding and declaration the class bodies of `file` make of +/// their members, as the semantic index records them, in source order +/// +/// these are the members the override checks walk: a binding or declaration in +/// code that is never reached makes no member +pub fn class_member_definitions<'db>( + db: &'db dyn Db, + file: ProgramFile<'db>, +) -> Vec> { + let index = semantic_index(db, file); + let module = parsed_module(db, file.python_file(db)).load(db); + let mut members = Vec::new(); + for scope in index.scope_ids() { + let Some(class_node) = scope.node(db).as_class() else { + continue; + }; + let class_name = &class_node.node(&module).name.id; + let file_scope = scope.file_scope_id(db); + let table = index.place_table(file_scope); + let mut seen = FxHashSet::default(); + for (symbol, declarations, bindings) in + index.use_def_map(file_scope).all_reachable_symbols() + { + let name = table.symbol(symbol).name(); + let definitions = declarations + .filter_map(|declaration| declaration.declaration.definition()) + .chain(bindings.filter_map(|binding| binding.binding.definition())); + for definition in definitions { + if !definition.kind(db).is_user_visible() || !seen.insert(definition) { + continue; + } + members.push(ClassMemberDefinition { + definition, + name: name.clone(), + class_name: class_name.clone(), + name_range: definition.focus_range(db, &module).range(), + }); + } + } + } + members.sort_by_key(|member| member.name_range.start()); + members +} + +/// basedpython: the superclass members a class member definition overrides +/// directly, and whether the member is itself abstract +#[derive(Debug, Clone, Default)] +pub struct MemberOverrides<'db> { + /// whether the member is itself abstract, in the sense of + /// [`OverriddenMember::is_abstract`]: an abstract member that overrides an + /// abstract one still leaves it to be implemented + pub is_abstract: bool, + /// what it overrides directly, in MRO order + pub overridden: Vec>, } -/// A superclass member that a class member overrides. +/// basedpython: a superclass member that a class member overrides #[derive(Debug, Clone)] pub struct OverriddenMember<'db> { - /// The superclass that declares the member. + /// the superclass that declares the member pub superclass: ClassLiteral<'db>, - /// The superclass's name. + /// the superclass's name pub superclass_name: Name, - /// Where the superclass declares it. For a property, the accessors that play - /// the overriding `def`'s part when it plays one — a setter's is the - /// setter. Empty when the superclass synthesizes the member rather than - /// writing it, as a dataclass does its `__init__`. + /// where the superclass's body writes it. for a property, the accessors that + /// play the overriding `def`'s part when any does: a setter's is the setter. + /// empty when the superclass synthesizes the member pub definitions: Vec>, + /// whether the superclass synthesizes the member rather than writing it in + /// its body, as a dataclass does its `__init__` + pub synthesized: bool, + /// whether the member is abstract where the superclass declares it, an + /// `@abstractmethod` or a protocol method with no implementation, so that a + /// member overriding it implements it rather than replacing an + /// implementation: what `abstract-instantiation` counts as abstract + pub is_abstract: bool, } -/// basedpython: the superclass members that the class member `member` -/// overrides directly, in MRO order, or `None` when `member` is not a member of -/// a class body. +/// basedpython: the superclass members that the class member `member` overrides +/// directly, in MRO order /// -/// What counts as an override is the override checks' own walk up the MRO, -/// [`overridden_declarations`], so a member is said to override exactly what -/// `invalid-method-override` and `missing-override-decorator` hold it to. -/// "Directly" is what that walk finds less what is reached through what it -/// found first: a member overriding `B.f`, where `B.f` overrides `A.f`, -/// overrides `B.f` — `A.f` is `B.f`'s to override. A class with several bases -/// that each declare the member overrides each of them. +/// what counts as an override is the override checks' own walk up the MRO, +/// `overridden_declarations`, from the class as the checks see it: +/// undecorated, and specialized to its own type parameters. "directly" is what +/// that walk finds less what is reached through what it found first: a member +/// overriding `B.f`, where `B.f` overrides `A.f`, overrides `B.f`, as `A.f` is +/// `B.f`'s to override. a class with several bases that each declare the member +/// overrides each of them /// -/// Empty for a member that overrides nothing — including a `private` one, which -/// python mangles into a name of its own. +/// empty for a member that overrides nothing, including a `private` one, which +/// python mangles into a name of its own pub fn overridden_members<'db>( - model: &SemanticModel<'db>, - member: ClassMemberNode<'_>, -) -> Option>> { - let db = model.db(); - let env = model.program_environment(); - let (definition, name) = match member { - ClassMemberNode::Function(function) => (function.definition(model), &function.name.id), - ClassMemberNode::Name(name) => ( - semantic_index(db, db.program_file(model.file())).try_definition(name)?, - &name.id, - ), + db: &'db dyn Db, + member: &ClassMemberDefinition<'db>, +) -> MemberOverrides<'db> { + let ClassMemberDefinition { + definition, name, .. + } = member; + let env = ProgramEnvironment::from_scope(definition.scope(db)); + let Some(class) = declaring_class(db, *definition) else { + return MemberOverrides::default(); }; - let scope = definition.scope(db); - let class_node = scope.node(db).as_class()?; - let class_definition = - semantic_index(db, scope.program_file(db)).expect_single_definition(class_node); - let class = extract_class_literal(db, &env, binding_type(db, class_definition))? - .default_specialization(db); - // a getter overrides a getter and a setter a setter - let accessor_role = matches!(definition.kind(db), DefinitionKind::Function(_)) - .then(|| { - binding_type(db, definition) - .as_property_instance() - .and_then(|property| property.accessor_role(db, definition)) - }) - .flatten(); + let accessor_role = if matches!(definition.kind(db), DefinitionKind::Function(_)) + && let Some(property) = binding_type(db, *definition).as_property_instance() + { + property.accessor_role(db, *definition) + } else { + None + }; - let mut members: Vec> = Vec::new(); + let mut overridden: Vec> = Vec::new(); let mut found: Vec> = Vec::new(); for step in overridden_declarations(db, &env, class, name) { - let OverrideStep::Overrides(OverriddenDeclaration { superclass, .. }) = step else { + let OverrideStep::Overrides(OverriddenDeclaration { + superclass, + scope, + symbol, + .. + }) = step + else { continue; }; // reached through a member already found, which overrides it in turn @@ -3032,20 +3094,86 @@ pub fn overridden_members<'db>( continue; } found.push(superclass); - let superclass = superclass.class_literal(db); - let definitions = own_member_definitions(db, superclass, name, accessor_role) - .filter(|definitions| !definitions.is_empty()) - // a setter overriding a property whose superclass has only a getter - // overrides that property all the same - .or_else(|| own_member_definitions(db, superclass, name, None)) + let is_abstract = is_abstract_in(db, &env, superclass, name); + let definitions = symbol + .map(|symbol| { + let written = written_member_definitions(db, scope, symbol, accessor_role); + if written.is_empty() { + // a setter overriding a property whose superclass has only a + // getter overrides that property all the same + written_member_definitions(db, scope, symbol, None) + } else { + written + } + }) .unwrap_or_default(); - members.push(OverriddenMember { + let superclass = superclass.class_literal(db); + overridden.push(OverriddenMember { superclass, superclass_name: superclass.name(db).clone(), definitions, + synthesized: symbol.is_none(), + is_abstract, }); } - Some(members) + MemberOverrides { + is_abstract: is_abstract_in(db, &env, class, name), + overridden, + } +} + +/// where a superclass's body, `scope`, writes its member `symbol`: each `def`, +/// nested class, import or name the body binds or declares it with, a `def` +/// with overloads as its implementation. for a property, only the accessors +/// that play `accessor_role`'s part +fn written_member_definitions<'db>( + db: &'db dyn Db, + scope: ScopeId<'db>, + symbol: ScopedSymbolId, + accessor_role: Option, +) -> Vec> { + let use_def = use_def_map(db, scope); + reachable_implementation_definitions( + db, + use_def + .reachable_symbol_declarations(symbol) + .filter_map(|declaration| declaration.declaration.definition()) + .chain( + use_def + .reachable_symbol_bindings(symbol) + .filter_map(|binding| binding.binding.definition()), + ), + ) + .into_iter() + .filter(|definition| property_accessor_role_matches(db, *definition, accessor_role)) + .filter_map(|definition| match definition.kind(db) { + DefinitionKind::Function(_) => member_implementation_definition(db, definition), + _ => Some(ResolvedDefinition::Definition(definition)), + }) + .collect() +} + +/// the class whose body makes `definition`, as the override checks see it: +/// undecorated, and specialized to its own type parameters. `None` when +/// `definition` is not made in a class body +fn declaring_class<'db>(db: &'db dyn Db, definition: Definition<'db>) -> Option> { + let scope = definition.scope(db); + let class_node = scope.node(db).as_class()?; + let class = semantic_index(db, scope.program_file(db)).try_definition(class_node)?; + Some(original_class_type(db, class)?.identity_specialization(db)) +} + +/// whether `class`'s own member `name`, as its body leaves it bound, is abstract +fn is_abstract_in<'db>( + db: &'db dyn Db, + env: &ProgramEnvironment<'db>, + class: ClassType<'db>, + name: &Name, +) -> bool { + class + .own_class_member(db, env, None, name) + .ignore_possibly_undefined() + .is_some_and(|ty| type_as_abstract_method(db, ty, class).is_some()) } /// The enum members `target` admits under their bare name, each as its name diff --git a/crates/ty_python_semantic/src/types/overrides.rs b/crates/ty_python_semantic/src/types/overrides.rs index 2a8245ff8c..48c764a407 100644 --- a/crates/ty_python_semantic/src/types/overrides.rs +++ b/crates/ty_python_semantic/src/types/overrides.rs @@ -1932,7 +1932,13 @@ fn stored_member_name(db: &dyn Db, class: ClassType<'_>, name: &str) -> String { } } -pub(crate) fn is_constructor_like_method(name: &str) -> bool { +/// whether a member named `name` is a constructor, or a method the constructors +/// call: `__init__`, `__new__`, `__post_init__`, `__init_subclass__` +/// +/// the override checks do not hold one to the signature of the member it +/// overrides, nor ask it to be marked `@override`: a subclass's constructor +/// takes what that subclass needs +pub fn is_constructor_like_method(name: &str) -> bool { matches!( name, "__init__" | "__new__" | "__post_init__" | "__init_subclass__" diff --git a/crates/ty_server/src/server/api.rs b/crates/ty_server/src/server/api.rs index dfdb193e6c..026377f0a0 100644 --- a/crates/ty_server/src/server/api.rs +++ b/crates/ty_server/src/server/api.rs @@ -207,6 +207,11 @@ pub(super) fn request(req: server::Request) -> Task { requests::SuperMembersRequestHandler::METHOD => background_document_request_task::< requests::SuperMembersRequestHandler, >(req, BackgroundSchedule::Worker), + requests::DocumentSuperMembersRequestHandler::METHOD => background_document_request_task::< + requests::DocumentSuperMembersRequestHandler, + >( + req, BackgroundSchedule::Worker + ), requests::PrepareTypeHierarchyRequestHandler::METHOD => background_document_request_task::< requests::PrepareTypeHierarchyRequestHandler, >( diff --git a/crates/ty_server/src/server/api/requests.rs b/crates/ty_server/src/server/api/requests.rs index 8aba1eca2c..1bde3dfb80 100644 --- a/crates/ty_server/src/server/api/requests.rs +++ b/crates/ty_server/src/server/api/requests.rs @@ -24,6 +24,7 @@ mod data_flow; mod detached_transpile; mod diagnostic; mod doc_highlights; +mod document_super_members; mod document_symbols; mod entry_point; mod execute_command; @@ -70,6 +71,7 @@ pub(super) use completion::CompletionRequestHandler; pub(super) use data_flow::DataFlowRequestHandler; pub(super) use diagnostic::DocumentDiagnosticRequestHandler; pub(super) use doc_highlights::DocumentHighlightRequestHandler; +pub(super) use document_super_members::DocumentSuperMembersRequestHandler; pub(super) use document_symbols::DocumentSymbolRequestHandler; pub(super) use entry_point::EntryPointHandler; pub(super) use execute_command::ExecuteCommand; diff --git a/crates/ty_server/src/server/api/requests/document_super_members.rs b/crates/ty_server/src/server/api/requests/document_super_members.rs new file mode 100644 index 0000000000..c373461486 --- /dev/null +++ b/crates/ty_server/src/server/api/requests/document_super_members.rs @@ -0,0 +1,137 @@ +//! `by/documentSuperMembers`: every class member of a document that overrides something, and the +//! superclass members each one overrides +//! +//! `by/superMembers` for the whole document at once, answered from the same code. it is what an +//! editor asks to draw an "overrides" marker beside each member that has one: asking +//! `by/superMembers` of each member instead would be a request per member on every pass over the +//! document, most of them answered with an empty list +//! +//! one document at a time, whole. `textHash` says which revision of this document the answer is +//! about, but the answer also reads every file the document's classes inherit from, and no change +//! to those is announced for this request. a client asks again after it opens, edits or closes +//! any document of the workspace, not only this one, and when the server asks it to refresh its +//! inlay hints, which the server does, for a client that declares inlay hint refresh support, +//! after a change on disk or to the environment that changes what it answers from + +use std::borrow::Cow; + +use lsp_types::{LspRequestMethod, MessageDirection, Range, Request, TextDocumentIdentifier, Uri}; +use ty_ide::document_super_members; +use ty_project::{ProjectDatabase, SemanticDb as _}; + +use crate::document::ToRangeExt; +use crate::server::api::requests::super_members::SuperMember; +use crate::server::api::traits::{ + BackgroundDocumentRequestHandler, RequestHandler, RetriableRequestHandler, +}; +use crate::session::DocumentSnapshot; +use crate::session::client::Client; + +pub(crate) enum DocumentSuperMembersRequest {} + +impl Request for DocumentSuperMembersRequest { + type Params = DocumentSuperMembersParams; + type Result = Option>; + const METHOD: LspRequestMethod<'static> = LspRequestMethod::Custom("by/documentSuperMembers"); + const MESSAGE_DIRECTION: MessageDirection = MessageDirection::ClientToServer; +} + +/// which document to answer for +#[derive(Debug, serde::Deserialize, serde::Serialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub(crate) struct DocumentSuperMembersParams { + text_document: TextDocumentIdentifier, +} + +/// a class member of the document that overrides at least one superclass member +#[derive(Debug, serde::Deserialize, serde::Serialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct OverridingMember { + /// the member's name + name: String, + /// the class whose body declares it + container_name: String, + /// its name where it is declared: the name of a `def` or a nested class, a name a class body + /// assigns, annotates or captures, or an import's alias. the position `by/superMembers` answers + /// the same members at + selection_range: Range, + /// whether the member is itself abstract, as [`SuperMember`]'s `abstract` says of what it + /// overrides + #[serde(rename = "abstract")] + is_abstract: bool, + /// what it overrides directly, in MRO order, as `by/superMembers` answers it. never empty + super_members: Vec, +} + +pub(crate) struct DocumentSuperMembersRequestHandler; + +impl RequestHandler for DocumentSuperMembersRequestHandler { + type RequestType = DocumentSuperMembersRequest; +} + +impl BackgroundDocumentRequestHandler for DocumentSuperMembersRequestHandler { + // what the project's files say, which is the same whether this one is a buffer or the file on + // disk; the answer reads no version + const ANSWERS_CLOSED_DOCUMENTS: bool = true; + + fn document_uri(params: &DocumentSuperMembersParams) -> Cow<'_, Uri> { + Cow::Borrowed(¶ms.text_document.uri) + } + + /// `null` when language services are off, or for a document whose members are not answered + /// here: a notebook, whose ranges are per cell, or a template. otherwise the members in source + /// order, and an empty list when none overrides anything. a member declared more than once is + /// listed at each declaration that names it in the source. constructors and the methods they + /// call, `__init__`, `__new__`, `__post_init__` and `__init_subclass__`, are left out: neither + /// `invalid-method-override` nor `missing-override-decorator` holds one to what it overrides, + /// and `by/superMembers` still answers one when asked + fn run_with_snapshot( + db: &ProjectDatabase, + snapshot: &DocumentSnapshot, + _client: &Client, + _params: DocumentSuperMembersParams, + ) -> crate::server::Result>> { + if snapshot + .workspace_settings() + .is_language_services_disabled() + { + return Ok(None); + } + if snapshot.document().is_cell_or_notebook() || snapshot.is_django_template() { + return Ok(None); + } + let Some(file) = snapshot.to_notebook_or_file(db) else { + return Ok(None); + }; + let encoding = snapshot.encoding(); + + Ok(Some( + document_super_members(db, db.program_file(file)) + .into_iter() + .filter_map(|member| { + let selection_range = member + .name_range + .to_lsp_range(db, file, encoding)? + .local_range(); + let super_members: Vec<_> = member + .super_members + .iter() + .filter_map(|member| SuperMember::from_ide(db, member, encoding)) + .collect(); + if super_members.is_empty() { + return None; + } + Some(OverridingMember { + name: member.name.to_string(), + container_name: member.class.to_string(), + selection_range, + is_abstract: member.is_abstract, + super_members, + }) + }) + .collect(), + )) + } +} + +impl RetriableRequestHandler for DocumentSuperMembersRequestHandler {} diff --git a/crates/ty_server/src/server/api/requests/super_members.rs b/crates/ty_server/src/server/api/requests/super_members.rs index ad24e6681b..4c58d063ab 100644 --- a/crates/ty_server/src/server/api/requests/super_members.rs +++ b/crates/ty_server/src/server/api/requests/super_members.rs @@ -17,6 +17,7 @@ use lsp_types::{ use ty_ide::super_members; use ty_project::{ProjectDatabase, SemanticDb as _}; +use crate::PositionEncoding; use crate::document::{PositionExt, ToLink}; use crate::server::api::traits::{ BackgroundDocumentRequestHandler, RequestHandler, RetriableRequestHandler, @@ -59,6 +60,32 @@ pub(crate) struct SuperMember { /// Whether the superclass synthesizes the member rather than writing it, as a dataclass does /// its `__init__`. The ranges are then the superclass's own. synthesized: bool, + /// Whether the member is abstract where the superclass declares it — an `@abstractmethod`, or a + /// protocol method with no implementation — so that overriding it implements it. What + /// `abstract-instantiation` counts as abstract. + #[serde(rename = "abstract")] + is_abstract: bool, +} + +impl SuperMember { + /// The member as a client reads it, or `None` when where it is cannot be said in the client's + /// terms. + pub(super) fn from_ide( + db: &ProjectDatabase, + member: &ty_ide::SuperMember, + encoding: PositionEncoding, + ) -> Option { + let link = member.target.to_link(db, None, encoding)?; + Some(SuperMember { + name: member.name.to_string(), + container_name: member.superclass.to_string(), + uri: link.target_uri, + range: link.target_range, + selection_range: link.target_selection_range, + synthesized: member.synthesized, + is_abstract: member.is_abstract, + }) + } } pub(crate) struct SuperMembersRequestHandler; @@ -105,18 +132,8 @@ impl BackgroundDocumentRequestHandler for SuperMembersRequestHandler { Ok(Some( members - .into_iter() - .filter_map(|member| { - let link = member.target.to_link(db, None, snapshot.encoding())?; - Some(SuperMember { - name: member.name.to_string(), - container_name: member.superclass.to_string(), - uri: link.target_uri, - range: link.target_range, - selection_range: link.target_selection_range, - synthesized: member.synthesized, - }) - }) + .iter() + .filter_map(|member| SuperMember::from_ide(db, member, snapshot.encoding())) .collect(), )) } diff --git a/crates/ty_server/tests/e2e/super_members.rs b/crates/ty_server/tests/e2e/super_members.rs index 1040fc5f24..8714e5418e 100644 --- a/crates/ty_server/tests/e2e/super_members.rs +++ b/crates/ty_server/tests/e2e/super_members.rs @@ -1,16 +1,23 @@ -//! `by/superMembers` over the wire +//! `by/superMembers` and `by/documentSuperMembers` over the wire //! //! which members a member overrides is `ty_ide`'s and its unit tests cover it. what these cover is //! the contract a client reads: a position on a declared name in, and out the members in MRO order, //! each in the file that declares it with the name to land on — and `null` kept apart from an empty -//! list, which a client tells a user two different things about +//! list, which a client tells a user two different things about. and for a whole document, every +//! member that overrides something with the same members, and the name each is declared with, about +//! the text the client names + +use std::time::Duration; use anyhow::Result; -use lsp_types::{LspRequestMethod, MessageDirection, Request}; +use lsp_types::{ + LspRequestMethod, MessageDirection, Request, TextDocumentContentChangeEvent, + TextDocumentContentChangeWholeDocument, +}; use ruff_db::system::SystemPath; use ty_server::ClientOptions; -use crate::{TestServer, TestServerBuilder}; +use crate::{AwaitResponseError, TestServer, TestServerBuilder}; /// the request as a client sends it, in json throughout enum SuperMembers {} @@ -22,6 +29,16 @@ impl Request for SuperMembers { const MESSAGE_DIRECTION: MessageDirection = MessageDirection::ClientToServer; } +/// `by/documentSuperMembers`, in json throughout +enum DocumentSuperMembers {} + +impl Request for DocumentSuperMembers { + type Params = serde_json::Value; + type Result = Option; + const METHOD: LspRequestMethod<'static> = LspRequestMethod::Custom("by/documentSuperMembers"); + const MESSAGE_DIRECTION: MessageDirection = MessageDirection::ClientToServer; +} + const BASES: &str = "\ class Root: def speak(self) -> str: @@ -83,6 +100,7 @@ fn a_method_of_a_class_with_two_bases_overrides_each_in_mro_order() -> Result<() assert_eq!(member["containerName"], container); assert_eq!(member["uri"], bases.as_str()); assert_eq!(member["synthesized"], false); + assert_eq!(member["abstract"], false); assert_eq!( member["selectionRange"], serde_json::json!({ @@ -107,3 +125,164 @@ fn a_member_that_overrides_nothing_is_an_empty_list_and_no_member_is_null() -> R assert_eq!(ask(&mut server, 4, 9), None); Ok(()) } + +fn ask_document(server: &mut TestServer, text_hash: Option<&str>) -> serde_json::Value { + let mut params = serde_json::json!({ + "textDocument": { "uri": server.file_uri(SystemPath::new("src/main.by")) }, + }); + if let Some(hash) = text_hash { + params["textHash"] = serde_json::json!(hash); + } + server + .send_request_await::(params) + .expect("an answer for a document") +} + +#[test] +fn a_document_lists_the_members_that_override_something_as_super_members_answers_each() -> Result<()> +{ + let mut server = server()?; + + let answer = ask_document(&mut server, None); + let members = answer.as_array().expect("a list of members"); + // `listen` overrides nothing, and `free` is no member + assert_eq!(members.len(), 1, "{answer:#}"); + let speak = &members[0]; + assert_eq!(speak["name"], "speak"); + assert_eq!(speak["containerName"], "Both"); + assert_eq!(speak["abstract"], false); + assert_eq!( + speak["selectionRange"], + serde_json::json!({ + "start": { "line": 3, "character": 17 }, + "end": { "line": 3, "character": 22 }, + }), + ); + // the very members `by/superMembers` answers at the name + assert_eq!( + Some(speak["superMembers"].clone()), + ask(&mut server, 3, 17), + "{answer:#}" + ); + Ok(()) +} + +/// A protocol member, and a member two classes up: what an editor marks as implemented rather +/// than overridden, and what it goes to past a class that does not declare it. +#[test] +fn a_protocol_member_is_abstract_and_a_member_two_levels_up_is_found() -> Result<()> { + let main = "\ +from typing import Protocol + +class Greeter(Protocol): + def greet(self) -> str: ... + +class A: + def f(self) -> None: ... + +class B(A): ... + +class C(B, Greeter): + def f(self) -> None: ... + + def greet(self) -> str: + return 'hi' +"; + let mut server = TestServerBuilder::new()? + .with_initialization_options(&ClientOptions::default()) + .with_workspace(SystemPath::new("src"), None)? + .with_file(SystemPath::new("src/main.by"), main)? + .build() + .wait_until_workspaces_are_initialized(); + server.open_text_document(SystemPath::new("src/main.by"), main, 1); + + let answer = ask_document(&mut server, None); + let members = answer.as_array().expect("a list of members"); + let summary: Vec<_> = members + .iter() + .map(|member| { + let overridden = &member["superMembers"][0]; + ( + member["name"].as_str().unwrap().to_string(), + member["selectionRange"]["start"]["line"].as_u64().unwrap(), + overridden["containerName"].as_str().unwrap().to_string(), + overridden["selectionRange"]["start"]["line"] + .as_u64() + .unwrap(), + overridden["abstract"].as_bool().unwrap(), + ) + }) + .collect(); + assert_eq!( + summary, + vec![ + ("f".to_string(), 11, "A".to_string(), 6, false), + ("greet".to_string(), 13, "Greeter".to_string(), 3, true), + ], + "{answer:#}" + ); + Ok(()) +} + +/// the same text-naming every document request takes: a document the client has not opened is +/// answered from the file when that is the text named, and a text the server does not hold yet is +/// waited for +#[test] +fn a_document_request_is_answered_about_the_text_it_names() -> Result<()> { + let mut server = TestServerBuilder::new()? + .with_initialization_options(&ClientOptions::default()) + .with_workspace(SystemPath::new("src"), None)? + .with_file(SystemPath::new("src/bases.by"), BASES)? + .with_file(SystemPath::new("src/main.by"), MAIN)? + .build() + .wait_until_workspaces_are_initialized(); + + // not opened: the file on disk, which is the text named + let answer = ask_document(&mut server, Some(&text_hash(MAIN))); + assert_eq!( + answer[0]["selectionRange"]["start"]["line"], 3, + "{answer:#}" + ); + + // an edit the server has not been sent moves `speak` down a line, and is waited for + server.open_text_document(SystemPath::new("src/main.by"), MAIN, 1); + let edited = format!("\n{MAIN}"); + let id = server.send_request::(serde_json::json!({ + "textDocument": { "uri": server.file_uri(SystemPath::new("src/main.by")) }, + "textHash": text_hash(&edited), + })); + assert!(matches!( + server.try_await_response::(&id, Some(Duration::from_millis(500))), + Err(AwaitResponseError::Timeout) + )); + server.change_text_document( + SystemPath::new("src/main.by"), + vec![ + TextDocumentContentChangeEvent::TextDocumentContentChangeWholeDocument( + TextDocumentContentChangeWholeDocument { text: edited }, + ), + ], + 2, + ); + let answer = server + .await_response::(&id) + .expect("an answer about the edit"); + assert_eq!( + answer[0]["selectionRange"]["start"]["line"], 4, + "{answer:#}" + ); + Ok(()) +} + +/// the text hash as a client computes it — see `named_text.rs`, which holds it to the definition +fn text_hash(text: &str) -> String { + let mut hash: u64 = 0xcbf2_9ce4_8422_2325; + let normalised = text.replace("\r\n", "\n").replace('\r', "\n"); + for unit in normalised.encode_utf16() { + for byte in unit.to_le_bytes() { + hash ^= u64::from(byte); + hash = hash.wrapping_mul(0x0000_0100_0000_01b3); + } + } + format!("{hash:016x}") +} diff --git a/docs/basedpython/features/editor.md b/docs/basedpython/features/editor.md index b36803c31c..efc65d26ba 100644 --- a/docs/basedpython/features/editor.md +++ b/docs/basedpython/features/editor.md @@ -403,8 +403,22 @@ where `B.f` overrides `A.f`, goes to `B.f`; a class with two bases that each declare the method goes to both. a `private` member overrides nothing, since python mangles it into a name of its own, and a member a superclass synthesizes rather than writes — a dataclass's `__init__` — goes to that superclass, marked -`synthesized`. the answer is `null` when no class member is declared at the -position, and an empty list for one that overrides nothing +`synthesized`. each is marked `abstract` when it is abstract where it is +declared — an `@abstractmethod`, or a protocol method with no implementation — +so that a member overriding it implements it. the answer is `null` when no class +member is declared at the position, and an empty list for one that overrides +nothing + +`by/documentSuperMembers` answers the same for a whole document in one request, +from the same code: every class member that overrides something, with its +class, the range of the name it is declared with, whether it is `abstract` +itself, and its `superMembers` as `by/superMembers` gives them. it is what an +editor asks to draw an "overrides" marker beside each member, on every pass over +the document. constructors and the methods they call — `__init__`, `__new__`, +`__post_init__`, `__init_subclass__` — are left out, as the override checks +leave them out, and `by/superMembers` still answers one. like the other document +requests it takes `textHash`, and answers a document the client has not opened +from the file on disk ## the program model @@ -446,7 +460,8 @@ counted as one `\n`, written as sixteen lowercase hex digits. for a notebook cel that is the cell's own text. the request is then answered about that text and no other. when the server already holds it — the open buffer, or for `by/syntaxOutline`, `by/injections`, -`textDocument/semanticTokens/full` and `textDocument/documentSymbol` the file on +`by/documentSuperMembers`, `textDocument/semanticTokens/full` and +`textDocument/documentSymbol` the file on disk of a document the client has not opened — it answers at once; otherwise the request waits for the `didOpen`, `didChange` or file write that brings the text, and is answered with `ServerCancelled` if nothing has after ten seconds. so a From aab0b7f4f280962fe182f0cbf89405528c9dbcd9 Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Fri, 25 Sep 2026 20:19:49 +1000 Subject: [PATCH 07/11] server: answer by/superMembers for a document the client has not opened when the request names the file's text, as by/documentSuperMembers does, and test that both are answered about the text they name --- .../src/server/api/requests/super_members.rs | 55 ++++---- crates/ty_server/tests/e2e/main.rs | 15 +++ crates/ty_server/tests/e2e/named_text.rs | 17 +-- crates/ty_server/tests/e2e/super_members.rs | 119 ++++++++++++++---- docs/basedpython/features/editor.md | 58 +++++---- 5 files changed, 179 insertions(+), 85 deletions(-) diff --git a/crates/ty_server/src/server/api/requests/super_members.rs b/crates/ty_server/src/server/api/requests/super_members.rs index 4c58d063ab..31157fab72 100644 --- a/crates/ty_server/src/server/api/requests/super_members.rs +++ b/crates/ty_server/src/server/api/requests/super_members.rs @@ -1,13 +1,13 @@ -//! `by/superMembers` — the superclass members a class member overrides. +//! `by/superMembers`: the superclass members a class member overrides //! -//! What an editor's *Go to Super* asks of a method or an attribute, and what an "overrides" gutter +//! what an editor's *Go to Super* asks of a method or an attribute, and what an "overrides" gutter //! marker points at. LSP has no request for it: `textDocument/implementation` goes the other way, //! from a member to the members that override it, and `typeHierarchy/supertypes` answers about -//! classes, not their members. +//! classes, not their members //! -//! The answer is the override checks' own: a member is said to override exactly the superclass -//! members `invalid-method-override` compares it with — the nearest along each branch of the MRO, -//! in MRO order, so a member of a class with several bases can override several. +//! what counts as an override is the override checks' own walk up the MRO. of the declarations it +//! finds, the answer is the nearest along each branch of the MRO, in MRO order, so a member of a +//! class with several bases can override several use std::borrow::Cow; @@ -34,8 +34,9 @@ impl Request for SuperMembersRequest { const MESSAGE_DIRECTION: MessageDirection = MessageDirection::ClientToServer; } -/// Which member to ask about: the document, and a position on the member's name where it is -/// declared — the name of a `def`, or a name a class body assigns or annotates. +/// which member to ask about: the document, and a position on the member's name where it is +/// declared: the name of a `def` or a nested class, a name a class body assigns, annotates or +/// captures, or an import's alias #[derive(Debug, serde::Deserialize, serde::Serialize)] #[serde(rename_all = "camelCase", deny_unknown_fields)] pub(crate) struct SuperMembersParams { @@ -43,33 +44,33 @@ pub(crate) struct SuperMembersParams { position: Position, } -/// One superclass member the member overrides. +/// one superclass member the member overrides #[derive(Debug, serde::Deserialize, serde::Serialize)] #[serde(rename_all = "camelCase")] pub(crate) struct SuperMember { - /// The member's name. + /// the member's name name: String, - /// The superclass that declares it. + /// the superclass that declares it container_name: String, - /// The file the superclass declares it in. + /// the file the superclass declares it in uri: Uri, - /// The whole declaration. + /// the whole declaration range: Range, - /// The declared name, which is where to go. + /// the declared name, which is where to go selection_range: Range, - /// Whether the superclass synthesizes the member rather than writing it, as a dataclass does - /// its `__init__`. The ranges are then the superclass's own. + /// whether the superclass synthesizes the member rather than writing it, as a dataclass does + /// its `__init__`. the ranges are then the superclass's own synthesized: bool, - /// Whether the member is abstract where the superclass declares it — an `@abstractmethod`, or a - /// protocol method with no implementation — so that overriding it implements it. What - /// `abstract-instantiation` counts as abstract. + /// whether the member is abstract where the superclass declares it, an `@abstractmethod` or a + /// protocol method with no implementation, so that overriding it implements it: what + /// `abstract-instantiation` counts as abstract #[serde(rename = "abstract")] is_abstract: bool, } impl SuperMember { - /// The member as a client reads it, or `None` when where it is cannot be said in the client's - /// terms. + /// the member as a client reads it, or `None` when where it is cannot be said in the client's + /// terms pub(super) fn from_ide( db: &ProjectDatabase, member: &ty_ide::SuperMember, @@ -95,12 +96,17 @@ impl RequestHandler for SuperMembersRequestHandler { } impl BackgroundDocumentRequestHandler for SuperMembersRequestHandler { + // what the project's files say about a position, the same whether this one is a buffer or the + // file on disk; the answer reads no version + const ANSWERS_CLOSED_DOCUMENTS: bool = true; + fn document_uri(params: &SuperMembersParams) -> Cow<'_, Uri> { Cow::Borrowed(¶ms.text_document.uri) } - /// `null` when there is no class member declared at the position; an empty list when there is - /// one and it overrides nothing. + /// `null` when language services are off, for a template, which has no python members, and + /// when there is no class member declared at the position; an empty list when there is one and + /// it overrides nothing fn run_with_snapshot( db: &ProjectDatabase, snapshot: &DocumentSnapshot, @@ -113,6 +119,9 @@ impl BackgroundDocumentRequestHandler for SuperMembersRequestHandler { { return Ok(None); } + if snapshot.is_django_template() { + return Ok(None); + } let Some(file) = snapshot.to_notebook_or_file(db) else { return Ok(None); diff --git a/crates/ty_server/tests/e2e/main.rs b/crates/ty_server/tests/e2e/main.rs index b921c90cf6..9ba139fe74 100644 --- a/crates/ty_server/tests/e2e/main.rs +++ b/crates/ty_server/tests/e2e/main.rs @@ -121,6 +121,21 @@ fn setup_tracing() { }); } +/// The text hash as a client computes it for a request's `textHash`, written out again here +/// rather than borrowed from the server, so that the two are held to the definition rather than +/// to each other: FNV-1a, 64 bits, over UTF-16 code units, every line ending counted as one `\n`. +pub(crate) fn text_hash(text: &str) -> String { + let mut hash: u64 = 0xcbf2_9ce4_8422_2325; + let normalised = text.replace("\r\n", "\n").replace('\r', "\n"); + for unit in normalised.encode_utf16() { + for byte in unit.to_le_bytes() { + hash ^= u64::from(byte); + hash = hash.wrapping_mul(0x0000_0100_0000_01b3); + } + } + format!("{hash:016x}") +} + /// Errors when receiving a notification or request from the server. #[derive(thiserror::Error, Debug)] pub(crate) enum ServerMessageError { diff --git a/crates/ty_server/tests/e2e/named_text.rs b/crates/ty_server/tests/e2e/named_text.rs index c84b5daf27..99f540fa8b 100644 --- a/crates/ty_server/tests/e2e/named_text.rs +++ b/crates/ty_server/tests/e2e/named_text.rs @@ -17,7 +17,7 @@ use ruff_db::system::SystemPath; use ty_server::ClientOptions; use crate::notebook::NotebookBuilder; -use crate::{AwaitResponseError, TestServer, TestServerBuilder}; +use crate::{AwaitResponseError, TestServer, TestServerBuilder, text_hash}; /// `by/syntaxOutline`, in json throughout, as a client sends it. enum SyntaxOutline {} @@ -50,21 +50,6 @@ impl Request for Hover { const MESSAGE_DIRECTION: MessageDirection = MessageDirection::ClientToServer; } -/// The text hash as a client computes it, written out again here rather than borrowed from the -/// server, so that the two are held to the definition rather than to each other: FNV-1a, 64 -/// bits, over UTF-16 code units, every line ending counted as one `\n`. -fn text_hash(text: &str) -> String { - let mut hash: u64 = 0xcbf2_9ce4_8422_2325; - let normalised = text.replace("\r\n", "\n").replace('\r', "\n"); - for unit in normalised.encode_utf16() { - for byte in unit.to_le_bytes() { - hash ^= u64::from(byte); - hash = hash.wrapping_mul(0x0000_0100_0000_01b3); - } - } - format!("{hash:016x}") -} - const MAIN: &str = "src/main.by"; /// On disk: one statement. diff --git a/crates/ty_server/tests/e2e/super_members.rs b/crates/ty_server/tests/e2e/super_members.rs index 8714e5418e..76c4f75fbf 100644 --- a/crates/ty_server/tests/e2e/super_members.rs +++ b/crates/ty_server/tests/e2e/super_members.rs @@ -7,17 +7,16 @@ //! member that overrides something with the same members, and the name each is declared with, about //! the text the client names -use std::time::Duration; - use anyhow::Result; use lsp_types::{ - LspRequestMethod, MessageDirection, Request, TextDocumentContentChangeEvent, + LanguageKind, LspRequestMethod, MessageDirection, Request, TextDocumentContentChangeEvent, TextDocumentContentChangeWholeDocument, }; use ruff_db::system::SystemPath; use ty_server::ClientOptions; -use crate::{AwaitResponseError, TestServer, TestServerBuilder}; +use crate::notebook::NotebookBuilder; +use crate::{TestServer, TestServerBuilder, text_hash}; /// the request as a client sends it, in json throughout enum SuperMembers {} @@ -167,8 +166,8 @@ fn a_document_lists_the_members_that_override_something_as_super_members_answers Ok(()) } -/// A protocol member, and a member two classes up: what an editor marks as implemented rather -/// than overridden, and what it goes to past a class that does not declare it. +/// a protocol member, and a member two classes up: what an editor marks as implemented rather +/// than overridden, and what it goes to past a class that does not declare it #[test] fn a_protocol_member_is_abstract_and_a_member_two_levels_up_is_found() -> Result<()> { let main = "\ @@ -244,17 +243,15 @@ fn a_document_request_is_answered_about_the_text_it_names() -> Result<()> { "{answer:#}" ); - // an edit the server has not been sent moves `speak` down a line, and is waited for + // an edit the server has not been sent moves `speak` down a line, and is waited for. the server + // takes its messages in order, so an answer made when the request arrived would be about the + // text before the edit, with `speak` on line 3 server.open_text_document(SystemPath::new("src/main.by"), MAIN, 1); let edited = format!("\n{MAIN}"); let id = server.send_request::(serde_json::json!({ "textDocument": { "uri": server.file_uri(SystemPath::new("src/main.by")) }, "textHash": text_hash(&edited), })); - assert!(matches!( - server.try_await_response::(&id, Some(Duration::from_millis(500))), - Err(AwaitResponseError::Timeout) - )); server.change_text_document( SystemPath::new("src/main.by"), vec![ @@ -274,15 +271,93 @@ fn a_document_request_is_answered_about_the_text_it_names() -> Result<()> { Ok(()) } -/// the text hash as a client computes it — see `named_text.rs`, which holds it to the definition -fn text_hash(text: &str) -> String { - let mut hash: u64 = 0xcbf2_9ce4_8422_2325; - let normalised = text.replace("\r\n", "\n").replace('\r', "\n"); - for unit in normalised.encode_utf16() { - for byte in unit.to_le_bytes() { - hash ^= u64::from(byte); - hash = hash.wrapping_mul(0x0000_0100_0000_01b3); - } - } - format!("{hash:016x}") +/// `by/superMembers` names its text as the document request does: answered from the file for a +/// document the client has not opened, and held for an edit the server has not been sent +#[test] +fn a_member_request_is_answered_about_the_text_it_names() -> Result<()> { + let mut server = TestServerBuilder::new()? + .with_initialization_options(&ClientOptions::default()) + .with_workspace(SystemPath::new("src"), None)? + .with_file(SystemPath::new("src/bases.by"), BASES)? + .with_file(SystemPath::new("src/main.by"), MAIN)? + .build() + .wait_until_workspaces_are_initialized(); + let uri = server.file_uri(SystemPath::new("src/main.by")); + + // not opened: the file on disk, which is the text named + let answer = server + .send_request_await::(serde_json::json!({ + "textDocument": { "uri": uri }, + "position": { "line": 3, "character": 17 }, + "textHash": text_hash(MAIN), + })) + .expect("the members `speak` overrides"); + assert_eq!(answer.as_array().map(Vec::len), Some(2), "{answer:#}"); + + // `speak` a line down in an edit not sent yet: held until it is. the server takes its messages + // in order, so an answer made when the request arrived would be about the text before the + // edit, where that position is inside a method body and has no member + server.open_text_document(SystemPath::new("src/main.by"), MAIN, 1); + let edited = format!("\n{MAIN}"); + let id = server.send_request::(serde_json::json!({ + "textDocument": { "uri": uri }, + "position": { "line": 4, "character": 17 }, + "textHash": text_hash(&edited), + })); + server.change_text_document( + SystemPath::new("src/main.by"), + vec![ + TextDocumentContentChangeEvent::TextDocumentContentChangeWholeDocument( + TextDocumentContentChangeWholeDocument { text: edited }, + ), + ], + 2, + ); + let answer = server + .await_response::(&id) + .expect("the members `speak` overrides, in the edit"); + assert_eq!(answer.as_array().map(Vec::len), Some(2), "{answer:#}"); + Ok(()) +} + +/// a notebook's ranges are per cell and a template is not python: neither has its members answered +/// for the whole document, and a template has none at a position either, even one whose text +/// would read as python +#[test] +fn a_notebook_or_a_template_is_answered_null() -> Result<()> { + let template = "class A:\n x = 1\n\nclass B(A):\n x = 2\n"; + let mut server = TestServerBuilder::new()? + .with_initialization_options(&ClientOptions::default()) + .with_workspace(SystemPath::new("src"), None)? + .with_file(SystemPath::new("src/templates/page.html"), template)? + .build() + .wait_until_workspaces_are_initialized(); + + let mut notebook = NotebookBuilder::virtual_file("test.ipynb"); + notebook.add_python_cell("class A:\n def f(self) -> None: ...\n"); + let cell = notebook.add_python_cell("class B(A):\n def f(self) -> None: ...\n"); + notebook.open(&mut server); + server.collect_publish_diagnostic_notifications(2); + let answer = server.send_request_await::(serde_json::json!({ + "textDocument": { "uri": cell }, + })); + assert_eq!(answer, None); + + server.open_text_document_as( + SystemPath::new("src/templates/page.html"), + template, + 1, + LanguageKind::new("django-html"), + ); + let uri = server.file_uri(SystemPath::new("src/templates/page.html")); + let answer = server.send_request_await::(serde_json::json!({ + "textDocument": { "uri": uri }, + })); + assert_eq!(answer, None); + let answer = server.send_request_await::(serde_json::json!({ + "textDocument": { "uri": uri }, + "position": { "line": 4, "character": 4 }, + })); + assert_eq!(answer, None); + Ok(()) } diff --git a/docs/basedpython/features/editor.md b/docs/basedpython/features/editor.md index efc65d26ba..a6376d2271 100644 --- a/docs/basedpython/features/editor.md +++ b/docs/basedpython/features/editor.md @@ -391,32 +391,42 @@ always was `textDocument/implementation` on a method goes down, to the methods that override it. `by/superMembers` goes up: given a position on the name a class -member is declared with — a `def`, or a name the class body assigns or -annotates — it answers the superclass members that member overrides, each with -its class, its file and the name to land on. a class goes up with -`typeHierarchy/supertypes` - -what counts as an override is the override checks' own walk up the MRO, so a -member goes to exactly what `invalid-method-override` compares it with: the -nearest declaration along each branch, in MRO order. a method overriding `B.f`, -where `B.f` overrides `A.f`, goes to `B.f`; a class with two bases that each -declare the method goes to both. a `private` member overrides nothing, since -python mangles it into a name of its own, and a member a superclass synthesizes -rather than writes — a dataclass's `__init__` — goes to that superclass, marked -`synthesized`. each is marked `abstract` when it is abstract where it is -declared — an `@abstractmethod`, or a protocol method with no implementation — -so that a member overriding it implements it. the answer is `null` when no class -member is declared at the position, and an empty list for one that overrides -nothing +member is declared with — a `def` or a nested class, a name the class body +assigns, annotates or captures, or an import's alias — it answers the +superclass members that member overrides, each with its class, its file and the +name to land on. a class goes up with `typeHierarchy/supertypes` + +what counts as an override is the override checks' own walk up the MRO, from +the class as it is written rather than as a decorator returns it. of what that +walk finds, a member goes to the nearest declaration along each branch, in MRO +order: a method overriding `B.f`, where `B.f` overrides `A.f`, goes to `B.f`, +and a class with two bases that each declare the method goes to both. the first +of them is the one `missing-override-decorator` names. a `private` member +overrides nothing, since it is emitted under a name of its own, and a member a +superclass synthesizes rather than writes — a dataclass's `__init__` — goes to +that superclass, marked `synthesized`. each is marked `abstract` when it is +abstract where it is declared — an `@abstractmethod`, or a protocol method with +no implementation — so that a member overriding it implements it. the answer is +`null` when no class member is declared at the position, and an empty list for +one that overrides nothing + +an answer depends on more than the text of the document it is about: on every +file the document's classes inherit from, and the server announces no change to +it. a client that keeps one asks again after it opens, edits or closes any +document of the workspace, not only that one, and on +`workspace/inlayHint/refresh`, which the server sends, to a client that supports +it, after a change on disk or to the environment changes what it answers from `by/documentSuperMembers` answers the same for a whole document in one request, from the same code: every class member that overrides something, with its class, the range of the name it is declared with, whether it is `abstract` itself, and its `superMembers` as `by/superMembers` gives them. it is what an editor asks to draw an "overrides" marker beside each member, on every pass over -the document. constructors and the methods they call — `__init__`, `__new__`, -`__post_init__`, `__init_subclass__` — are left out, as the override checks -leave them out, and `by/superMembers` still answers one. like the other document +the document. a property with `get` and `set` blocks is one member, listed once. +constructors and the methods they call — `__init__`, `__new__`, +`__post_init__`, `__init_subclass__` — are left out, since neither +`invalid-method-override` nor `missing-override-decorator` holds one to what it +overrides, and `by/superMembers` still answers one. like the other document requests it takes `textHash`, and answers a document the client has not opened from the file on disk @@ -458,11 +468,11 @@ which text a document request is about, by adding `textHash` to the request's params: FNV-1a, 64 bits, over the text's UTF-16 code units with every line ending counted as one `\n`, written as sixteen lowercase hex digits. for a notebook cell that is the cell's own text. the request is then answered about that text and no -other. when the server already holds it — the -open buffer, or for `by/syntaxOutline`, `by/injections`, +other. when the server already holds it — the open buffer, or for +`by/syntaxOutline`, `by/injections`, `by/superMembers`, `by/documentSuperMembers`, `textDocument/semanticTokens/full` and -`textDocument/documentSymbol` the file on -disk of a document the client has not opened — it answers at once; otherwise the +`textDocument/documentSymbol` the file on disk of a document the client has not +opened — it answers at once; otherwise the request waits for the `didOpen`, `didChange` or file write that brings the text, and is answered with `ServerCancelled` if nothing has after ten seconds. so a client need not know whether its `didOpen` has gone out before it asks, and an From a9a94ad9e3c63055c7d7b783ac6569cede8600b0 Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Fri, 25 Sep 2026 15:43:10 +1000 Subject: [PATCH 08/11] refuse a rename with the reason when there is a name but it cannot be renamed, rather than answering prepareRename with null, which an editor takes as nothing to say --- crates/ty_ide/src/lib.rs | 2 +- crates/ty_ide/src/rename.rs | 217 +++++++++++++++--- crates/ty_server/src/server/api.rs | 7 +- .../src/server/api/requests/prepare_rename.rs | 23 +- crates/ty_server/tests/e2e/rename.rs | 104 ++++++++- crates/ty_wasm/src/lib.rs | 55 ++++- playground/ty/src/Editor/Editor.tsx | 29 ++- 7 files changed, 369 insertions(+), 68 deletions(-) diff --git a/crates/ty_ide/src/lib.rs b/crates/ty_ide/src/lib.rs index c56ecf5245..9964c2ed86 100644 --- a/crates/ty_ide/src/lib.rs +++ b/crates/ty_ide/src/lib.rs @@ -81,7 +81,7 @@ pub use module_rename::{ }; pub use refactor::{Refactor, RefactorKind, RefactorOffer, refactor, refactors}; pub use references::ReferencesMode; -pub use rename::{can_rename, rename}; +pub use rename::{PreparedRename, can_rename, invalid_new_name, prepare_rename, rename}; pub use selection_range::selection_range; pub use semantic_tokens::{ Fragments, SemanticToken, SemanticTokenModifier, SemanticTokenType, SemanticTokens, diff --git a/crates/ty_ide/src/rename.rs b/crates/ty_ide/src/rename.rs index d3923482cf..fe514b7d5f 100644 --- a/crates/ty_ide/src/rename.rs +++ b/crates/ty_ide/src/rename.rs @@ -1,55 +1,148 @@ -use crate::goto::find_goto_target; +use crate::goto::{GotoTarget, find_goto_target}; use crate::references::{ReferencesMode, references}; use crate::{Db, ReferenceTarget}; use ruff_db::files::File; -use ruff_text_size::{Ranged, TextSize}; +use ruff_python_ast as ast; +use ruff_python_stdlib::identifiers::is_identifier; +use ruff_python_stdlib::keyword::is_keyword; +use ruff_text_size::{Ranged, TextRange, TextSize}; use ty_python_core::ProgramFile; -use ty_python_semantic::SemanticModel; +use ty_python_semantic::{ResolvedDefinition, SemanticModel}; /// Returns the range of the symbol if it can be renamed, None if not. -pub fn can_rename( - db: &dyn Db, - file: ProgramFile<'_>, - offset: TextSize, -) -> Option { +pub fn can_rename(db: &dyn Db, file: ProgramFile<'_>, offset: TextSize) -> Option { + match prepare_rename(db, file, offset) { + PreparedRename::Ready { range, .. } => Some(range), + PreparedRename::Refused(_) | PreparedRename::NoSymbol => None, + } +} + +/// what a rename at an offset would be, worked out before an editor offers one +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum PreparedRename { + /// the name the editor offers to rename + Ready { + /// what the rename replaces + range: TextRange, + /// the name as the editor offers it for editing, which is not always the + /// text at `range`: a `.by` keyword argument named with a string, as in + /// `f("timeout"=1)`, is offered as `timeout`, and the rename replaces the + /// quotes along with it + placeholder: String, + }, + /// there is a name here, and this is why it cannot be renamed + /// + /// said rather than kept quiet: an editor asked to rename `print` that + /// hears nothing back does nothing, and leaves the user to guess whether + /// the key was pressed at all + Refused(String), + /// there is nothing here a rename could be about: whitespace, punctuation, + /// a literal + NoSymbol, +} + +/// whether the symbol at `offset` can be renamed, and if it cannot, why not +pub fn prepare_rename(db: &dyn Db, file: ProgramFile<'_>, offset: TextSize) -> PreparedRename { let parsed = ruff_db::parsed::parsed_module(db, file.python_file(db)); let module = parsed.load(db); let source_file = file.file(db); let model = SemanticModel::new(db, file); // Get the definitions for the symbol at the offset - let goto_target = find_goto_target(&model, &module, offset)?; + let Some(goto_target) = find_goto_target(&model, &module, offset) else { + return PreparedRename::NoSymbol; + }; + + // `None`, `True` and `False` are goto targets like any name, but nothing + // declares them that a rename could reach + if let GotoTarget::Expression(expression) + | GotoTarget::Call { + callable: expression, + .. + } = &goto_target + && let Some(keyword) = keyword_literal(*expression) + { + return PreparedRename::Refused(format!("`{keyword}` is a keyword")); + } + + // this is the name `references` searches for, so a target without one is + // nothing a rename could be about: a literal, an operator. a string may yet + // be one of the django names a module writes as plain text, which is + // answered once this declines + let Some(name) = goto_target.to_string() else { + return PreparedRename::NoSymbol; + }; // Don't allow renaming of import module components - if matches!( - goto_target, - crate::goto::GotoTarget::ImportModuleComponent { .. } - ) { - return None; + if matches!(goto_target, GotoTarget::ImportModuleComponent { .. }) { + return PreparedRename::Refused(format!( + "`{name}` names a module, which is renamed by renaming its file" + )); } let current_file_in_project = is_file_in_project(db, source_file); - let declaration_targets = goto_target - .definitions(&model, ReferencesMode::Rename.to_import_alias_resolution())? - .goto_declaration(&model, &goto_target)? - .into_navigation_targets(model.db()); + let Some(declarations) = goto_target + .definitions(&model, ReferencesMode::Rename.to_import_alias_resolution()) + .and_then(|definitions| definitions.goto_declaration(&model, &goto_target)) + else { + // with no declaration there is nothing to rename, and renaming every + // other use of the same spelling would be a guess + return PreparedRename::Refused(format!("cannot find where `{name}` is declared")); + }; - for target in &declaration_targets { - let target_file = target.file(); + for declaration in &declarations { + let target_file = match declaration { + ResolvedDefinition::Definition(definition) => definition.file(db), + ResolvedDefinition::Module(module) => module.file(db), + ResolvedDefinition::FileWithRange(range) => range.file(), + }; // If definition is outside the project, refuse rename if !is_file_in_project(db, target_file) { - return None; + return PreparedRename::Refused(match declaration { + ResolvedDefinition::Module(_) => { + format!("`{name}` names a module outside this project") + } + ResolvedDefinition::Definition(_) | ResolvedDefinition::FileWithRange(_) => { + format!("`{name}` is declared outside this project") + } + }); } // If current file is not in project and any definition is outside current file, refuse rename if !current_file_in_project && target_file != source_file { - return None; + return PreparedRename::Refused(format!( + "`{name}` is declared in another file, and this file is not part of the project" + )); } } - Some(goto_target.range()) + PreparedRename::Ready { + range: goto_target.range(), + placeholder: name.into_owned(), + } +} + +/// why a symbol cannot be renamed to `new_name`, if it cannot: a name that is +/// not an identifier, or is a keyword, would leave the code unable to parse +pub fn invalid_new_name(new_name: &str) -> Option { + if is_keyword(new_name) { + Some(format!("`{new_name}` is a keyword")) + } else if is_identifier(new_name) { + None + } else { + Some(format!("`{new_name}` is not a valid name")) + } +} + +/// the keyword a `None`, `True` or `False` literal is spelled with +fn keyword_literal(expression: ast::ExprRef<'_>) -> Option<&'static str> { + match expression { + ast::ExprRef::NoneLiteral(_) => Some("None"), + ast::ExprRef::BooleanLiteral(literal) => Some(if literal.value { "True" } else { "False" }), + _ => None, + } } /// Perform a rename operation on the symbol at the given position. @@ -101,21 +194,27 @@ mod tests { use insta::assert_snapshot; use ruff_db::diagnostic::{Annotation, Diagnostic, DiagnosticId, LintName, Severity, Span}; use ruff_db::files::FileRange; + use ruff_ranged_value::ValueSource; use ruff_text_size::Ranged; + use ty_project::metadata::options::Options; impl CursorTest { fn prepare_rename(&self) -> String { - let Some(range) = salsa::attach(&self.db, || { - can_rename( + let prepared = salsa::attach(&self.db, || { + prepare_rename( &self.db, self.program_file(self.cursor.file), self.cursor.offset, ) - }) else { - return "Cannot rename".to_string(); - }; + }); - format!("Can rename symbol at range {range:?}") + match prepared { + PreparedRename::Ready { range, placeholder } => { + format!("Can rename symbol `{placeholder}` at range {range:?}") + } + PreparedRename::Refused(why) => format!("Cannot rename: {why}"), + PreparedRename::NoSymbol => "Cannot rename".to_string(), + } } fn rename(&self, new_name: &str) -> String { @@ -352,7 +451,7 @@ value = 0 ", ); - assert_snapshot!(test.prepare_rename(), @"Can rename symbol at range 10..15"); + assert_snapshot!(test.prepare_rename(), @"Can rename symbol `value` at range 10..15"); } #[test] @@ -1051,7 +1150,7 @@ x = os.path.join('a', 'b') ", ); - assert_snapshot!(test.prepare_rename(), @"Cannot rename"); + assert_snapshot!(test.prepare_rename(), @"Cannot rename: `os` names a module, which is renamed by renaming its file"); } #[test] @@ -1064,7 +1163,7 @@ result = join('a', 'b') ", ); - assert_snapshot!(test.prepare_rename(), @"Cannot rename"); + assert_snapshot!(test.prepare_rename(), @"Cannot rename: `path` names a module, which is renamed by renaming its file"); } #[test] @@ -1078,7 +1177,7 @@ x = os.path.join('a', 'b') ", ); - assert_snapshot!(test.prepare_rename(), @"Cannot rename"); + assert_snapshot!(test.prepare_rename(), @"Cannot rename: `os` names a module outside this project"); } #[test] @@ -1267,7 +1366,7 @@ def process_value(value): ", ); - assert_snapshot!(test.prepare_rename(), @"Cannot rename"); + assert_snapshot!(test.prepare_rename(), @"Cannot rename: `None` is a keyword"); } #[test] @@ -1280,7 +1379,7 @@ def convert_to_number(value): ", ); - assert_snapshot!(test.prepare_rename(), @"Cannot rename"); + assert_snapshot!(test.prepare_rename(), @"Cannot rename: `int` is declared outside this project"); } /// basedpython: a name that means a `typing` member without an import is a @@ -1292,7 +1391,51 @@ def convert_to_number(value): .source("main.by", "a: Mapping\n") .build(); - assert_snapshot!(test.prepare_rename(), @"Cannot rename"); + assert_snapshot!(test.prepare_rename(), @"Cannot rename: `Mapping` is declared outside this project"); + } + + /// a file the project leaves out is renamed in on its own, so a name it takes + /// from another file cannot be renamed from it + #[test] + fn cannot_rename_from_a_file_outside_the_project() { + let test = CursorTest::builder() + .options( + Options::from_toml_str("[src]\nexclude = [\"main.py\"]\n", ValueSource::Cli) + .expect("valid options"), + ) + .source("lib.py", "def f(): ...\n") + .source("main.py", "from lib import f\n\nf()\n") + .build(); + + assert_snapshot!(test.prepare_rename(), @"Cannot rename: `f` is declared in another file, and this file is not part of the project"); + } + + /// basedpython: a keyword argument may be named with a string, and that name is + /// renamed like any other keyword argument's, quotes and all + #[test] + fn rename_quoted_keyword_argument() { + let test = CursorTest::builder() + .source( + "main.by", + " +def f(timeout: int): ... + +f(\"timeout\"=1) +", + ) + .build(); + + assert_snapshot!(test.prepare_rename(), @"Can rename symbol `timeout` at range 29..38"); + assert_snapshot!(test.rename("retries"), @r#" + info[rename]: Rename symbol (found 2 locations) + --> main.by:2:7 + | + 2 | def f(timeout: int): ... + | ^^^^^^^ + 3 | + 4 | f("timeout"=1) + | --------- + "#); } #[test] @@ -1304,7 +1447,7 @@ def convert_to_number(value): .source("main.py", "_T_co\n") .build(); - assert_snapshot!(test.prepare_rename(), @"Cannot rename"); + assert_snapshot!(test.prepare_rename(), @"Cannot rename: cannot find where `_T_co` is declared"); } #[test] diff --git a/crates/ty_server/src/server/api.rs b/crates/ty_server/src/server/api.rs index 026377f0a0..a208c99760 100644 --- a/crates/ty_server/src/server/api.rs +++ b/crates/ty_server/src/server/api.rs @@ -658,7 +658,12 @@ fn respond( Req: RequestHandler, { if let Err(err) = &result { - tracing::error!("An error occurred with request ID {id}: {err}"); + // a deliberate refusal is an answer, not a failure of the server + if matches!(err.code, ErrorCode::RequestFailed) { + tracing::debug!("Request ID {id} was refused: {err}"); + } else { + tracing::error!("An error occurred with request ID {id}: {err}"); + } report_unexpected_failure(client, err, log_guidance); } client.respond(id, result); diff --git a/crates/ty_server/src/server/api/requests/prepare_rename.rs b/crates/ty_server/src/server/api/requests/prepare_rename.rs index 47867e10bd..f2e89d2e01 100644 --- a/crates/ty_server/src/server/api/requests/prepare_rename.rs +++ b/crates/ty_server/src/server/api/requests/prepare_rename.rs @@ -5,7 +5,7 @@ use lsp_server::ErrorCode; use lsp_types::{ PrepareRenameParams, PrepareRenamePlaceholder, PrepareRenameRequest, PrepareRenameResult, Uri, }; -use ty_ide::{PreparedTemplateRename, can_rename, django_prepare_rename}; +use ty_ide::{PreparedRename, PreparedTemplateRename, django_prepare_rename, prepare_rename}; use ty_project::{ProjectDatabase, SemanticDb as _}; use crate::document::{PositionExt, ToRangeExt}; @@ -55,16 +55,27 @@ impl BackgroundDocumentRequestHandler for PrepareRenameRequestHandler { let template = snapshot.is_django_template(); - // a python symbol is answered exactly as it was; the django names a - // module writes as plain strings are what is left over once it declines - if !template && let Some(range) = can_rename(db, db.program_file(file), offset) { + // a template holds no python symbol, so only the django names are asked + // about there. in a module, the django names it writes as plain strings + // are what is left over once the python symbol is not one + let python = (!template).then(|| prepare_rename(db, db.program_file(file), offset)); + if let Some(PreparedRename::Ready { range, placeholder }) = python { return Ok(range .to_lsp_range(db, file, snapshot.encoding()) - .map(|lsp_range| lsp_range.local_range().into())); + .map(|lsp_range| { + PrepareRenamePlaceholder::new(lsp_range.local_range(), placeholder).into() + })); } match django_prepare_rename(db, file, offset, template) { - None => Ok(None), + // the editor shows why rather than doing nothing, which from where + // the user sits cannot be told apart from the key not working + None => match python { + Some(PreparedRename::Refused(why)) => { + Err(Error::new(anyhow!(why), ErrorCode::RequestFailed)) + } + Some(PreparedRename::Ready { .. } | PreparedRename::NoSymbol) | None => Ok(None), + }, // the editor shows this rather than offering a rename it could not // finish, which is the whole point of asking first Some(PreparedTemplateRename::Refused(why)) => { diff --git a/crates/ty_server/tests/e2e/rename.rs b/crates/ty_server/tests/e2e/rename.rs index e7aac2108f..8d7f1b209f 100644 --- a/crates/ty_server/tests/e2e/rename.rs +++ b/crates/ty_server/tests/e2e/rename.rs @@ -1,5 +1,9 @@ -use crate::TestServerBuilder; +use std::time::Duration; + +use lsp_types::ShowMessageNotification; + use crate::notebook::NotebookBuilder; +use crate::{AwaitResponseError, TestServerBuilder}; use insta::assert_json_snapshot; #[test] @@ -80,3 +84,101 @@ type Style = Literal["italic", "bold", "underline"]"#, server.collect_publish_diagnostic_notifications(2); Ok(()) } + +/// a name that cannot be renamed is refused with the reason, which the editor +/// shows: a `null` answer gives it nothing to say, and to the user the rename +/// key then looks broken. the refusal is an answer rather than a failure of the +/// server, so no popup says the server hit a problem +#[test] +fn a_builtin_is_refused_with_the_reason() -> anyhow::Result<()> { + let mut server = TestServerBuilder::new()? + .with_file("foo.py", "")? + .build() + .wait_until_workspaces_are_initialized(); + + server.open_text_document("foo.py", "print(1)\n", 1); + + let id = server.send_request::(prepare_rename_params( + server.file_uri("foo.py"), + lsp_types::Position::new(0, 2), + )); + let response = server.try_await_response::(&id, None); + + let Err(AwaitResponseError::RequestFailed(failure)) = response else { + panic!("expected a refusal, got {response:?}"); + }; + assert_eq!(failure.code, lsp_server::ErrorCode::RequestFailed as i32); + assert_eq!(failure.message, "`print` is declared outside this project"); + + let popup = + server.try_await_notification::(Some(Duration::from_millis(100))); + assert!(popup.is_err(), "got {popup:?}"); + + Ok(()) +} + +/// where there is no name at all there is nothing to explain, and the answer +/// stays `null` +#[test] +fn nothing_to_rename_is_still_a_null_answer() -> anyhow::Result<()> { + let mut server = TestServerBuilder::new()? + .with_file("foo.py", "")? + .build() + .wait_until_workspaces_are_initialized(); + + server.open_text_document("foo.py", "x = 1\n", 1); + + let prepared = server.send_request_await::( + prepare_rename_params(server.file_uri("foo.py"), lsp_types::Position::new(0, 4)), + ); + + assert!(prepared.is_none(), "got {prepared:?}"); + + Ok(()) +} + +/// a keyword argument a `.by` file names with a string is offered for renaming +/// by its name, so the editor does not start the new name with the quotes +#[test] +fn a_quoted_keyword_argument_is_offered_by_its_name() -> anyhow::Result<()> { + let mut server = TestServerBuilder::new()? + .with_file("foo.by", QUOTED_KEYWORD)? + .build() + .wait_until_workspaces_are_initialized(); + + server.open_text_document("foo.by", QUOTED_KEYWORD, 1); + + let prepared = server.send_request_await::( + prepare_rename_params(server.file_uri("foo.by"), lsp_types::Position::new(2, 4)), + ); + + assert_eq!( + prepared, + Some(lsp_types::PrepareRenameResult::PrepareRenamePlaceholder( + lsp_types::PrepareRenamePlaceholder::new( + lsp_types::Range::new( + lsp_types::Position::new(2, 2), + lsp_types::Position::new(2, 11) + ), + "timeout".to_string(), + ) + )) + ); + + Ok(()) +} + +const QUOTED_KEYWORD: &str = "def f(timeout: int): ...\n\nf(\"timeout\"=1)\n"; + +fn prepare_rename_params( + uri: lsp_types::Uri, + position: lsp_types::Position, +) -> lsp_types::PrepareRenameParams { + lsp_types::PrepareRenameParams { + text_document_position_params: lsp_types::TextDocumentPositionParams { + text_document: lsp_types::TextDocumentIdentifier { uri }, + position, + }, + work_done_progress_params: lsp_types::WorkDoneProgressParams::default(), + } +} diff --git a/crates/ty_wasm/src/lib.rs b/crates/ty_wasm/src/lib.rs index 751daa7092..c75dd45859 100644 --- a/crates/ty_wasm/src/lib.rs +++ b/crates/ty_wasm/src/lib.rs @@ -20,7 +20,7 @@ use ruff_text_size::{Ranged, TextSize}; use ty_ide::{ CompletionCapabilities, Hint as IdeHint, InlayHintSettings, MarkupKind, RangedValue, can_rename, document_highlights, find_references, goto_declaration, goto_definition, - goto_type_definition, hover, inlay_hints, rename, + goto_type_definition, hover, inlay_hints, invalid_new_name, prepare_rename, rename, }; use ty_ide::{NavigationTarget, NavigationTargets, hints, signature_help}; use ty_project::metadata::options::Options; @@ -444,22 +444,36 @@ impl Workspace { &self, file_id: &FileHandle, position: Position, - ) -> Result, Error> { + ) -> Result { let source = source_text(&self.db, file_id.file); let index = line_index(&self.db, file_id.file); let offset = position.to_text_size(&source, &index, self.position_encoding)?; - let Some(range) = can_rename(&self.db, self.db.program_file(file_id.file), offset) else { - return Ok(None); - }; - - Ok(Some(Range::from_text_range( - range, - &index, - &source, - self.position_encoding, - ))) + Ok( + match prepare_rename(&self.db, self.db.program_file(file_id.file), offset) { + ty_ide::PreparedRename::Ready { range, placeholder } => PreparedRename { + range: Some(Range::from_text_range( + range, + &index, + &source, + self.position_encoding, + )), + placeholder: Some(placeholder), + refusal: None, + }, + ty_ide::PreparedRename::Refused(why) => PreparedRename { + range: None, + placeholder: None, + refusal: Some(why), + }, + ty_ide::PreparedRename::NoSymbol => PreparedRename { + range: None, + placeholder: None, + refusal: None, + }, + }, + ) } #[wasm_bindgen] @@ -479,6 +493,10 @@ impl Workspace { return Ok(Vec::new()); } + if let Some(why) = invalid_new_name(new_name) { + return Err(Error::new(&why)); + } + let Some(rename_results) = rename(&self.db, program_file, offset, new_name) else { return Ok(Vec::new()); }; @@ -1450,6 +1468,19 @@ pub struct TextEdit { pub new_text: String, } +/// what a rename at a position would be: the range of the name to rename and +/// the name as it is offered for editing, or, when there is a name that cannot +/// be renamed, why not +#[wasm_bindgen] +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct PreparedRename { + pub range: Option, + #[wasm_bindgen(getter_with_clone)] + pub placeholder: Option, + #[wasm_bindgen(getter_with_clone)] + pub refusal: Option, +} + #[wasm_bindgen] #[derive(Debug, Clone, PartialEq, Eq)] pub struct RenameEdit { diff --git a/playground/ty/src/Editor/Editor.tsx b/playground/ty/src/Editor/Editor.tsx index 08537d918e..dd5473fe7c 100644 --- a/playground/ty/src/Editor/Editor.tsx +++ b/playground/ty/src/Editor/Editor.tsx @@ -265,11 +265,11 @@ class PlaygroundServer signatureHelpTriggerCharacters: string[] = ["(", ","]; signatureHelpRetriggerCharacters: string[] = [")"]; - private renameRejection(): T & languages.Rejection { + private renameRejection(reason?: string): T & languages.Rejection { // This assertion is intentionally unchecked: Monaco checks `rejectReason` // before reading the success fields, but its RenameProvider declaration // models rejections as part of the success type. - return { rejectReason: "this element can't be renamed" } as T & + return { rejectReason: reason ?? "this element can't be renamed" } as T & languages.Rejection; } @@ -879,20 +879,21 @@ class PlaygroundServer return this.renameRejection(); } - const range = this.props.workspace.prepareRename( + const prepared = this.props.workspace.prepareRename( fileHandle, new TyPosition(position.lineNumber, position.column), ); + const range = prepared.range; if (range == null) { - return this.renameRejection(); + return this.renameRejection(prepared.refusal); } const monacoRange = tyRangeToMonacoRange(range); return { range: monacoRange, - text: model.getValueInRange(monacoRange), + text: prepared.placeholder ?? model.getValueInRange(monacoRange), }; } @@ -908,11 +909,19 @@ class PlaygroundServer return this.renameRejection(); } - const renameEdits = this.props.workspace.rename( - fileHandle, - new TyPosition(position.lineNumber, position.column), - newName, - ); + let renameEdits; + try { + renameEdits = this.props.workspace.rename( + fileHandle, + new TyPosition(position.lineNumber, position.column), + newName, + ); + } catch (error) { + // a new name that is not a valid name is refused with the reason + return this.renameRejection( + error instanceof Error ? error.message : undefined, + ); + } if (renameEdits.length === 0) { return this.renameRejection(); From 332a57c72ec0165aaff4e3eba47ac67b0a4e61d9 Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Fri, 25 Sep 2026 15:42:32 +1000 Subject: [PATCH 09/11] answer a whole-workspace check once it is done with by/checkWorkspace, and keep every workspace diagnostics request held open rather than only the last --- crates/ty_server/src/server.rs | 1 + crates/ty_server/src/server/api.rs | 6 +- crates/ty_server/src/server/api/changes.rs | 2 +- crates/ty_server/src/server/api/requests.rs | 3 + .../server/api/requests/check_workspace.rs | 161 +++++++ .../src/server/api/requests/shutdown.rs | 10 +- .../api/requests/workspace_diagnostic.rs | 118 +++-- crates/ty_server/src/server/main_loop.rs | 2 +- crates/ty_server/src/session.rs | 90 ++-- crates/ty_server/tests/e2e/check_workspace.rs | 417 ++++++++++++++++++ crates/ty_server/tests/e2e/main.rs | 1 + .../ty_server/tests/e2e/pull_diagnostics.rs | 46 ++ docs/basedpython/features/editor.md | 38 ++ 13 files changed, 808 insertions(+), 87 deletions(-) create mode 100644 crates/ty_server/src/server/api/requests/check_workspace.rs create mode 100644 crates/ty_server/tests/e2e/check_workspace.rs diff --git a/crates/ty_server/src/server.rs b/crates/ty_server/src/server.rs index 1544eb82db..7ce88a4a6f 100644 --- a/crates/ty_server/src/server.rs +++ b/crates/ty_server/src/server.rs @@ -24,6 +24,7 @@ mod script_progress; use crate::project_server::Listener as ProjectServerListener; use crate::session::client::Client; pub(crate) use api::Error; +pub(crate) use api::SuspendedWorkspaceRequestKind; pub(crate) use api::{ publish_all_document_diagnostics, publish_diagnostics_if_needed, publish_settings_diagnostics, }; diff --git a/crates/ty_server/src/server/api.rs b/crates/ty_server/src/server/api.rs index a208c99760..239fcea404 100644 --- a/crates/ty_server/src/server/api.rs +++ b/crates/ty_server/src/server/api.rs @@ -27,6 +27,7 @@ use crate::session::client::Client; pub(crate) use diagnostics::{ publish_all_document_diagnostics, publish_diagnostics_if_needed, publish_settings_diagnostics, }; +pub(crate) use requests::SuspendedWorkspaceRequestKind; use ruff_db::panic::PanicError; /// Processes a request from the client to the server. @@ -58,6 +59,9 @@ pub(super) fn request(req: server::Request) -> Task { >( req, BackgroundSchedule::Worker ), + requests::CheckWorkspaceRequestHandler::METHOD => background_request_task::< + requests::CheckWorkspaceRequestHandler, + >(req, BackgroundSchedule::Worker), requests::GotoTypeDefinitionRequestHandler::METHOD => background_document_request_task::< requests::GotoTypeDefinitionRequestHandler, >( @@ -568,7 +572,7 @@ fn sync_notification_task( // If there's a pending workspace diagnostic long-polling request, // resume it, but only if the session revision changed (e.g. because some document changed). - session.resume_suspended_workspace_diagnostic_request(client); + session.resume_suspended_workspace_diagnostic_requests(client); })) } diff --git a/crates/ty_server/src/server/api/changes.rs b/crates/ty_server/src/server/api/changes.rs index c39e78e65a..de46a700c0 100644 --- a/crates/ty_server/src/server/api/changes.rs +++ b/crates/ty_server/src/server/api/changes.rs @@ -56,7 +56,7 @@ pub(crate) fn apply(session: &mut Session, client: &Client, changes: &[ChangeEve // a workspace diagnostic request waiting for the session to change is answered here rather // than by a notification's handler: most changes come from the session's own watcher - session.resume_suspended_workspace_diagnostic_request(client); + session.resume_suspended_workspace_diagnostic_requests(client); if client_capabilities.supports_workspace_diagnostic_refresh() { client.send_request::(session, (), |_, ()| {}); diff --git a/crates/ty_server/src/server/api/requests.rs b/crates/ty_server/src/server/api/requests.rs index 1bde3dfb80..4c9d730f2a 100644 --- a/crates/ty_server/src/server/api/requests.rs +++ b/crates/ty_server/src/server/api/requests.rs @@ -16,6 +16,7 @@ mod alignment_groups; mod build_output; mod call_hierarchy_incoming_calls; mod call_hierarchy_outgoing_calls; +mod check_workspace; mod code_action; mod code_action_resolve; mod code_lens; @@ -64,6 +65,8 @@ pub(super) use alignment_groups::AlignmentGroupsRequestHandler; pub(super) use build_output::BuildOutputRequestHandler; pub(super) use call_hierarchy_incoming_calls::CallHierarchyIncomingCallsRequestHandler; pub(super) use call_hierarchy_outgoing_calls::CallHierarchyOutgoingCallsRequestHandler; +pub(super) use check_workspace::CheckWorkspaceRequestHandler; +pub(crate) use check_workspace::SuspendedWorkspaceRequestKind; pub(super) use code_action::CodeActionRequestHandler; pub(super) use code_action_resolve::CodeActionResolveRequestHandler; pub(super) use code_lens::CodeLensRequestHandler; diff --git a/crates/ty_server/src/server/api/requests/check_workspace.rs b/crates/ty_server/src/server/api/requests/check_workspace.rs new file mode 100644 index 0000000000..72279f4476 --- /dev/null +++ b/crates/ty_server/src/server/api/requests/check_workspace.rs @@ -0,0 +1,161 @@ +//! `by/checkWorkspace` — the workspace's diagnostics, answered once they are checked +//! +//! the same report as `workspace/diagnostic`, from the same check, for a client that needs the +//! answer rather than the next change. `workspace/diagnostic` is long-polled: when nothing differs +//! from the result ids the client sent, the request is held open until something does. that is +//! what an editor keeping a problems list current wants, and it leaves a client that asks once — +//! an inspection run over the whole project, a batch run with no editor at all — waiting forever +//! on a project with no diagnostics, because "nothing" is never different from nothing. this one +//! answers as soon as the check is done, unchanged or not +//! +//! the one thing it still waits for is the workspace being checkable: while a script's initial +//! environment is being set up it is held open the way a long poll is, and answered once the +//! environment is there + +use anyhow::anyhow; +use lsp_server::{ErrorCode, RequestId}; +use lsp_types::{ + LspRequestMethod, MessageDirection, Request, WorkspaceDiagnosticParams, + WorkspaceDiagnosticReport, WorkspaceDiagnosticRequest, +}; +use serde_json::json; + +use crate::server::Action; +use crate::server::api::Error; +use crate::server::api::requests::workspace_diagnostic::{WorkspaceCheck, check_workspace}; +use crate::server::api::traits::{ + BackgroundRequestHandler, RequestHandler, RetriableRequestHandler, +}; +use crate::session::client::Client; +use crate::session::{SessionSnapshot, SuspendedWorkspaceDiagnosticRequest}; + +/// the request a client sends to have the workspace checked and be told the result +pub(crate) enum CheckWorkspaceRequest {} + +impl Request for CheckWorkspaceRequest { + type Params = WorkspaceDiagnosticParams; + type Result = WorkspaceDiagnosticReport; + // not a method LSP defines, so it goes across as a custom one, under the same `by/` prefix as + // the rest of them + const METHOD: LspRequestMethod<'static> = LspRequestMethod::Custom("by/checkWorkspace"); + const MESSAGE_DIRECTION: MessageDirection = MessageDirection::ClientToServer; +} + +pub(crate) struct CheckWorkspaceRequestHandler; + +impl RequestHandler for CheckWorkspaceRequestHandler { + type RequestType = CheckWorkspaceRequest; +} + +impl BackgroundRequestHandler for CheckWorkspaceRequestHandler { + // the request is answered by `handle_request`, which holds the request open while the + // workspace is not ready rather than refuse it. this is the answer on its own, without the + // holding + fn run( + snapshot: &SessionSnapshot, + client: &Client, + params: WorkspaceDiagnosticParams, + ) -> crate::server::Result { + match check_workspace(snapshot, client, answered_whole(params)) { + WorkspaceCheck::Checked(report) => Ok(report), + // an answer that would be different a moment later, which is what `ContentModified` + // says + WorkspaceCheck::NotReady => Err(Error::new( + anyhow!("the workspace is not ready to be checked"), + ErrorCode::ContentModified, + )), + WorkspaceCheck::Disabled => Err(not_checked()), + } + } + + fn handle_request( + id: &RequestId, + snapshot: SessionSnapshot, + client: &Client, + params: WorkspaceDiagnosticParams, + ) { + let params = answered_whole(params); + match check_workspace(&snapshot, client, params.clone()) { + WorkspaceCheck::Checked(report) => client.respond(id, Ok(report)), + WorkspaceCheck::NotReady => { + tracing::debug!("Holding `by/checkWorkspace` open until the workspace is ready"); + client.queue_action(Action::SuspendWorkspaceDiagnostics(Box::new( + SuspendedWorkspaceDiagnosticRequest { + id: id.clone(), + kind: SuspendedWorkspaceRequestKind::CheckWorkspace, + params: json!(¶ms), + revision: snapshot.revision(), + }, + ))); + } + WorkspaceCheck::Disabled => { + client.respond::(id, Err(not_checked())); + } + } + } +} + +impl RetriableRequestHandler for CheckWorkspaceRequestHandler { + // an edit that lands mid-check cancels it; the client asked for the answer, not for the + // chance of one, so the check starts again on the edited workspace rather than telling the + // client to ask again + const RETRY_ON_CANCELLATION: bool = true; +} + +/// `params` without the tokens that would have the check report anything but its answer +/// +/// a check that an edit cancels starts again, so anything sent under the client's tokens before +/// the cancellation would be sent a second time: a second first batch of partial results, with +/// reports for the same files from before the edit, and a second start of progress on a token +/// whose progress has ended. so the report comes whole, in the response, and progress, where +/// the client supports it, under a token the server creates for each attempt +fn answered_whole(mut params: WorkspaceDiagnosticParams) -> WorkspaceDiagnosticParams { + params.partial_result_params.partial_result_token = None; + params.work_done_progress_params.work_done_token = None; + params +} + +/// an empty report would say the workspace has no diagnostics, which nobody checked +fn not_checked() -> Error { + Error::new( + anyhow!("the workspace is not checked: the diagnostic mode is not `workspace`"), + ErrorCode::RequestFailed, + ) +} + +/// which of the two requests that wait on the workspace a held one is +#[derive(Debug, Clone, Copy)] +pub(crate) enum SuspendedWorkspaceRequestKind { + /// `workspace/diagnostic`, held until something differs from what the client has + LongPoll, + + /// `by/checkWorkspace`, held until the workspace can be checked + CheckWorkspace, +} + +impl SuspendedWorkspaceRequestKind { + /// the method the request is run again as + pub(crate) fn method(self) -> String { + match self { + Self::LongPoll => WorkspaceDiagnosticRequest::METHOD.to_string(), + Self::CheckWorkspace => CheckWorkspaceRequest::METHOD.to_string(), + } + } + + /// answers the held request `id` when the server shuts down before it could be run again + pub(crate) fn answer_at_shutdown(self, id: &RequestId, client: &Client) { + match self { + // an empty report tells the client nothing changed, which is so + Self::LongPoll => client.respond(id, Ok(WorkspaceDiagnosticReport::default())), + // `by/checkWorkspace` promises the workspace's diagnostics, and an empty report would + // say it has none. it was never checked, so it is not answered as if it was + Self::CheckWorkspace => client.respond::( + id, + Err(Error::new( + anyhow!("the server shut down before the workspace could be checked"), + ErrorCode::RequestFailed, + )), + ), + } + } +} diff --git a/crates/ty_server/src/server/api/requests/shutdown.rs b/crates/ty_server/src/server/api/requests/shutdown.rs index 95addad3a1..62d50f5651 100644 --- a/crates/ty_server/src/server/api/requests/shutdown.rs +++ b/crates/ty_server/src/server/api/requests/shutdown.rs @@ -2,7 +2,6 @@ use crate::Session; use crate::server::api::traits::{RequestHandler, SyncRequestHandler}; use crate::session::client::Client; -use lsp_types::WorkspaceDiagnosticReport; use salsa::Database; pub(crate) struct ShutdownHandler; @@ -16,13 +15,8 @@ impl SyncRequestHandler for ShutdownHandler { tracing::debug!("Received shutdown request, waiting for exit notification"); // Respond to any pending workspace diagnostic requests - if let Some(suspended_workspace_request) = - session.take_suspended_workspace_diagnostic_request() - { - client.respond( - &suspended_workspace_request.id, - Ok(WorkspaceDiagnosticReport::default()), - ); + for suspended_workspace_request in session.take_suspended_workspace_diagnostic_requests() { + suspended_workspace_request.answer_at_shutdown(client); } session.set_shutdown_requested(true); diff --git a/crates/ty_server/src/server/api/requests/workspace_diagnostic.rs b/crates/ty_server/src/server/api/requests/workspace_diagnostic.rs index 1553e33069..eb89c59221 100644 --- a/crates/ty_server/src/server/api/requests/workspace_diagnostic.rs +++ b/crates/ty_server/src/server/api/requests/workspace_diagnostic.rs @@ -29,7 +29,7 @@ use crate::server::api::traits::{ BackgroundRequestHandler, RequestHandler, RetriableRequestHandler, }; use crate::server::lazy_work_done_progress::LazyWorkDoneProgress; -use crate::server::{Action, Result}; +use crate::server::{Action, Result, SuspendedWorkspaceRequestKind}; use crate::session::client::Client; use crate::session::index::Index; use crate::session::{GlobalSettings, SessionSnapshot, SuspendedWorkspaceDiagnosticRequest}; @@ -115,49 +115,13 @@ impl BackgroundRequestHandler for WorkspaceDiagnosticRequestHandler { client: &Client, params: WorkspaceDiagnosticParams, ) -> Result { - if !snapshot.global_settings().diagnostic_mode().is_workspace() { - tracing::debug!("Workspace diagnostics is disabled; returning empty report"); - return Ok(WorkspaceDiagnosticReport { items: vec![] }); - } - - if snapshot - .projects() - .iter() - .any(|db| db.uv_environments().has_pending_initializations()) - { - tracing::debug!( - "Deferring workspace diagnostics until script initialization completes" - ); - // Returning an empty workspace report makes `handle_request` suspend the request. - // Skip the response writer: it would clear diagnostics for previous result IDs - // that we have not checked yet. Suspension retains the request, not this snapshot. - return Ok(WorkspaceDiagnosticReport { items: vec![] }); - } - - let writer = ResponseWriter::new( - params.partial_result_params.partial_result_token, - params.previous_result_ids, - snapshot, - client, - ); - - // Use the work done progress token from the client request, if provided - // Note: neither VS Code nor Zed currently support this, - // see https://github.com/microsoft/vscode-languageserver-node/issues/528 - // That's why we fall back to server-initiated progress if no token is provided. - let work_done_progress = LazyWorkDoneProgress::new( - client, - params.work_done_progress_params.work_done_token, - "Checking", - snapshot.resolved_client_capabilities(), - ); - let mut reporter = WorkspaceDiagnosticsProgressReporter::new(work_done_progress, writer); - - for db in snapshot.projects() { - db.check_with_reporter(&mut reporter); - } - - Ok(reporter.into_final_report()) + Ok(match check_workspace(snapshot, client, params) { + WorkspaceCheck::Checked(report) => report, + // an empty report, which `handle_request` holds open + WorkspaceCheck::NotReady | WorkspaceCheck::Disabled => { + WorkspaceDiagnosticReport::default() + } + }) } fn handle_request( @@ -195,6 +159,7 @@ impl BackgroundRequestHandler for WorkspaceDiagnosticRequestHandler { client.queue_action(Action::SuspendWorkspaceDiagnostics(Box::new( SuspendedWorkspaceDiagnosticRequest { id: id.clone(), + kind: SuspendedWorkspaceRequestKind::LongPoll, params: json!(¶ms), revision: snapshot.revision(), }, @@ -209,6 +174,71 @@ impl BackgroundRequestHandler for WorkspaceDiagnosticRequestHandler { } } +/// what checking the workspace for a workspace diagnostics request came to +pub(crate) enum WorkspaceCheck { + /// the workspace was checked, and this is the report + Checked(WorkspaceDiagnosticReport), + + /// the workspace was not checked because a script's initial environment is still being set + /// up, and a report now would describe the script against an environment it does not have + /// yet. the request waits for it, held open like a long poll + NotReady, + + /// the workspace was not checked because the diagnostic mode does not report it + Disabled, +} + +/// checks the workspace for a request with `params`, streaming the reports that changed when the +/// params carry a partial result token +/// +/// shared by `workspace/diagnostic` and `by/checkWorkspace`, which differ in when they answer +pub(crate) fn check_workspace( + snapshot: &SessionSnapshot, + client: &Client, + params: WorkspaceDiagnosticParams, +) -> WorkspaceCheck { + if !snapshot.global_settings().diagnostic_mode().is_workspace() { + tracing::debug!("Workspace diagnostics is disabled; not checking the workspace"); + return WorkspaceCheck::Disabled; + } + + if snapshot + .projects() + .iter() + .any(|db| db.uv_environments().has_pending_initializations()) + { + tracing::debug!("Deferring workspace diagnostics until script initialization completes"); + // Skip the response writer: it would clear diagnostics for previous result IDs + // that we have not checked yet. Suspension retains the request, not this snapshot. + return WorkspaceCheck::NotReady; + } + + let writer = ResponseWriter::new( + params.partial_result_params.partial_result_token, + params.previous_result_ids, + snapshot, + client, + ); + + // Use the work done progress token from the client request, if provided + // Note: neither VS Code nor Zed currently support this, + // see https://github.com/microsoft/vscode-languageserver-node/issues/528 + // That's why we fall back to server-initiated progress if no token is provided. + let work_done_progress = LazyWorkDoneProgress::new( + client, + params.work_done_progress_params.work_done_token, + "Checking", + snapshot.resolved_client_capabilities(), + ); + let mut reporter = WorkspaceDiagnosticsProgressReporter::new(work_done_progress, writer); + + for db in snapshot.projects() { + db.check_with_reporter(&mut reporter); + } + + WorkspaceCheck::Checked(reporter.into_final_report()) +} + impl RetriableRequestHandler for WorkspaceDiagnosticRequestHandler { fn salsa_cancellation_error() -> lsp_server::ResponseError { lsp_server::ResponseError { diff --git a/crates/ty_server/src/server/main_loop.rs b/crates/ty_server/src/server/main_loop.rs index 3154f0293a..bb94c2a6f8 100644 --- a/crates/ty_server/src/server/main_loop.rs +++ b/crates/ty_server/src/server/main_loop.rs @@ -192,7 +192,7 @@ impl Server { } Action::SuspendWorkspaceDiagnostics(suspended_request) => { - self.session.set_suspended_workspace_diagnostics_request( + self.session.add_suspended_workspace_diagnostics_request( *suspended_request, &client, ); diff --git a/crates/ty_server/src/session.rs b/crates/ty_server/src/session.rs index 886f5eee35..ebe8f64830 100644 --- a/crates/ty_server/src/session.rs +++ b/crates/ty_server/src/session.rs @@ -15,8 +15,7 @@ use lsp_types::{ }; use lsp_types::{DidChangeWatchedFilesNotification, ExitNotification, Notification}; use lsp_types::{ - DocumentDiagnosticRequest, RegistrationRequest, Request, ShutdownRequest, - UnregistrationRequest, WorkspaceDiagnosticRequest, + DocumentDiagnosticRequest, RegistrationRequest, Request, ShutdownRequest, UnregistrationRequest, }; use ruff_db::Db; use ruff_db::files::{File, system_path_to_file}; @@ -40,8 +39,8 @@ use crate::capabilities::{ResolvedClientCapabilities, server_diagnostic_options} use crate::db::Db as _; use crate::document::{DocumentKey, DocumentVersion, LanguageId, NotebookDocument}; use crate::server::{ - Action, LazyWorkDoneProgress, ScriptProgress, publish_all_document_diagnostics, - publish_diagnostics_if_needed, publish_settings_diagnostics, + Action, LazyWorkDoneProgress, ScriptProgress, SuspendedWorkspaceRequestKind, + publish_all_document_diagnostics, publish_diagnostics_if_needed, publish_settings_diagnostics, }; use crate::session::client::Client; use crate::session::index::Document; @@ -110,11 +109,21 @@ pub(crate) struct Session { /// could result in different workspace diagnostics. revision: u64, - /// A pending workspace diagnostics request because there were no diagnostics - /// or no changes when when the request ran last time. - /// We'll re-run the request after every change to `Session` (see `revision`) - /// to see if there are now changes and, if so, respond to the client. - suspended_workspace_diagnostics_request: Option, + /// the workspace diagnostics requests held open, see [`SuspendedWorkspaceDiagnosticRequest`]. + /// each is run again after every change to `Session` (see `revision`), and answered once + /// it has something to say + /// + /// more than one, because a client can have a `workspace/diagnostic` request held open and + /// send a `by/checkWorkspace` beside it, and neither may be dropped: a request that is never + /// answered is one the client waits on until it gives up + /// + /// each is run again on its own, so a change costs one workspace check per request held. the + /// checks share the per-file results salsa caches, so the second and later ones mostly redo + /// the walk over the project's files and the building of each report. they are not merged + /// into one check, because each request compares against its own previous result ids and + /// reports its own progress, and is re-dispatched like any other request so that a + /// cancellation or a retry reaches it alone + suspended_workspace_diagnostics_requests: Vec, /// Document requests about a text the session does not hold yet, each waiting for the session /// to change. See [`HeldRequest`]. @@ -207,7 +216,7 @@ impl Session { request_queue: RequestQueue::new(), shutdown_requested: false, in_test, - suspended_workspace_diagnostics_request: None, + suspended_workspace_diagnostics_requests: Vec::new(), held_requests: Vec::new(), revision: 0, registrations: HashSet::new(), @@ -246,35 +255,42 @@ impl Session { self.shutdown_requested = requested; } - pub(crate) fn set_suspended_workspace_diagnostics_request( + /// holds `request` open beside any others already held + /// + /// once the server is shutting down nothing resumes a held request, so one whose check was + /// still running when the `shutdown` request came is answered now, as `shutdown` answered + /// the ones held before it + pub(crate) fn add_suspended_workspace_diagnostics_request( &mut self, request: SuspendedWorkspaceDiagnosticRequest, client: &Client, ) { - self.suspended_workspace_diagnostics_request = Some(request); + if self.shutdown_requested { + request.answer_at_shutdown(client); + return; + } + self.suspended_workspace_diagnostics_requests.push(request); // Run the suspended workspace diagnostic request immediately in case there // were changes since the workspace diagnostics background thread queued // the action to suspend the workspace diagnostic request. - self.resume_suspended_workspace_diagnostic_request(client); + self.resume_suspended_workspace_diagnostic_requests(client); } - pub(crate) fn take_suspended_workspace_diagnostic_request( + pub(crate) fn take_suspended_workspace_diagnostic_requests( &mut self, - ) -> Option { - self.suspended_workspace_diagnostics_request.take() + ) -> Vec { + std::mem::take(&mut self.suspended_workspace_diagnostics_requests) } - /// Resumes (retries) the workspace diagnostic request if there - /// were any changes to the [`Session`] (the revision got bumped) - /// since the workspace diagnostic request ran last time. + /// runs each held workspace diagnostics request again if the [`Session`] changed (the + /// revision got bumped) since it last ran /// - /// The workspace diagnostic requests is ignored if the request - /// was cancelled in the meantime. - pub(crate) fn resume_suspended_workspace_diagnostic_request(&mut self, client: &Client) { - self.suspended_workspace_diagnostics_request = self - .suspended_workspace_diagnostics_request - .take() - .and_then(|request| { + /// a request the client cancelled in the meantime is dropped + pub(crate) fn resume_suspended_workspace_diagnostic_requests(&mut self, client: &Client) { + let suspended = std::mem::take(&mut self.suspended_workspace_diagnostics_requests); + self.suspended_workspace_diagnostics_requests = suspended + .into_iter() + .filter_map(|request| { if !self.request_queue.incoming().is_pending(&request.id) { // Clear out the suspended request if the request has been cancelled. tracing::debug!( @@ -286,7 +302,8 @@ impl Session { } request.resume_if_revision_changed(self.revision, client) - }); + }) + .collect(); } /// Keeps `request` until the session changes, because what it asks about is a text the session @@ -420,7 +437,7 @@ impl Session { self.bump_revision(); - self.resume_suspended_workspace_diagnostic_request(client); + self.resume_suspended_workspace_diagnostic_requests(client); let capabilities = self.client_capabilities(); if capabilities.supports_workspace_diagnostic_refresh() { @@ -2147,14 +2164,18 @@ impl Workspace { } } -/// A workspace diagnostic request that didn't yield any changes or diagnostic -/// when it ran the last time. +/// a workspace diagnostics request held open until the session changes: a `workspace/diagnostic` +/// request whose last run found nothing that differs from what the client has, or either kind of +/// request when the workspace was not ready to be checked #[derive(Debug)] pub(crate) struct SuspendedWorkspaceDiagnosticRequest { /// The LSP request id pub(crate) id: RequestId, - /// The params passed to the `workspace/diagnostic` request. + /// which request this is + pub(crate) kind: SuspendedWorkspaceRequestKind, + + /// the params the request was sent with pub(crate) params: serde_json::Value, /// The session's revision when the request ran the last time. @@ -2179,12 +2200,17 @@ impl SuspendedWorkspaceDiagnosticRequest { tracing::debug!("Resuming workspace diagnostics request after revision bump"); client.queue_action(Action::RetryRequest(lsp_server::Request { id: self.id, - method: WorkspaceDiagnosticRequest::METHOD.to_string(), + method: self.kind.method(), params: self.params, })); None } + + /// answers the request when the server shuts down before it could be run again + pub(crate) fn answer_at_shutdown(self, client: &Client) { + self.kind.answer_at_shutdown(&self.id, client); + } } /// How long a request may wait for its text before it is answered with `ServerCancelled`. diff --git a/crates/ty_server/tests/e2e/check_workspace.rs b/crates/ty_server/tests/e2e/check_workspace.rs new file mode 100644 index 0000000000..390969c7a8 --- /dev/null +++ b/crates/ty_server/tests/e2e/check_workspace.rs @@ -0,0 +1,417 @@ +//! `by/checkWorkspace` over the wire +//! +//! the report is `workspace/diagnostic`'s, from the same check, and the pull diagnostics tests +//! cover what is in it. what these cover is when it comes back: straight away, whatever it says — +//! which is the difference between the two, since `workspace/diagnostic` holds on to a request +//! whose answer the client already has — and how it comes back + +use std::time::Duration; + +use anyhow::Result; +use lsp_server::ErrorCode; +use lsp_types::{ + LspRequestMethod, MessageDirection, PartialResultParams, PreviousResultId, + ProgressNotification, ProgressToken, Request, WorkDoneProgressParams, + WorkspaceDiagnosticParams, WorkspaceDiagnosticReport, WorkspaceDocumentDiagnosticReport, +}; +use ruff_db::system::SystemPath; +use ty_server::{ClientOptions, DiagnosticMode}; + +use crate::pull_diagnostics::{ + assert_workspace_diagnostics_suspends_for_long_polling, send_workspace_diagnostic_request, + shutdown_and_await_workspace_diagnostic, +}; +use crate::{AwaitResponseError, TestServer, TestServerBuilder}; + +/// the request as a client sends it +enum CheckWorkspace {} + +impl Request for CheckWorkspace { + type Params = WorkspaceDiagnosticParams; + type Result = WorkspaceDiagnosticReport; + const METHOD: LspRequestMethod<'static> = LspRequestMethod::Custom("by/checkWorkspace"); + const MESSAGE_DIRECTION: MessageDirection = MessageDirection::ClientToServer; +} + +fn params(previous_result_ids: Vec) -> WorkspaceDiagnosticParams { + WorkspaceDiagnosticParams { + identifier: None, + previous_result_ids, + work_done_progress_params: WorkDoneProgressParams::default(), + partial_result_params: PartialResultParams::default(), + } +} + +/// asks, and waits no longer than a request that is answered straight away takes +fn check( + server: &mut TestServer, + previous_result_ids: Vec, +) -> Result { + let id = server.send_request::(params(previous_result_ids)); + server.try_await_response::(&id, Some(Duration::from_secs(10))) +} + +fn workspace_server(files: &[(&str, &str)], mode: DiagnosticMode) -> Result { + let mut builder = TestServerBuilder::new()? + .with_workspace(SystemPath::new("src"), None)? + .with_initialization_options(&ClientOptions::default().with_diagnostic_mode(mode)); + for (path, content) in files { + builder = builder.with_file(SystemPath::new(path), *content)?; + } + Ok(builder.build().wait_until_workspaces_are_initialized()) +} + +const CLEAN: &str = "def hello() -> str:\n return \"world\"\n"; +const BROKEN: &str = "def hello() -> str:\n return 42\n"; + +fn result_ids(report: &WorkspaceDiagnosticReport) -> Vec { + report + .items + .iter() + .filter_map(|item| match item { + WorkspaceDocumentDiagnosticReport::WorkspaceFullDocumentDiagnosticReport(full) => { + Some(PreviousResultId { + uri: full.uri.clone(), + value: full.full_document_diagnostic_report.result_id.clone()?, + }) + } + WorkspaceDocumentDiagnosticReport::WorkspaceUnchangedDocumentDiagnosticReport(_) => { + None + } + }) + .collect() +} + +/// a project with nothing wrong in it is an answer too: an empty report, not a request held open +/// until something goes wrong +#[test] +fn a_project_with_no_diagnostics_is_answered() -> Result<()> { + let mut server = workspace_server(&[("src/clean.py", CLEAN)], DiagnosticMode::Workspace)?; + + // the long poll holds on to exactly this request + let long_poll = send_workspace_diagnostic_request(&mut server); + assert_workspace_diagnostics_suspends_for_long_polling(&mut server, &long_poll); + + let report = check(&mut server, Vec::new()).expect("answered straight away"); + assert!(report.items.is_empty(), "{report:#?}"); + + // and the long poll still has nothing to say until the server goes away + let long_poll_report = shutdown_and_await_workspace_diagnostic(server, &long_poll); + assert!(long_poll_report.items.is_empty(), "{long_poll_report:#?}"); + + Ok(()) +} + +/// a check that finds what the client already has says so, file by file, rather than waiting for +/// a change +#[test] +fn an_unchanged_workspace_is_answered_with_unchanged_reports() -> Result<()> { + let mut server = workspace_server(&[("src/broken.py", BROKEN)], DiagnosticMode::Workspace)?; + + let first = check(&mut server, Vec::new()).expect("answered"); + let ids = result_ids(&first); + assert_eq!(ids.len(), 1, "{first:#?}"); + + let second = check(&mut server, ids).expect("answered although nothing changed"); + assert!( + matches!( + second.items.as_slice(), + [WorkspaceDocumentDiagnosticReport::WorkspaceUnchangedDocumentDiagnosticReport(_)] + ), + "{second:#?}" + ); + + Ok(()) +} + +/// in the mode that checks open files only, the workspace is not checked, and an empty report +/// would say it has no diagnostics +#[test] +fn a_workspace_that_is_not_checked_is_refused() -> Result<()> { + let mut server = workspace_server(&[("src/broken.py", BROKEN)], DiagnosticMode::OpenFilesOnly)?; + + match check(&mut server, Vec::new()) { + Err(AwaitResponseError::RequestFailed(error)) => { + assert_eq!(error.code, ErrorCode::RequestFailed as i32, "{error:?}"); + } + other => panic!("expected the request to be refused, got {other:?}"), + } + + Ok(()) +} + +/// the report comes whole, in the response, even when the params ask for partial results and +/// progress under the client's tokens: a check that an edit interrupts starts again, and nothing +/// sent under those tokens can be taken back +#[test] +fn the_report_comes_whole_in_the_response() -> Result<()> { + // the server streams partial results in batches of two in tests, so three files make at + // least one batch + let mut server = workspace_server( + &[ + ("src/a.py", BROKEN), + ("src/b.py", BROKEN), + ("src/c.py", BROKEN), + ], + DiagnosticMode::Workspace, + )?; + + let id = server.send_request::(WorkspaceDiagnosticParams { + work_done_progress_params: WorkDoneProgressParams { + work_done_token: Some(ProgressToken::String("work-done".to_string())), + }, + partial_result_params: PartialResultParams { + partial_result_token: Some(ProgressToken::String("partial-result".to_string())), + }, + ..params(Vec::new()) + }); + let report = server + .try_await_response::(&id, Some(Duration::from_secs(10))) + .expect("answered"); + assert_eq!(result_ids(&report).len(), 3, "{report:#?}"); + + if let Ok(progress) = + server.try_await_notification::(Some(Duration::from_secs(1))) + { + panic!("expected no progress under the client's tokens, got {progress:#?}"); + } + + Ok(()) +} + +/// a workspace whose script is still having its environment set up is not checked yet, because +/// the script would be checked against an environment it does not have. the check is held open +/// until the environment is there, like a long poll, and these cover what happens to it +/// meanwhile +/// +/// `uv` here is a stand-in that holds the setup until the test releases it +#[cfg(unix)] +mod a_script_environment_being_set_up { + use std::fs; + use std::os::unix::fs::PermissionsExt; + use std::time::{Duration, Instant}; + + use anyhow::{Result, bail}; + use lsp_server::{ErrorCode, RequestId}; + use lsp_types::{Uri, WorkspaceDiagnosticReport, WorkspaceDiagnosticRequest}; + use ruff_db::system::{SystemPath, SystemPathBuf}; + use ty_project::UseUv; + use ty_server::{ClientOptions, DiagnosticMode}; + + use super::{CheckWorkspace, params}; + use crate::pull_diagnostics::{ + assert_workspace_diagnostics_suspends_for_long_polling, send_workspace_diagnostic_request, + }; + use crate::{AwaitResponseError, TestServer, TestServerBuilder}; + + /// run beside the script, it says it has started, waits to be released, and fails, which + /// ends the setup with an environment that has an error rather than one that is pending + const UV: &str = "#!/bin/sh +touch started +i=0 +while [ ! -e release ] && [ \"$i\" -lt 3000 ]; do + sleep 0.01 + i=$((i + 1)) +done +exit 1 +"; + + /// the stand-in `uv`'s two signals, as files beside the script + struct Setup { + started: SystemPathBuf, + release: SystemPathBuf, + } + + impl Setup { + fn wait_until_started(&self) -> Result<()> { + let deadline = Instant::now() + Duration::from_secs(10); + while !self.started.as_std_path().exists() { + if Instant::now() > deadline { + bail!("the script's environment setup never started"); + } + std::thread::sleep(Duration::from_millis(10)); + } + Ok(()) + } + + fn release(&self) { + // an error here leaves the setup to the stand-in's own time limit + let _ = fs::write(self.release.as_std_path(), ""); + } + } + + impl Drop for Setup { + // the server waits for the setup when it exits + fn drop(&mut self) { + self.release(); + } + } + + fn server() -> Result<(TestServer, Setup)> { + let builder = TestServerBuilder::new()? + .with_workspace(SystemPath::new("src"), None)? + .with_initialization_options( + &ClientOptions::default().with_diagnostic_mode(DiagnosticMode::Workspace), + ) + .with_use_uv(UseUv::Scripts) + .with_file("src/main.py", "missing\n")? + .with_file( + "src/script.py", + "# /// script\n# dependencies = []\n# ///\n", + )? + .with_file("bin/uv", UV)?; + let uv = builder.file_path("bin/uv"); + fs::set_permissions(uv.as_std_path(), fs::Permissions::from_mode(0o755))?; + let setup = Setup { + started: builder.file_path("src/started"), + release: builder.file_path("src/release"), + }; + let server = builder + .with_env_var("UV", uv.as_str()) + .build() + .wait_until_workspaces_are_initialized(); + setup.wait_until_started()?; + Ok((server, setup)) + } + + fn answer>( + server: &mut TestServer, + id: &RequestId, + ) -> WorkspaceDiagnosticReport { + server + .try_await_response::(id, Some(Duration::from_secs(10))) + .unwrap_or_else(|err| panic!("request {id} was not answered: {err}")) + } + + fn reports_on(report: &WorkspaceDiagnosticReport, uri: &Uri) -> bool { + report.items.iter().any(|item| match item { + lsp_types::WorkspaceDocumentDiagnosticReport::WorkspaceFullDocumentDiagnosticReport( + full, + ) => &full.uri == uri, + lsp_types::WorkspaceDocumentDiagnosticReport::WorkspaceUnchangedDocumentDiagnosticReport( + unchanged, + ) => &unchanged.uri == uri, + }) + } + + fn assert_refused(server: &mut TestServer, id: &RequestId, code: ErrorCode) { + match server.try_await_response::(id, Some(Duration::from_secs(10))) { + Err(AwaitResponseError::RequestFailed(error)) => { + assert_eq!(error.code, code as i32, "{error:?}"); + } + other => panic!("expected the check to be refused with {code:?}, got {other:?}"), + } + } + + #[track_caller] + fn assert_held(server: &mut TestServer, id: &RequestId) { + match server.try_await_response::(id, Some(Duration::from_secs(2))) { + Err(AwaitResponseError::Timeout) => {} + other => panic!("expected the check to be held open, got {other:?}"), + } + } + + fn shutdown(server: &mut TestServer) -> RequestId { + server.send_request::(()) + } + + fn exit(server: &mut TestServer, shutdown: &RequestId) { + server.await_response::(shutdown); + server.send_notification::(()); + } + + /// the check is answered once the environment is there, with the whole workspace checked + #[test] + fn a_check_is_answered_once_the_environment_is_there() -> Result<()> { + let (mut server, setup) = server()?; + + let check = server.send_request::(params(Vec::new())); + assert_held(&mut server, &check); + + setup.release(); + let report = answer::(&mut server, &check); + assert!( + reports_on(&report, &server.file_uri("src/main.py")), + "{report:#?}" + ); + + Ok(()) + } + + /// a long poll waits for the environment too, and holding the one does not drop the other + #[test] + fn a_held_check_and_a_held_long_poll_are_both_answered() -> Result<()> { + let (mut server, setup) = server()?; + + let long_poll = send_workspace_diagnostic_request(&mut server); + assert_workspace_diagnostics_suspends_for_long_polling(&mut server, &long_poll); + let check = server.send_request::(params(Vec::new())); + assert_held(&mut server, &check); + + setup.release(); + let main = server.file_uri("src/main.py"); + let report = answer::(&mut server, &long_poll); + assert!(reports_on(&report, &main), "{report:#?}"); + let report = answer::(&mut server, &check); + assert!(reports_on(&report, &main), "{report:#?}"); + + Ok(()) + } + + /// at shutdown the long poll is answered with an empty report, since nothing changed, but the + /// check is refused: an empty report would say the workspace has no diagnostics, and it was + /// never checked + #[test] + fn a_check_held_at_shutdown_is_refused() -> Result<()> { + let (mut server, _setup) = server()?; + + let long_poll = send_workspace_diagnostic_request(&mut server); + assert_workspace_diagnostics_suspends_for_long_polling(&mut server, &long_poll); + let check = server.send_request::(params(Vec::new())); + assert_held(&mut server, &check); + + let shutdown = shutdown(&mut server); + let report = answer::(&mut server, &long_poll); + assert!(report.items.is_empty(), "{report:#?}"); + assert_refused(&mut server, &check, ErrorCode::RequestFailed); + exit(&mut server, &shutdown); + + Ok(()) + } + + /// a check that has not been held yet when the `shutdown` request comes, because it is still + /// finding out that the workspace is not ready, is refused as a held one is rather than held + /// by a server that will not run it again + #[test] + fn a_check_still_running_at_shutdown_is_refused() -> Result<()> { + let (mut server, _setup) = server()?; + + let check = server.send_request::(params(Vec::new())); + let shutdown = shutdown(&mut server); + assert_refused(&mut server, &check, ErrorCode::RequestFailed); + exit(&mut server, &shutdown); + + Ok(()) + } + + /// a held check the client cancels is answered as cancelled, once, and is not run again when + /// the environment is there + #[test] + fn a_held_check_can_be_cancelled() -> Result<()> { + let (mut server, setup) = server()?; + + let check = server.send_request::(params(Vec::new())); + assert_held(&mut server, &check); + server.cancel(&check); + assert_refused(&mut server, &check, ErrorCode::RequestCanceled); + + // a second check is answered once the environment is there, by which time the cancelled + // one would have been run again too. the test server's drop asserts that no other + // response came + setup.release(); + let second = server.send_request::(params(Vec::new())); + answer::(&mut server, &second); + + Ok(()) + } +} diff --git a/crates/ty_server/tests/e2e/main.rs b/crates/ty_server/tests/e2e/main.rs index 9ba139fe74..f8deb3949e 100644 --- a/crates/ty_server/tests/e2e/main.rs +++ b/crates/ty_server/tests/e2e/main.rs @@ -31,6 +31,7 @@ mod alignment_groups; mod build_output; mod build_trees; mod call_hierarchy; +mod check_workspace; mod code_actions; mod commands; mod completions; diff --git a/crates/ty_server/tests/e2e/pull_diagnostics.rs b/crates/ty_server/tests/e2e/pull_diagnostics.rs index 76b0f801d8..50c5b717e9 100644 --- a/crates/ty_server/tests/e2e/pull_diagnostics.rs +++ b/crates/ty_server/tests/e2e/pull_diagnostics.rs @@ -1346,6 +1346,52 @@ def hello() -> str: Ok(()) } +/// every long-polling request held open at once is answered when something changes, not only the +/// last one held +#[test] +fn workspace_diagnostic_long_polling_answers_every_held_request() -> Result<()> { + let workspace_root = SystemPath::new("src"); + let file_path = SystemPath::new("src/test.py"); + let file_content_no_error = "\ +def hello() -> str: + return \"world\" +"; + let file_content_with_error = "\ +def hello() -> str: + return 42 +"; + + let mut server = + create_workspace_server_with_file(workspace_root, file_path, file_content_no_error)?; + server.open_text_document(file_path, file_content_no_error, 1); + + let first = send_workspace_diagnostic_request(&mut server); + assert_workspace_diagnostics_suspends_for_long_polling(&mut server, &first); + let second = send_workspace_diagnostic_request(&mut server); + assert_workspace_diagnostics_suspends_for_long_polling(&mut server, &second); + + server.change_text_document( + file_path, + vec![ + lsp_types::TextDocumentContentChangeEvent::TextDocumentContentChangeWholeDocument( + TextDocumentContentChangeWholeDocument { + text: file_content_with_error.to_string(), + }, + ), + ], + 2, + ); + + for id in [first, second] { + let report = server + .try_await_response::(&id, Some(Duration::from_secs(10))) + .unwrap_or_else(|err| panic!("request {id} was not answered: {err}")); + assert_eq!(report.items.len(), 1, "{report:#?}"); + } + + Ok(()) +} + /// Regression test for diagnostics disappearing in some cases. /// /// The specific way this fails is when a file that was never in the "open diff --git a/docs/basedpython/features/editor.md b/docs/basedpython/features/editor.md index a6376d2271..e676cff7ac 100644 --- a/docs/basedpython/features/editor.md +++ b/docs/basedpython/features/editor.md @@ -430,6 +430,44 @@ overrides, and `by/superMembers` still answers one. like the other document requests it takes `textHash`, and answers a document the client has not opened from the file on disk +## the whole project's diagnostics + +with `diagnosticMode` set to `workspace`, the server checks every file in the +project, open or not, and reports what it finds two ways: + +| request | answers | +| ---------------------- | ---------------------------------------------------------------------- | +| `workspace/diagnostic` | once something differs from the result ids sent — held open until then | +| `by/checkWorkspace` | as soon as the check is done, whether or not anything differs | + +both take the same parameters and report the same thing, file by file, with an +unchanged report for a file whose result id still matches. `workspace/diagnostic` +is the one an editor keeps a problems list current with: it asks again as soon +as it is answered, and the server answers when an edit or a change on disk +changes something. `by/checkWorkspace` is for a client that asks once — an +inspection over the whole project, a batch run with no editor — and would +otherwise wait forever on a project with nothing wrong in it. it is refused, +rather than answered empty, when the diagnostic mode does not check the +workspace + +they differ in how the report arrives. `workspace/diagnostic` streams it as +partial results when the client sends a `partialResultToken`, and reports +progress under the client's `workDoneToken`. `by/checkWorkspace` ignores both +tokens: its report comes whole, in the response, and its progress, when the +client supports `window/workDoneProgress`, comes under a token the server +creates + +`by/checkWorkspace` can take a while to answer, for two reasons: + +- an edit or a change on disk that lands while the workspace is being checked + starts the check again, on the changed workspace. the answer describes the + workspace as it was when a check finished, and a workspace that never stops + changing is never answered +- while a PEP 723 script's environment is being set up for the first time, the + workspace is not checked, and the request waits until the environment is + there. the first setup installs the script's dependencies, so it can take as + long as that does + ## the program model a run configuration, a test list and a "go to generated file" are all questions From 601eea8f8e6ec2d9303de64a9436cfa30e2d5a78 Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Sat, 26 Sep 2026 00:34:43 +1000 Subject: [PATCH 10/11] read whether a call to an unannotated function can return off the callee's control flow instead of inferring its whole body, so a module-level call no longer ties every body it reaches into one cycle with the module's globals --- crates/ty_python_core/src/builder.rs | 22 +- crates/ty_python_core/src/use_def.rs | 100 +- .../mdtest/basedpython_terminal_calls.md | 1071 +++++++++++++++++ crates/ty_python_semantic/src/reachability.rs | 236 +++- crates/ty_python_semantic/src/types.rs | 1 + .../src/types/definition_resolution.rs | 33 +- .../ty_python_semantic/src/types/function.rs | 375 +++--- .../src/types/infer/builder/function.rs | 2 +- .../src/types/inferred_signature.rs | 24 +- .../src/types/normal_completion.rs | 912 ++++++++++++++ docs/basedpython/features/sound-types.md | 21 +- 11 files changed, 2577 insertions(+), 220 deletions(-) create mode 100644 crates/ty_python_semantic/resources/mdtest/basedpython_terminal_calls.md create mode 100644 crates/ty_python_semantic/src/types/normal_completion.rs diff --git a/crates/ty_python_core/src/builder.rs b/crates/ty_python_core/src/builder.rs index 47a533630e..6af822d206 100644 --- a/crates/ty_python_core/src/builder.rs +++ b/crates/ty_python_core/src/builder.rs @@ -3179,6 +3179,21 @@ impl<'db, 'ast> SemanticIndexBuilder<'db, 'ast> { .record_negated_narrowing_constraint_for_places(predicate_id, &possibly_narrowed); } + /// basedpython: visits the `finally` suite of a `try` statement reached with `try_entry` + /// + /// the suite is visited once, in a single entry state, but an exception can enter it from + /// anywhere in the statement; see `UseDefMapBuilder::try_entries_of_finally_suites` + fn visit_finally_suite( + &mut self, + finalbody: &'ast [ast::Stmt], + try_entry: ScopedReachabilityConstraintId, + ) { + self.current_use_def_map_mut() + .enter_finally_suite(try_entry); + self.visit_block_body(finalbody); + self.current_use_def_map_mut().exit_finally_suite(); + } + /// Records that all remaining statements in the current block are unreachable. fn mark_unreachable(&mut self) { self.current_use_def_map_mut().mark_unreachable(); @@ -6484,6 +6499,7 @@ impl<'db, 'ast> SemanticIndexBuilder<'db, 'ast> { let was_in_finally_statement = self.in_finally_statement; self.in_finally_statement |= !finalbody.is_empty(); self.record_ambiguous_reachability(); + let try_entry = self.current_reachability(); let exception_handlers = if handlers.is_empty() { ExceptionHandlers::None @@ -6627,7 +6643,7 @@ impl<'db, 'ast> SemanticIndexBuilder<'db, 'ast> { for snapshot in snapshots { self.flow_merge(snapshot); } - self.visit_block_body(finalbody); + self.visit_finally_suite(finalbody, try_entry); if !self.flow_snapshot().is_always_unreachable() { if !finalbody.is_empty() && has_escaping_exception { self.record_exception_checkpoint(); @@ -6685,7 +6701,7 @@ impl<'db, 'ast> SemanticIndexBuilder<'db, 'ast> { } // Mixed normal and terminal entry states are still handled by the normal path // only. See the corresponding TODO tests in `terminal_statements.md`. - self.visit_block_body(finalbody); + self.visit_finally_suite(finalbody, try_entry); if !finalbody.is_empty() && has_escaping_exception && self.current_use_def_map().reachability @@ -6767,6 +6783,8 @@ impl<'db, 'ast> SemanticIndexBuilder<'db, 'ast> { self.record_returned_place_members(value); } self.record_return_exit(stmt); + self.current_use_def_map_mut() + .record_return_reachability(stmt.range()); self.record_terminal_finally_entry(); // Everything in the current block after a terminal statement is unreachable. self.mark_unreachable(); diff --git a/crates/ty_python_core/src/use_def.rs b/crates/ty_python_core/src/use_def.rs index ef7d4ed90c..77efdb903a 100644 --- a/crates/ty_python_core/src/use_def.rs +++ b/crates/ty_python_core/src/use_def.rs @@ -674,6 +674,10 @@ struct UseDefMapExtra { /// basedpython: the state of a function's parameter places over every way out of its body, /// merged. normal_exit: Box<[PlaceBindings]>, + + /// basedpython: whether each `return` in this scope can be reached, sorted by range. a + /// `return` that is always reachable is left out + return_reachability: Box<[(TextRange, ScopedReachabilityConstraintId)]>, } /// basedpython: the state of a function's parameter places where a `return` leaves the body. @@ -842,6 +846,13 @@ pub struct UseDefMap<'db> { /// /// This is used by `can_implicitly_return_none` in the `ty_python_semantic` crate. end_of_scope_reachability: ScopedReachabilityConstraintId, + + /// Whether control can leave the scope normally: reach a `return`, or the end of the scope. + /// + /// This is the union of [`Self::end_of_scope_reachability`] and the reachability of every + /// `return` statement in the scope. A function body for which it is always false never hands + /// control back to its caller. + normal_exit_reachability: ScopedReachabilityConstraintId, } /// Shares equivalent scope-local binding and declaration tables within a file. @@ -961,6 +972,11 @@ impl<'db> UseDefMap<'db> { self.end_of_scope_reachability } + /// Whether control can reach a `return` in this scope, or the end of it. + pub fn normal_exit_reachability(&self) -> ScopedReachabilityConstraintId { + self.normal_exit_reachability + } + /// Definitions relevant to usage analysis, including standalone declarations. /// /// The early declaration part of a combined definition is omitted: its later binding entry @@ -1070,6 +1086,23 @@ impl<'db> UseDefMap<'db> { )) } + /// basedpython: whether the `return` statement at `range` can be reached + /// + /// a `return` in a `finally` suite is reached through an exception entering the suite as well + /// as through the flow it is visited in; see `UseDefMapBuilder::record_return_reachability` + pub fn return_reachability(&self, range: TextRange) -> ScopedReachabilityConstraintId { + let returns = self + .extra + .as_deref() + .map(|extra| extra.return_reachability.as_ref()) + .unwrap_or_default(); + returns + .binary_search_by_key(&range.start(), |(range, _)| range.start()) + .map_or(ScopedReachabilityConstraintId::ALWAYS_TRUE, |index| { + returns[index].1 + }) + } + fn return_exit_records(&self) -> &[ReturnExit] { self.extra .as_deref() @@ -1958,6 +1991,21 @@ pub(super) struct UseDefMapBuilder<'db> { /// start of the scope. pub(super) reachability: ScopedReachabilityConstraintId, + /// The union of the reachability of every `return` recorded so far. + return_reachability: ScopedReachabilityConstraintId, + + /// basedpython: the reachability of each `return` recorded so far, when it is not always + /// reachable + return_reachability_by_range: Vec<(TextRange, ScopedReachabilityConstraintId)>, + + /// basedpython: for each `finally` suite being visited, the reachability of its `try` statement + /// + /// the suite is visited once, in the state that finishing the `try` statement leaves (or, when + /// it cannot finish, the states its `return`s and `raise`s leave). an exception can enter the + /// suite from anywhere in the statement, and a `return` in the suite swallows that exception, + /// so it is a way out of the scope wherever the statement is reached + try_entries_of_finally_suites: Vec, + /// Tracks the reachability constraint for statements and certain sub-expressions, /// keyed by their text range. range_reachability: Vec<(TextRange, RangeInfo)>, @@ -2017,6 +2065,9 @@ impl<'db> UseDefMapBuilder<'db> { return_exits: Vec::new(), normal_exit: Vec::new(), reachability: ScopedReachabilityConstraintId::ALWAYS_TRUE, + return_reachability: ScopedReachabilityConstraintId::ALWAYS_FALSE, + return_reachability_by_range: Vec::new(), + try_entries_of_finally_suites: Vec::new(), range_reachability: Vec::new(), checkpoint_flow: ScopedReachabilityConstraintId::ALWAYS_TRUE, checkpoint_state: ExceptionCheckpointState::default(), @@ -2674,6 +2725,39 @@ impl<'db> UseDefMapBuilder<'db> { } } + /// Record that control can leave the scope through a `return` at the current point. + /// + /// basedpython: a `return` in a `finally` suite is also reached through an exception entering + /// the suite. what the suite does before the `return` is not tracked on that path, so the + /// `return` counts as reached wherever the `try` statement is + pub(super) fn record_return_reachability(&mut self, range: TextRange) { + let reachability = self.try_entries_of_finally_suites.iter().fold( + self.reachability, + |reachability, &entry| { + self.reachability_constraints + .add_or_constraint(reachability, entry) + }, + ); + self.return_reachability = self + .reachability_constraints + .add_or_constraint(self.return_reachability, reachability); + if reachability != ScopedReachabilityConstraintId::ALWAYS_TRUE { + self.return_reachability_by_range + .push((range, reachability)); + } + } + + /// basedpython: start visiting the `finally` suite of a `try` statement reached with + /// `try_entry`; see [`Self::try_entries_of_finally_suites`] + pub(super) fn enter_finally_suite(&mut self, try_entry: ScopedReachabilityConstraintId) { + self.try_entries_of_finally_suites.push(try_entry); + } + + /// basedpython: finish visiting the `finally` suite [`Self::enter_finally_suite`] started + pub(super) fn exit_finally_suite(&mut self) { + self.try_entries_of_finally_suites.pop(); + } + /// basedpython: record the state of `places` where the `return` at `range` leaves the body. /// /// Unlike a use, this is the state a caller sees once the call has returned, so pending @@ -3144,6 +3228,11 @@ impl<'db> UseDefMapBuilder<'db> { } } self.reachability_constraints.mark_used(self.reachability); + let normal_exit_reachability = self + .reachability_constraints + .add_or_constraint(self.return_reachability, self.reachability); + self.reachability_constraints + .mark_used(normal_exit_reachability); let symbol_states = Self::zip_place_states(end_of_scope_symbols, reachable_definitions_by_symbol); let member_states = @@ -3152,12 +3241,19 @@ impl<'db> UseDefMapBuilder<'db> { let loop_headers = self.loop_headers; let return_exits = self.return_exits.into_boxed_slice(); let normal_exit = self.normal_exit.into_boxed_slice(); + let mut return_reachability = self.return_reachability_by_range; + return_reachability.sort_unstable_by_key(|(range, _)| range.start()); + for &(_, reachability) in &return_reachability { + self.reachability_constraints.mark_used(reachability); + } + let return_reachability = return_reachability.into_boxed_slice(); let extra = (!bindings_by_use.is_empty() || !member_states.is_empty() || !enclosing_snapshots.is_empty() || !loop_headers.is_empty() || !return_exits.is_empty() - || !normal_exit.is_empty()) + || !normal_exit.is_empty() + || !return_reachability.is_empty()) .then(|| { Box::new(UseDefMapExtra { bindings_by_use: bindings_by_use.into(), @@ -3167,6 +3263,7 @@ impl<'db> UseDefMapBuilder<'db> { loop_headers: loop_headers.into(), return_exits, normal_exit, + return_reachability, }) }); let predicates = self.predicates.build(); @@ -3196,6 +3293,7 @@ impl<'db> UseDefMapBuilder<'db> { definitions_by_definition, extra, end_of_scope_reachability: self.reachability, + normal_exit_reachability, } } diff --git a/crates/ty_python_semantic/resources/mdtest/basedpython_terminal_calls.md b/crates/ty_python_semantic/resources/mdtest/basedpython_terminal_calls.md new file mode 100644 index 0000000000..4d94ed96ca --- /dev/null +++ b/crates/ty_python_semantic/resources/mdtest/basedpython_terminal_calls.md @@ -0,0 +1,1071 @@ +# basedpython: calls to unannotated functions that never return + +A statement-level call ends the flow when the function it calls cannot return, that is, when its +return type is `Never`. A function annotated `-> NoReturn` says so in its signature. With +`infer-unannotated-signatures` (which `sound-types` implies), a function that leaves its return type +out has it recovered from its body: when no path through the body reaches a `return` or the end of +the body without passing something that cannot complete — a `raise`, a loop that never exits, or +another call that cannot return — the function returns `Never`, and the code after a call to it is +unreachable. + +```toml +[environment] +python-version = "3.13" + +[analysis] +infer-unannotated-signatures = true +``` + +## a function that exits + +`die` never returns, because `sys.exit` is declared `NoReturn`. The code after a call to it is +unreachable, so `x` is only ever `"test"` once the branches meet: + +```py +import sys + +def die(): + sys.exit(1) + +def f(flag: bool): + if flag: + x = "terminal" + die() + else: + x = "test" + reveal_type(x) # revealed: Literal["test"] +``` + +A function that ends with such a call does not implicitly return `None`: + +```py +def g() -> int: + die() +``` + +## a chain of functions that exit + +`a` calls `b`, which calls `sys.exit`, so neither returns: + +```py +import sys + +def b(): + sys.exit(1) + +def a(): + b() + +def f(flag: bool): + if flag: + x = "terminal" + a() + else: + x = "test" + reveal_type(x) # revealed: Literal["test"] + +def g() -> int: + a() +``` + +## a function that sometimes returns + +A function with a path that reaches the end of its body, or a `return`, can return, so the code +after a call to it stays reachable: + +```py +import sys + +def exit_if(flag: bool): + if flag: + sys.exit(1) + +def exit_unless(flag: bool): + if flag: + return + sys.exit(1) + +def f(flag: int): + if flag == 1: + x = "first" + exit_if(True) + elif flag == 2: + x = "second" + exit_unless(False) + else: + x = "test" + reveal_type(x) # revealed: Literal["first", "second", "test"] + +# error: [invalid-return-type] +def g() -> int: + exit_if(True) +``` + +## `raise` in every branch + +```py +def fail(flag: bool): + if flag: + raise ValueError + else: + raise TypeError + +def f(flag: bool): + if flag: + x = "terminal" + fail(flag) + else: + x = "test" + reveal_type(x) # revealed: Literal["test"] +``` + +## a loop that never exits + +`while True:` without a `break` never falls through: + +```py +def spin(): + while True: + pass + +def f(flag: bool): + if flag: + x = "terminal" + spin() + else: + x = "test" + reveal_type(x) # revealed: Literal["test"] +``` + +A `break` is a way out of the loop, and out of the function: + +```py +def spin_until(flag: bool): + while True: + if flag: + break + +def g(flag: bool): + if flag: + x = "call" + spin_until(flag) + else: + x = "test" + reveal_type(x) # revealed: Literal["call", "test"] +``` + +## `try` and `finally` + +A `finally` suite runs on the way out, but the exception that `sys.exit` raises still propagates +after it. An exit in the `finally` suite itself ends every path through the `try`: + +```py +import sys + +def exits_in_try(): + try: + sys.exit(1) + finally: + print("cleanup") + +def exits_in_finally(): + try: + print("work") + finally: + sys.exit(1) + +def f(flag: int): + if flag == 1: + x = "try" + exits_in_try() + elif flag == 2: + x = "finally" + exits_in_finally() + else: + x = "test" + reveal_type(x) # revealed: Literal["test"] +``` + +An `except` clause that catches the exception is a way out, whether the exception is raised in the +`try` suite itself or in a function it calls: + +```py +def caught(): + try: + sys.exit(1) + except SystemExit: + pass + +def caught_from_a_call(): + try: + exits_in_try() + except SystemExit: + pass + +def g(flag: int): + if flag == 1: + x = "raised" + caught() + elif flag == 2: + x = "called" + caught_from_a_call() + else: + x = "test" + reveal_type(x) # revealed: Literal["raised", "called", "test"] +``` + +A `return` in the `finally` suite swallows the exception, so a function whose `try` suite exits +still returns when its `finally` suite does. That holds whether the `try` suite exits itself or +calls a function that does: + +```py +def returns_in_finally(): + try: + sys.exit(1) + finally: + return + +def returns_in_finally_after_a_call(): + try: + exits_in_try() + finally: + return + +def h(flag: int): + if flag == 1: + x = "exit" + returns_in_finally() + elif flag == 2: + x = "call" + returns_in_finally_after_a_call() + else: + x = "test" + reveal_type(x) # revealed: Literal["exit", "call", "test"] +``` + +A `break` in the `finally` suite swallows the exception too, and the function goes on after the +loop. The flow after a `finally` suite is only followed from the state the `try` suite finished in, +so that way out is not seen yet: + +```py +def breaks_in_finally(): + while True: + try: + sys.exit(1) + finally: + break + +def k(flag: bool): + if flag: + x = "call" + breaks_in_finally() + else: + x = "test" + # TODO: `breaks_in_finally()` returns, so this should be `Literal["call", "test"]` + reveal_type(x) # revealed: Literal["test"] +``` + +## recursion + +A function that calls itself unconditionally never returns: every call makes another, until +`RecursionError` is raised. + +```py +def forever(): + forever() + +def f(flag: bool): + if flag: + x = "terminal" + forever() + else: + x = "test" + reveal_type(x) # revealed: Literal["test"] +``` + +A recursive call behind a condition leaves the other path, which returns: + +```py +def countdown(n: int): + if n: + countdown(n - 1) + +def g(flag: bool): + if flag: + x = "call" + countdown(3) + else: + x = "test" + reveal_type(x) # revealed: Literal["call", "test"] +``` + +Mutual recursion is read the same way. `ping` and `pong` only ever call each other, so neither +returns: + +```py +def ping(): + pong() + +def pong(): + ping() + +def h(flag: bool): + if flag: + x = "terminal" + ping() + else: + x = "test" + reveal_type(x) # revealed: Literal["test"] +``` + +When one of them has a way out, both do: + +```py +def tick(n: int): + if n: + tock(n - 1) + +def tock(n: int): + tick(n) + +def k(flag: bool): + if flag: + x = "call" + tock(3) + else: + x = "test" + reveal_type(x) # revealed: Literal["call", "test"] +``` + +## a body that only types can read + +A body that exits only on a path that its types decide is read through those types, like any other +body. + +`exit` is a builtin object rather than a function, and calling it never returns: + +```py +import sys + +def exits_through_a_builtin(): + exit(1) + +def f(flag: bool): + if flag: + x = "terminal" + exits_through_a_builtin() + else: + x = "test" + reveal_type(x) # revealed: Literal["test"] + reveal_type(exits_through_a_builtin()) # revealed: Never +``` + +A condition that is always true because of its type, rather than because it is a literal, is read +through that type: + +```py +def exits_behind_isinstance(): + if isinstance(sys.argv, list): + sys.exit(1) + +def g(flag: bool): + if flag: + x = "terminal" + exits_behind_isinstance() + else: + x = "test" + reveal_type(x) # revealed: Literal["test"] + reveal_type(exits_behind_isinstance()) # revealed: Never +``` + +A function that ends with such a call does not implicitly return `None`: + +```py +def h() -> int: + exits_behind_isinstance() +``` + +A method called on a module global is read through the global's type. `ArgumentParser.error` never +returns: + +```py +import argparse + +parser = argparse.ArgumentParser() + +def usage(message: str): + parser.error(message) + +def k(flag: bool): + if flag: + x = "terminal" + usage("bad") + else: + x = "test" + reveal_type(x) # revealed: Literal["test"] +``` + +A method called through `self` in a class with bases is read through the type of `self`: + +```py +class Tool: + def describe(self) -> str: + return "tool" + +class Runner(Tool): + def die(self): + sys.exit(1) + + def run(self): + self.die() + +def m(flag: bool, runner: Runner): + if flag: + x = "terminal" + runner.run() + else: + x = "test" + reveal_type(x) # revealed: Literal["test"] +``` + +A function called through another name bound to it is read through that name's type: + +```py +def die(): + sys.exit(1) + +alias = die + +def through_an_alias(): + alias() + +def n(flag: bool): + if flag: + x = "terminal" + through_an_alias() + else: + x = "test" + reveal_type(x) # revealed: Literal["test"] +``` + +## what a `return` hands back + +A `return` that can never be reached hands nothing back, so it plays no part in the return type. A +function that exits before its only `return` returns `Never`, and a call to it ends the flow: + +```py +import sys + +def exits_before_returning(): + sys.exit(1) + return 1 + +def f(flag: bool): + if flag: + x = "terminal" + exits_before_returning() + else: + x = "test" + reveal_type(x) # revealed: Literal["test"] + reveal_type(exits_before_returning()) # revealed: Never +``` + +A `return` whose value never finishes evaluating returns `Never` too, but a call to it is still read +as one that reaches its `return`: + +```py +def returns_an_exit(): + return sys.exit(1) + +def g(flag: bool): + if flag: + x = "call" + returns_an_exit() + else: + x = "test" + # TODO: `returns_an_exit()` returns `Never`, so this should be `Literal["test"]` + reveal_type(x) # revealed: Literal["call", "test"] + reveal_type(returns_an_exit()) # revealed: Never +``` + +## a callee that cannot be resolved + +A call the checker cannot see through is treated as returning. A call to a parameter is one: which +function it calls is up to the caller. + +```py +import sys + +def call(callback): + callback() + +def exit_one(): + sys.exit(1) + +def f(flag: bool): + if flag: + x = "call" + call(exit_one) + else: + x = "test" + reveal_type(x) # revealed: Literal["call", "test"] +``` + +A name that is bound to one of two functions, depending on a condition, is another, even though +neither function returns: + +```py +def exit_two(): + sys.exit(2) + +def choose(flag: bool): + if flag: + chosen = exit_one + else: + chosen = exit_two + chosen() + +def g(flag: bool): + if flag: + x = "call" + choose(True) + else: + x = "test" + reveal_type(x) # revealed: Literal["call", "test"] +``` + +A call to something that does not resolve at all returns too: + +```py +from unknown_module import helper # error: [unresolved-import] + +def calls_unknown(): + helper() + +def h(flag: bool): + if flag: + x = "call" + calls_unknown() + else: + x = "test" + reveal_type(x) # revealed: Literal["call", "test"] +``` + +## methods + +A method and a `classmethod` are read the same way as a function, whether they are called directly +or through `self` and `cls`: + +```py +import sys + +class Exiter: + def die(self): + sys.exit(1) + + @classmethod + def class_die(cls): + sys.exit(1) + + def die_through_self(self): + self.die() + + @classmethod + def die_through_cls(cls): + cls.class_die() + +def f(flag: int, exiter: Exiter): + if flag == 1: + x = "method" + exiter.die() + elif flag == 2: + x = "classmethod" + Exiter.class_die() + elif flag == 3: + x = "self" + exiter.die_through_self() + elif flag == 4: + x = "cls" + Exiter.die_through_cls() + else: + x = "test" + reveal_type(x) # revealed: Literal["test"] +``` + +A `staticmethod` is read the same way, whether it is called through the class directly or from the +body of another function: + +```py +class Tools: + @staticmethod + def die(): + sys.exit(1) + +def dies_through_the_class(): + Tools.die() + +def h(flag: int): + if flag == 1: + x = "direct" + Tools.die() + elif flag == 2: + x = "through a function" + dies_through_the_class() + else: + x = "test" + reveal_type(x) # revealed: Literal["test"] +``` + +Every class inherits what `object` defines, and all of it states its return type. A method that +overrides one of those, such as `__init__`, takes its return type from `object` rather than from its +body, so a call to it returns: + +```py +class Resettable: + def __init__(self): + sys.exit(1) + + def reset(self): + self.__init__() + +def k(flag: bool, resettable: Resettable): + if flag: + x = "call" + resettable.reset() + else: + x = "test" + reveal_type(x) # revealed: Literal["call", "test"] +``` + +An override that leaves its return type out takes the one its base declares, so its body does not +decide whether a call to it returns: + +```py +class Base: + def run(self) -> int: + return 1 + +class Child(Base): + def run(self): + sys.exit(1) + +def g(flag: bool, child: Child): + if flag: + x = "call" + child.run() + else: + x = "test" + reveal_type(x) # revealed: Literal["call", "test"] +``` + +A base that leaves its own return type out supplies the one recovered from its body in turn, so an +override of a method that cannot return cannot return either: + +```py +class Stopper: + def stop(self): + sys.exit(1) + +class LoggingStopper(Stopper): + def stop(self): + print("stopping") + +def m(flag: bool, stopper: LoggingStopper): + if flag: + x = "call" + stopper.stop() + else: + x = "test" + # TODO: `LoggingStopper.stop` can return, which the `Never` it takes from its base hides + reveal_type(x) # revealed: Literal["test"] + reveal_type(stopper.stop()) # revealed: Never +``` + +A base whose return type cannot stand for an override's leaves the override's return type gradual, +and a call to it returns. Which type an overloaded base returns depends on the arguments, and a type +variable in the base's return type belongs to the base: + +```py +from typing import overload + +class Parser: + @overload + def parse(self, text: str) -> str: ... + @overload + def parse(self, text: bytes) -> bytes: ... + def parse(self, text: str | bytes) -> str | bytes: + return text + +class StrictParser(Parser): + def parse(self, text): + sys.exit(1) + +class Echo: + def echo[T](self, value: T) -> T: + return value + +class SilentEcho(Echo): + def echo(self, value): + sys.exit(1) + +def n(flag: int, parser: StrictParser, echo: SilentEcho): + if flag == 1: + x = "overloaded" + parser.parse("text") + elif flag == 2: + x = "generic" + echo.echo(1) + else: + x = "test" + reveal_type(x) # revealed: Literal["overloaded", "generic", "test"] +``` + +A narrowing return type is not handed to an override at all: it is a claim about what the base's +body tests, which the override's body need not test. The override's body decides instead: + +```py +from typing_extensions import TypeIs + +class Checker: + def is_int(self, value: object) -> TypeIs[int]: + return isinstance(value, int) + +class FailingChecker(Checker): + def is_int(self, value): + sys.exit(1) + +def p(flag: bool, checker: FailingChecker): + if flag: + x = "call" + checker.is_int(1) + else: + x = "test" + reveal_type(x) # revealed: Literal["test"] + reveal_type(checker.is_int(1)) # revealed: Never +``` + +## decorated functions + +A decorator can replace the function it decorates, so a call to a decorated function returns +whatever the decorator's result does. Here that result is a function that returns, so neither call +ends the flow, though the body the decorator was handed exits: + +```py +import sys +from typing import Callable + +def replace(function) -> Callable[[], None]: + return lambda: None + +@replace +def exits(): + sys.exit(1) + +def calls_decorated(): + exits() + +def f(flag: int): + if flag == 1: + x = "direct" + exits() + elif flag == 2: + x = "through a function" + calls_decorated() + else: + x = "test" + reveal_type(x) # revealed: Literal["direct", "through a function", "test"] +``` + +## callees in other modules + +A call to a function in another module is read the same way, whichever form of import names the +function or its module: + +`tools/exits.py`: + +```py +import sys + +def die(): + sys.exit(1) +``` + +`everything.py`: + +```py +from tools.exits import * + +def through_a_star_import(): + die() +``` + +`main.py`: + +```py +import tools.exits as exits +from tools.exits import die +from everything import through_a_star_import + +def through_an_alias(): + exits.die() + +def through_the_name(): + die() + +def f(flag: int): + if flag == 1: + x = "alias" + through_an_alias() + elif flag == 2: + x = "name" + through_the_name() + elif flag == 3: + x = "star import" + through_a_star_import() + else: + x = "test" + reveal_type(x) # revealed: Literal["test"] +``` + +A stub only exports what it re-exports, so a function it imports without re-exporting is not one of +its members, and a call that names it through the stub returns: + +`lib/_impl.pyi`: + +```pyi +from typing import NoReturn + +def die() -> NoReturn: ... +def stop() -> NoReturn: ... +``` + +`lib/__init__.pyi`: + +```pyi +from lib._impl import die as die +from lib._impl import stop +``` + +`uses_lib.py`: + +```py +import lib + +def dies(): + lib.die() + +def stops(): + lib.stop() # error: [unresolved-attribute] + +def g(flag: int): + if flag == 1: + x = "re-exported" + dies() + elif flag == 2: + x = "not re-exported" + stops() + else: + x = "test" + reveal_type(x) # revealed: Literal["not re-exported", "test"] +``` + +## generators and coroutines + +Calling a generator function builds a generator without running the body, so the call returns even +when the body never does: + +```py +import sys + +def exits_later(): + sys.exit(1) + yield 1 + +def f(flag: bool): + if flag: + x = "call" + exits_later() + else: + x = "test" + reveal_type(x) # revealed: Literal["call", "test"] +``` + +Calling an `async def` builds a coroutine, which is what returns; awaiting it runs the body: + +```py +async def exits_when_awaited(): + sys.exit(1) + +async def g(flag: bool): + if flag: + x = "created" + exits_when_awaited() # error: [unused-awaitable] + else: + x = "test" + reveal_type(x) # revealed: Literal["created", "test"] + + if flag: + y = "awaited" + await exits_when_awaited() + else: + y = "test" + reveal_type(y) # revealed: Literal["test"] +``` + +An `async def` that awaits one that never returns cannot return either once it is awaited: + +```py +async def awaits_the_exit(): + await exits_when_awaited() + +async def h(flag: bool): + if flag: + x = "awaited" + await awaits_the_exit() + else: + x = "test" + reveal_type(x) # revealed: Literal["test"] +``` + +## without the setting + +Without `infer-unannotated-signatures`, a function that leaves its return type out returns +`Unknown`, and a call to it is treated as returning, whatever its body does: + +```toml +[environment] +python-version = "3.13" + +[analysis] +infer-unannotated-signatures = false +``` + +```py +import sys + +def die(): + sys.exit(1) + +def f(flag: bool): + if flag: + x = "call" + die() + else: + x = "test" + reveal_type(x) # revealed: Literal["call", "test"] +``` + +## `sound-types` implies the setting + +`sound-types` turns `infer-unannotated-signatures` on, so a call to a function that cannot return +ends the flow even where the setting itself is off: + +```toml +[environment] +python-version = "3.13" + +[analysis] +infer-unannotated-signatures = false +sound-types = true +``` + +```py +import sys + +def die(): + sys.exit(1) + +def f(flag: bool): + if flag: + x = "terminal" + die() + else: + x = "test" + reveal_type(x) # revealed: Literal["test"] +``` + +## a module-level call into a module that reads the package back + +A package can call, at module level and behind a condition, a function in a submodule that reads the +package's own globals. Whether the rest of the package is reachable then depends on whether that +function returns, and reading its body must not need those globals to be known first: + +`pkg/__init__.py`: + +```py +import os + +VALUE = os.environ.get("VALUE") + +if "FLAG" in os.environ: + import pkg.sub as sub + + sub.enable() + AFTER = 2 +``` + +`pkg/sub.py`: + +```py +import sys + +import pkg + +def enable(): + check() + +def check(): + if pkg.VALUE: + sys.exit(1) + sys.exit(2) +``` + +`main.py`: + +```py +import pkg + +reveal_type(pkg.VALUE) # revealed: str | None +# error: [unresolved-attribute] +reveal_type(pkg.AFTER) # revealed: Unknown +``` + +## a callee whose annotation depends on the call + +A callee's return annotation can name a global of the module whose reachability the call decides. +Here `NoReturn = int` runs only once `enable()` has returned, and whether `enable()` returns depends +on what `pkg.NoReturn` is. Reading the call as one that does not return is consistent, and it is +what happens at run time: `stop` raises before `NoReturn = int` is reached, so `AFTER` is never +bound: + +`pkg/__init__.py`: + +```py +import os +from typing import NoReturn + +if "FLAG" in os.environ: + import pkg.sub as sub + + sub.enable() + NoReturn = int + AFTER = 1 +``` + +`pkg/sub.py`: + +```py +import pkg + +def enable(): + stop() + +def stop() -> pkg.NoReturn: + raise SystemExit +``` + +`main.py`: + +```py +from pkg import AFTER # error: [unresolved-import] +``` diff --git a/crates/ty_python_semantic/src/reachability.rs b/crates/ty_python_semantic/src/reachability.rs index 9001e6220b..c6a86ebf31 100644 --- a/crates/ty_python_semantic/src/reachability.rs +++ b/crates/ty_python_semantic/src/reachability.rs @@ -195,10 +195,12 @@ use crate::ProgramEnvironment; use std::cell::RefCell; +use std::ops::ControlFlow; use crate::types::context_sensitive::case_name_pattern_type; use crate::types::function::KnownFunction; use crate::types::narrow::pattern_subject_type; +use crate::types::normal_completion::body_recovered_call_returns; use crate::{ Db, dunder_all::dunder_all_names, @@ -785,30 +787,20 @@ fn evaluate_reachability_path<'db>( constraints: &ReachabilityConstraints, predicates: &IndexSlice>, call_predicates: Option<&[ScopedPredicateId]>, - mut id: ScopedReachabilityConstraintId, + id: ScopedReachabilityConstraintId, mut use_checkpoint: bool, ) -> Truthiness { let env = ProgramEnvironment::from_scope(scope); let mut visited = 0; - loop { - if let Some(reachability) = terminal_reachability(id) { - return reachability; + walk_decision_diagram(constraints, id, |id, atom| { + if use_checkpoint && is_reachability_checkpoint(call_predicates, atom, visited) { + return ControlFlow::Break(evaluate_reachability_checkpoint(db, scope, id)); } - - let node = constraints.get_interior_node(id); - if use_checkpoint && is_reachability_checkpoint(call_predicates, node.atom(), visited) { - return evaluate_reachability_checkpoint(db, scope, id); - } - - id = match analyze_single(db, &env, &predicates[node.atom()]) { - Truthiness::AlwaysTrue => node.if_true(), - Truthiness::Ambiguous => node.if_ambiguous(), - Truthiness::AlwaysFalse => node.if_false(), - }; use_checkpoint = true; visited += 1; - } + ControlFlow::Continue(analyze_single(db, &env, &predicates[atom])) + }) } /// Evaluates a canonical suffix of a reachability decision diagram. @@ -1837,6 +1829,23 @@ fn analyze_non_terminal_call<'db>( return Truthiness::AlwaysTrue; } + // basedpython: whether a function whose return type is recovered from its body returns is + // first read off the body's control flow. Recovering the type infers the whole body, and a + // body reached from a module-level call reads the module's globals, whose visibility depends + // on whether this very call returns. Only when the control flow leaves the answer open is the + // recovered type read, below. See `normal_completion`. + let function = match ty { + Type::FunctionLiteral(function) => Some(function), + Type::BoundMethod(method) => Some(method.function(db)), + _ => None, + }; + if let Some(function) = function + && let Some(returns @ (Truthiness::AlwaysTrue | Truthiness::AlwaysFalse)) = + body_recovered_call_returns(db, function, is_await) + { + return returns; + } + let overloads_iterator = if let Some(callable) = ty .try_upcast_to_callable(db, &env) .and_then(CallableTypes::exactly_one) @@ -1969,19 +1978,122 @@ fn analyze_condition<'db>(db: &'db dyn Db, expression: Expression<'db>) -> Truth .unwrap_or(Truthiness::Ambiguous) } +/// basedpython: how the predicates of a reachability decision diagram are decided +/// +/// the checker decides them by inferring the types they test ([`TypeReading`]). whether a call +/// to a function whose return type is recovered from its body can return is read off the same +/// diagram without inferring any expression (`normal_completion`), where a predicate only types +/// decide is ambiguous. both decide each predicate with [`analyze_predicate`] +pub(crate) trait PredicateReading<'db> { + /// the environment types are inferred in, or `None` for a reading that infers no expression + /// + /// a reading without types answers `Ambiguous` wherever only types could decide, and its + /// `Ambiguous` then says that it cannot tell, not that the predicate can go either way + fn types(&self) -> Option<&ProgramEnvironment<'db>>; + + /// whether the statement-level call `call` can return + fn statement_call(&mut self, call: CallableAndCallExpr<'db>) -> Truthiness; + + /// the reachability of `continuation`, a node of `scope`'s own diagram + fn continuation( + &mut self, + scope: ScopeId<'db>, + continuation: ScopedReachabilityConstraintId, + ) -> Truthiness; +} + +/// the checker's reading of a diagram: every predicate is decided by the types it tests +struct TypeReading<'a, 'db> { + db: &'db dyn Db, + env: &'a ProgramEnvironment<'db>, +} + +impl<'db> PredicateReading<'db> for TypeReading<'_, 'db> { + fn types(&self) -> Option<&ProgramEnvironment<'db>> { + Some(self.env) + } + + fn statement_call(&mut self, call: CallableAndCallExpr<'db>) -> Truthiness { + analyze_non_terminal_call(self.db, call.callable, call.call_expr, call.is_await) + } + + fn continuation( + &mut self, + scope: ScopeId<'db>, + continuation: ScopedReachabilityConstraintId, + ) -> Truthiness { + evaluate_finally_continuation(self.db, scope, continuation) + } +} + +/// walks `constraints` from `id` to one of its terminals, deciding each predicate on the way with +/// `reading` +pub(crate) fn evaluate_with_reading<'db>( + db: &'db dyn Db, + constraints: &ReachabilityConstraints, + predicates: &IndexSlice>, + id: ScopedReachabilityConstraintId, + reading: &mut impl PredicateReading<'db>, +) -> Truthiness { + walk_decision_diagram(constraints, id, |_, atom| { + ControlFlow::Continue(analyze_predicate(db, reading, &predicates[atom])) + }) +} + +/// walks `constraints` from `id` to one of its terminals, taking at each node the branch for the +/// truthiness `decide` gives the node's predicate. `decide` can instead break with the +/// reachability of the node it was given, which ends the walk there +fn walk_decision_diagram( + constraints: &ReachabilityConstraints, + mut id: ScopedReachabilityConstraintId, + mut decide: impl FnMut( + ScopedReachabilityConstraintId, + ScopedPredicateId, + ) -> ControlFlow, +) -> Truthiness { + loop { + if let Some(reachability) = terminal_reachability(id) { + return reachability; + } + let node = constraints.get_interior_node(id); + id = match decide(id, node.atom()) { + ControlFlow::Break(reachability) => return reachability, + ControlFlow::Continue(Truthiness::AlwaysTrue) => node.if_true(), + ControlFlow::Continue(Truthiness::Ambiguous) => node.if_ambiguous(), + ControlFlow::Continue(Truthiness::AlwaysFalse) => node.if_false(), + }; + } +} + fn analyze_single(db: &dyn Db, env: &ProgramEnvironment<'_>, predicate: &Predicate) -> Truthiness { - let _span = tracing::trace_span!("analyze_single", ?predicate).entered(); + analyze_predicate(db, &mut TypeReading { db, env }, predicate) +} + +/// the truthiness of `predicate`, as `reading` decides it +fn analyze_predicate<'db>( + db: &'db dyn Db, + reading: &mut impl PredicateReading<'db>, + predicate: &Predicate<'db>, +) -> Truthiness { + let _span = tracing::trace_span!("analyze_predicate", ?predicate).entered(); match predicate.node { - PredicateNode::Expression(test_expr) => { - infer_same_file_expression_type(db, test_expr, TypeContext::default()) - .bool(db, env) - .negate_if(!predicate.is_positive) + PredicateNode::Expression(test_expr) => match reading.types() { + Some(env) => { + infer_same_file_expression_type(db, test_expr, TypeContext::default()).bool(db, env) + } + None => analyze_literal_condition(db, test_expr), } - PredicateNode::Condition(test_expr) => { - analyze_condition(db, test_expr).negate_if(!predicate.is_positive) + .negate_if(!predicate.is_positive), + PredicateNode::Condition(test_expr) => match reading.types() { + Some(_) => analyze_condition(db, test_expr), + None => analyze_literal_condition(db, test_expr), } + .negate_if(!predicate.is_positive), PredicateNode::ChainedComparisonCondition(test_expr) => { + let Some(env) = reading.types() else { + return Truthiness::Ambiguous; + }; let inference = infer_expression_types(db, test_expr, TypeContext::default()); let expression = test_expr.node_ref(db); inference @@ -1992,42 +2104,69 @@ fn analyze_single(db: &dyn Db, env: &ProgramEnvironment<'_>, predicate: &Predica PredicateNode::ContextManagerSuppresses { expression, is_async, - } => Truthiness::from( - infer_same_file_expression_type(db, expression, TypeContext::default()) - .can_suppress_exceptions(db, env, EvaluationMode::from_is_async(is_async)), - ) - .negate_if(!predicate.is_positive), + } => { + let Some(env) = reading.types() else { + return Truthiness::Ambiguous; + }; + Truthiness::from( + infer_same_file_expression_type(db, expression, TypeContext::default()) + .can_suppress_exceptions(db, env, EvaluationMode::from_is_async(is_async)), + ) + .negate_if(!predicate.is_positive) + } PredicateNode::FinallyNormalPathImpossible { scope, continuation, - } => Truthiness::from( - evaluate_finally_continuation(db, scope, continuation).is_always_false(), - ) - .negate_if(!predicate.is_positive), - PredicateNode::IsNonTerminalCall(CallableAndCallExpr { - callable, - call_expr, - is_await, - }) => analyze_non_terminal_call(db, callable, call_expr, is_await) + } => { + let decides_every_predicate = reading.types().is_some(); + match reading.continuation(scope, continuation) { + Truthiness::AlwaysFalse => Truthiness::AlwaysTrue, + Truthiness::AlwaysTrue => Truthiness::AlwaysFalse, + Truthiness::Ambiguous if decides_every_predicate => Truthiness::AlwaysFalse, + // basedpython: a continuation a reading without types cannot decide may still be + // unreachable once types are read + Truthiness::Ambiguous => Truthiness::Ambiguous, + } + .negate_if(!predicate.is_positive) + } + PredicateNode::IsNonTerminalCall(call) => reading + .statement_call(call) .negate_if(!predicate.is_positive), // basedpython: an assertion guard is recorded as an unconditional narrowing // constraint, so its truth value is the fact that the call returned PredicateNode::AssertsCall(_) => Truthiness::AlwaysTrue.negate_if(!predicate.is_positive), - PredicateNode::Pattern(inner) => analyze_pattern_predicate(db, inner), + PredicateNode::Pattern(inner) => { + if reading.types().is_none() { + return Truthiness::Ambiguous; + } + analyze_pattern_predicate(db, inner) + } PredicateNode::OrPatternAlternative(_) => Truthiness::Ambiguous, // basedpython: the capture a bare `case A:` would bind exists only where // the name did not resolve to an enum member of the subject PredicateNode::CaseNameCapture(capture) => { + let Some(env) = reading.types() else { + return Truthiness::Ambiguous; + }; Truthiness::from(case_name_pattern_type(db, env, capture.kind(db)).is_none()) .negate_if(!predicate.is_positive) } PredicateNode::SubjectElementPattern(subject_element) => { + if reading.types().is_none() { + return Truthiness::Ambiguous; + } analyze_pattern_predicate(db, subject_element.pattern) } PredicateNode::IsNonEmptyIterable(iterable) => { + if reading.types().is_none() { + return Truthiness::Ambiguous; + } analyze_non_empty_iterable(db, iterable).negate_if(!predicate.is_positive) } PredicateNode::StarImportPlaceholder(star_import) => { + let Some(env) = reading.types() else { + return Truthiness::Ambiguous; + }; let place_table = place_table(db, star_import.scope(db)); let symbol = place_table.symbol(star_import.symbol_id(db)); let program_file = star_import.referenced_file(db); @@ -2070,6 +2209,27 @@ fn analyze_single(db: &dyn Db, env: &ProgramEnvironment<'_>, predicate: &Predica } } +/// basedpython: the truthiness of a condition read without types, where only a literal whose +/// type states its truthiness counts: `True`, `False`, `None`, an integer, a string or bytes. +/// types would decide those the same way +fn analyze_literal_condition(db: &dyn Db, condition: Expression<'_>) -> Truthiness { + let module = parsed_module(db, condition.python_file(db)).load(db); + analyze_condition_expression(condition.node_ref(db).node(&module), &|leaf| { + Some(match leaf { + ast::Expr::BooleanLiteral(literal) => Truthiness::from(literal.value), + ast::Expr::NoneLiteral(_) => Truthiness::AlwaysFalse, + ast::Expr::NumberLiteral(ast::ExprNumberLiteral { + value: ast::Number::Int(int), + .. + }) => Truthiness::from(*int != 0), + ast::Expr::StringLiteral(literal) => Truthiness::from(!literal.value.is_empty()), + ast::Expr::BytesLiteral(literal) => Truthiness::from(!literal.value.is_empty()), + _ => Truthiness::Ambiguous, + }) + }) + .unwrap_or(Truthiness::Ambiguous) +} + /// Check whether a diagnostic emitted at `range` is in reachable code, considering both /// scope reachability and statement-level reachability within the scope. pub(crate) fn is_range_reachable<'db>( diff --git a/crates/ty_python_semantic/src/types.rs b/crates/ty_python_semantic/src/types.rs index 684584e4f2..e08e009fc1 100644 --- a/crates/ty_python_semantic/src/types.rs +++ b/crates/ty_python_semantic/src/types.rs @@ -223,6 +223,7 @@ pub(crate) mod name_fallback; pub(crate) mod narrow; pub(crate) mod narrowing_guards; mod newtype; +pub(crate) mod normal_completion; mod overlapping; mod overrides; mod protocol_class; diff --git a/crates/ty_python_semantic/src/types/definition_resolution.rs b/crates/ty_python_semantic/src/types/definition_resolution.rs index 48b7fac311..8c5998290c 100644 --- a/crates/ty_python_semantic/src/types/definition_resolution.rs +++ b/crates/ty_python_semantic/src/types/definition_resolution.rs @@ -189,6 +189,29 @@ pub(crate) fn scoped_definitions_for_name<'db>( alias_resolution: ImportAliasResolution, ) -> Vec> { let env = ProgramEnvironment::from_scope(scope); + let all_definitions = visible_definitions_for_name(db, scope, name_str); + + // Resolve import definitions to their targets + let mut resolved_definitions = Vec::new(); + + for definition in &all_definitions { + let resolved = resolve_definition(db, &env, *definition, Some(name_str), alias_resolution); + resolved_definitions.extend(resolved); + } + + resolved_definitions +} + +/// The definitions a use of `name_str` in `scope` can refer to, from the nearest visible scope +/// that has any, without following imports and without falling back to implicit builtins. +/// +/// This reads only the semantic index: every binding and declaration of the name in that scope, +/// whether or not it is reachable from the use. +pub(crate) fn visible_definitions_for_name<'db>( + db: &'db dyn Db, + scope: ScopeId<'db>, + name_str: &str, +) -> FxIndexSet> { let file = scope.program_file(db); let index = semantic_index(db, file); let file_scope = scope.file_scope_id(db); @@ -282,15 +305,7 @@ pub(crate) fn scoped_definitions_for_name<'db>( } } - // Resolve import definitions to their targets - let mut resolved_definitions = Vec::new(); - - for definition in &all_definitions { - let resolved = resolve_definition(db, &env, *definition, Some(name_str), alias_resolution); - resolved_definitions.extend(resolved); - } - - resolved_definitions + all_definitions } /// Resolves a symbol in an implicit builtins scope. diff --git a/crates/ty_python_semantic/src/types/function.rs b/crates/ty_python_semantic/src/types/function.rs index 5ae93afbc9..4391a16605 100644 --- a/crates/ty_python_semantic/src/types/function.rs +++ b/crates/ty_python_semantic/src/types/function.rs @@ -482,8 +482,53 @@ pub(crate) struct CallbackParameterModifiers { pub(crate) kwarg_once: Option, } +/// basedpython: what a function's `def` line says; see [`OverloadLiteral::header`] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, get_size2::GetSize, salsa::SalsaValue)] +pub(crate) struct FunctionHeader { + /// an `async def` + pub(crate) is_async: bool, + /// `-> asserts x` + pub(crate) is_asserts_return: bool, + pub(crate) form: FunctionForm, +} + +/// basedpython: what a `def` defines, where that is more than a function +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, get_size2::GetSize, salsa::SalsaValue)] +pub(crate) enum FunctionForm { + Function, + /// the getter of a property with an accessor block + PropertyGetter, + /// a `decorator def` + DecoratorKeyword, +} + +impl FunctionHeader { + pub(crate) fn of(function: &ast::StmtFunctionDef) -> Self { + let form = if function.property_construct_range().is_some() { + FunctionForm::PropertyGetter + } else if is_decorator_keyword_def(function) { + FunctionForm::DecoratorKeyword + } else { + FunctionForm::Function + }; + Self { + is_async: function.is_async, + is_asserts_return: function.is_asserts_return, + form, + } + } +} + #[salsa::tracked] impl<'db> OverloadLiteral<'db> { + /// basedpython: what this function's `def` line says, read once so that asking it of a + /// function in another module does not tie the asker to that module's syntax tree + #[salsa::tracked(returns(copy), heap_size=ruff_memory_usage::heap_size)] + pub(crate) fn header(self, db: &'db dyn Db) -> FunctionHeader { + let module = parsed_module(db, self.python_file(db)).load(db); + FunctionHeader::of(self.node(db, self.file(db), &module)) + } + /// basedpython: the type an untyped property declares through its initialiser, when this /// function is one of that property's accessors /// @@ -1079,11 +1124,7 @@ impl<'db> OverloadLiteral<'db> { let program_file = self.program_file(db); let module = parsed_module(db, program_file.python_file(db)).load(db); let function_node = scope.node(db).expect_function().node(&module); - let is_decorator_keyword = function_node.decorator_list.iter().any(|decorator| { - matches!(&decorator.expression, ast::Expr::Name(name) - if name.ctx.is_invalid() && name.id == "decorator_keyword") - }); - if !is_decorator_keyword { + if !is_decorator_keyword_def(function_node) { return None; } let index = semantic_index(db, program_file); @@ -1336,15 +1377,12 @@ impl<'db> OverloadLiteral<'db> { .declare_unannotated_parameter(&value.parameter.name.id, property_type); } } - let returns_written = - function_stmt_node.returns.is_some() || (is_property_getter && property_type.is_some()); // basedpython: if this is the implementation of an overloaded function // (i.e. preceded by `@overload` stubs), infer unannotated parameter - // types and an unannotated return type from the union of the sibling - // overloads. this lets `def foo(i)` after `def foo(i: str) -> int` - // and `def foo(i: int) -> str` see `i` as `str | int` and have its - // return statements checked against `int | str` + // types from the union of the sibling overloads. this lets `def foo(i)` + // after `def foo(i: str) -> int` and `def foo(i: int) -> str` see `i` as + // `str | int`. the return type is `return_type_source`'s to decide if !self.is_overload(db) && let Some(previous) = self.previous_overload(db) { @@ -1354,38 +1392,23 @@ impl<'db> OverloadLiteral<'db> { .iter() .map(|ol| ol.raw_signature(db, ReturnCallableTypeVarScope::Public)) .collect(); - raw_signature.inherit_unannotated_from_overloads( - db, - env, - &overload_sigs, - !returns_written, - ); + raw_signature.inherit_unannotated_from_overloads(db, env, &overload_sigs, false); } } - // basedpython: an unannotated method inherits its parameter and return types from the - // method it overrides, so `def m(self, a)` over `def m(self, a: int) -> bytes` sees - // `a: int` and returns `bytes`. this is the first place a missing annotation is recovered - // from, so it shares the gate with the body-derived recovery below — reading the body - // while skipping this would answer with what the placeholder body happens to do rather - // than with what the base already declared + // basedpython: an unannotated method inherits its parameter types from the method it + // overrides, so `def m(self, a)` over `def m(self, a: int)` sees `a: int`. it shares the + // gate with the recovery of a return type, which draws on the same method first; see + // `return_type_source` if infers_unannotated_signatures(db, self.file(db)) - && raw_signature.has_inherited_annotations_to_fill(!returns_written) + && raw_signature.has_inherited_annotations_to_fill(false) && let Some(base_signature) = self.overridden_signature(db, env) { - // a narrowing return type is the exception. it is a claim about what the body tests, - // so an override that tests something else — or nothing — would be handed a claim it - // does not make, and every call through it would narrow on the strength of it - let inherits_return_type = !returns_written - && !matches!( - base_signature.return_ty, - Type::TypeIs(_) | Type::TypeGuard(_) - ); raw_signature.inherit_unannotated_from_overloads( db, env, std::slice::from_ref(&base_signature), - inherits_return_type, + false, ); } @@ -1404,24 +1427,6 @@ impl<'db> OverloadLiteral<'db> { raw_signature.inherit_defaults_from(db, env, base_raw_signature(db, base)); } - // basedpython: an overridden base that could not supply a whole signature can still - // supply its return type - // - // a narrowing return type is the exception: it is a claim about what the body tests, and - // an override that tests something else — or nothing — would be handed a claim it does - // not make, and every call through it would narrow on the strength of it. so that one is - // left to the override's own body - if infers_unannotated_signatures(db, self.file(db)) - && !returns_written - && !function_stmt_node.is_asserts_return - && raw_signature.return_ty.is_unknown() - && let OverriddenReturnType::Declared(base_return) = - self.overridden_return_type(db, env) - && !matches!(base_return, Type::TypeIs(_) | Type::TypeGuard(_)) - { - raw_signature.return_ty = base_return; - } - // basedpython: `@d` on `def f(i)` hands `f` straight to `d`, so a parameter left // unannotated takes its type from the callable `d` declares it accepts — the same context // that types the parameters of `d(lambda i: ...)`. this runs after the sources that say @@ -1445,35 +1450,36 @@ impl<'db> OverloadLiteral<'db> { ); } - // basedpython: a return type nothing else supplied is recovered from the body — the - // union of what it returns, and `None` when it can also fall off the end. this runs - // last so that an annotation, a sibling overload and an overridden base all still win - // - // the three blocks above plus this one are the sources a `def` that leaves its return - // type out draws on; [`OverloadLiteral::return_type_without_annotation`] mirrors them in - // this order, so a change here belongs there too + // basedpython: a return type nobody wrote is drawn from the first source that supplies + // one; see `return_type_source`. the body is the last of them, and recovering from it + // takes the union of what it returns, and `None` when it can also fall off the end let mut reads_body_for_guards = false; - if infers_unannotated_signatures(db, self.file(db)) - && !returns_written - && !function_stmt_node.is_asserts_return - && raw_signature.return_ty.is_unknown() - && self.recovers_return_type_from_body(db, env) - { - raw_signature.return_ty = inferred_return_type(db, self); - - // basedpython: a returned expression says more than its own type does — - // `return a is int` tells every caller what a truthy result means about the argument. - // That reading is recovered beside the return type and only where it is: a base's - // return type stands for its overrides, so a body that answered nothing about the - // return type answers nothing about the narrowing either - let guards = inferred_predicate_guards(db, self); - if !guards.is_empty() { - if let Some(narrowed) = Self::symmetric_guard_type(db, env, guards) { - raw_signature.return_ty = TypeIsType::from_type_expression(db, narrowed); + match self.return_type_source(db, env, false, |base| { + Self::overridden_return_type(db, env, base) + }) { + ReturnTypeSource::Overridden(ty) | ReturnTypeSource::Overloads(ty) => { + raw_signature.return_ty = ty; + } + ReturnTypeSource::Body => { + raw_signature.return_ty = inferred_return_type(db, self); + + // basedpython: a returned expression says more than its own type does — + // `return a is int` tells every caller what a truthy result means about the + // argument. That reading is recovered beside the return type and only where it + // is: a base's return type stands for its overrides, so a body that answered + // nothing about the return type answers nothing about the narrowing either + let guards = inferred_predicate_guards(db, self); + if !guards.is_empty() { + if let Some(narrowed) = Self::symmetric_guard_type(db, env, guards) { + raw_signature.return_ty = TypeIsType::from_type_expression(db, narrowed); + } + raw_signature.narrowing_guards.clone_from(guards); } - raw_signature.narrowing_guards.clone_from(guards); + reads_body_for_guards = true; } - reads_body_for_guards = true; + ReturnTypeSource::Written + | ReturnTypeSource::PropertyInitialiser(_) + | ReturnTypeSource::Gradual => {} } // basedpython: a body that hands back nothing makes assertions instead of predicates, and @@ -1620,11 +1626,6 @@ impl<'db> OverloadLiteral<'db> { /// basedpython: the return type this function would have if its return annotation were /// deleted. /// - /// The sources are the ones [`OverloadLiteral::raw_signature`] consults for a `def` that - /// leaves its return type out, in the same order: a sibling overload group, then an - /// overridden base method under `sound-types`, and — only when neither of those supplied - /// anything — the body. - /// /// The body is the one source not computed here. It reads the function's own scope /// inference, and the `redundant-return-annotation` lint calls this from inside that very /// inference, so the caller supplies it. @@ -1634,22 +1635,78 @@ impl<'db> OverloadLiteral<'db> { env: &ProgramEnvironment<'db>, from_body: impl FnOnce() -> Type<'db>, ) -> Type<'db> { - // the getter of an untyped property with an initialiser — the setter always writes its - // `-> None` — is held to that initialiser's type ahead of everything below - if !self.has_explicit_return_annotation(db) + match self.return_type_source(db, env, true, |base| { + Self::overridden_return_type(db, env, base) + }) { + ReturnTypeSource::PropertyInitialiser(ty) + | ReturnTypeSource::Overridden(ty) + | ReturnTypeSource::Overloads(ty) => ty, + ReturnTypeSource::Body => from_body(), + ReturnTypeSource::Written | ReturnTypeSource::Gradual => Type::unknown(), + } + } + + /// basedpython: where this function's return type comes from, in the order every reader of + /// it consults the sources: [`OverloadLiteral::raw_signature`], which builds the signature; + /// [`OverloadLiteral::return_type_without_annotation`], which the + /// `redundant-return-annotation` lint compares an annotation against; and the reading of + /// whether a call to the function can return (`normal_completion`) + /// + /// `without_annotation` answers for the function as though its return annotation were + /// deleted. `overridden` says what the method this one overrides supplies. the signature and + /// the lint read the base's signature for that; whether a call can return is answered without + /// recovering any return type, so that reading says what it can of the base without one + /// + /// the body is not read when a base answers. that includes a base whose answer cannot be + /// copied, because it is overloaded or in terms of its own type variables: the body must not + /// answer differently from the base, so the return type stays gradual. a narrowing return + /// type is not copied at all. it is a claim about what the base's body tests, so an override + /// that tests something else — or nothing — would be handed a claim it does not make, and + /// every call through it would narrow on the strength of it. that one is left to the + /// override's own body + /// + /// `__new__`'s return type is not read off its body. construction reads it structurally — it + /// decides whether the result is an instance and so whether `__init__` runs at all — and the + /// usual body, `return super().__new__(cls)`, does not currently describe one: + /// `object.__new__(cls)` solves to `Never` + pub(crate) fn return_type_source( + self, + db: &'db dyn Db, + env: &ProgramEnvironment<'db>, + without_annotation: bool, + overridden: impl FnOnce(FunctionType<'db>) -> OverriddenReturnType<'db>, + ) -> ReturnTypeSource<'db> { + let header = self.header(db); + if header.is_asserts_return + || (!without_annotation && self.has_explicit_return_annotation(db)) + { + return ReturnTypeSource::Written; + } + if header.form == FunctionForm::PropertyGetter && let Some(property_type) = self.property_initialiser_type(db) { - return property_type; + return ReturnTypeSource::PropertyInitialiser(property_type); } - let mut return_ty = Type::unknown(); + let infers = infers_unannotated_signatures(db, self.file(db)); + let base = if infers { + self.overridden_method(db, env) + .map_or(OverriddenReturnType::NotOverridden, overridden) + } else { + OverriddenReturnType::NotOverridden + }; + if let OverriddenReturnType::Declared(base_return) = base + && !matches!(base_return, Type::TypeIs(_) | Type::TypeGuard(_)) + { + return ReturnTypeSource::Overridden(base_return); + } if !self.is_overload(db) && let Some(previous) = self.previous_overload(db) { let (overload_list, _) = previous.overloads_and_implementation(db); if !overload_list.is_empty() { - return_ty = UnionType::from_elements( + let returned = UnionType::from_elements( db, env, overload_list.iter().map(|overload| { @@ -1658,67 +1715,35 @@ impl<'db> OverloadLiteral<'db> { .return_ty }), ); + if !returned.is_unknown() { + return ReturnTypeSource::Overloads(returned); + } } } - if db.analysis_settings(self.file(db)).sound_types - && let Some(base_signature) = self.overridden_signature(db, env) - { - return_ty = base_signature.return_ty; + if !infers || self.name(db) == "__new__" || base == OverriddenReturnType::Inexpressible { + return ReturnTypeSource::Gradual; } - - if return_ty.is_unknown() - && infers_unannotated_signatures(db, self.file(db)) - && let OverriddenReturnType::Declared(base_return) = - self.overridden_return_type(db, env) - { - return_ty = base_return; - } - - if infers_unannotated_signatures(db, self.file(db)) - && return_ty.is_unknown() - && self.recovers_return_type_from_body(db, env) - { - return_ty = from_body(); - } - - return_ty + ReturnTypeSource::Body } - /// basedpython: what the method this one overrides says its return type is, for the cases - /// where [`OverloadLiteral::overridden_signature`] could not supply the whole signature. + /// basedpython: what the method `base` says its return type is, as it would stand for the + /// return type of a method that overrides it and writes none /// - /// Inheriting a whole signature has to refuse a base whose *parameters* mention a type - /// variable — the implicit `Self` alone is enough — and a base that is overloaded, since - /// neither can be copied wholesale. Neither of those says anything about the return type, and - /// a base that declares one is still where a missing annotation should draw from: /// `def __len__(self): ...` under `Collection.__len__` returns `int`, not the `None` that - /// running its placeholder body would give. + /// running its placeholder body would give. this reads `base`'s whole signature, so a base + /// that writes no return type either supplies the one it recovers in turn fn overridden_return_type( - self, db: &'db dyn Db, env: &ProgramEnvironment<'db>, + base: FunctionType<'db>, ) -> OverriddenReturnType<'db> { - let Some(base) = self.overridden_method(db, env) else { - return OverriddenReturnType::NotOverridden; - }; if base.signature(db).overloads.len() != 1 { // which type an overloaded base returns depends on the arguments, so there is no one // type to copy return OverriddenReturnType::Inexpressible; } - let return_ty = base_raw_signature(db, base).return_ty; - if return_ty.is_unknown() { - return OverriddenReturnType::NotOverridden; - } - // a type variable in the base's return type is bound to the base method's scope, so - // copying it here would silently rebind it - if any_over_type(db, env, return_ty, false, |ty| { - matches!(ty, Type::TypeVar(_)) - }) { - return OverriddenReturnType::Inexpressible; - } - OverriddenReturnType::Declared(return_ty) + OverriddenReturnType::of(db, env, base_raw_signature(db, base).return_ty) } /// basedpython: how to name the method this one overrides, for a diagnostic about it. @@ -1820,35 +1845,6 @@ impl<'db> OverloadLiteral<'db> { self.node(db, self.file(db), &module).is_asserts_return } - /// basedpython: whether a return type nobody wrote is read off the body. - /// - /// It always is, except for `__new__`. Construction *reads* `__new__`'s return type - /// structurally — it decides whether the result is an instance and so whether `__init__` runs - /// at all — and the usual body, `return super().__new__(cls)`, does not currently describe - /// one: `object.__new__(cls)` solves to `Never`. That is a pre-existing gap, reproducible with - /// `infer-unannotated-signatures` turned off, so recovering a return type here would only - /// hand that gap the power to change what a call means. - /// - /// A base method that already answers the question is the other exception, even when what it - /// answered cannot be written down here — an overloaded base, or one whose return type is in - /// terms of its own type variables. The body must not answer differently from the base, so - /// where the base's answer cannot be copied the return type stays gradual. - /// - /// [`OverloadLiteral::raw_signature`] and [`OverloadLiteral::return_type_without_annotation`] - /// both consult this, so the signature and the `redundant-return-annotation` lint cannot - /// disagree about whether the body was the source. - fn recovers_return_type_from_body( - self, - db: &'db dyn Db, - env: &ProgramEnvironment<'db>, - ) -> bool { - self.name(db) != "__new__" - && !matches!( - self.overridden_return_type(db, env), - OverriddenReturnType::Inexpressible - ) - } - /// basedpython: the signature of the method that this method overrides, resolved by looking /// `self`'s name up in the enclosing class's MRO starting *after* the class itself — the same /// lookup `super()` performs. @@ -2310,6 +2306,15 @@ pub(super) fn same_module_uncached_raw_signature<'db>( .last_definition_raw_signature(db, return_callable_typevar_scope) } +/// basedpython: whether `function` is a `decorator def`, which the parser marks with a decorator +/// no source can spell. +fn is_decorator_keyword_def(function: &ast::StmtFunctionDef) -> bool { + function.decorator_list.iter().any(|decorator| { + matches!(&decorator.expression, ast::Expr::Name(name) + if name.ctx.is_invalid() && name.id == "decorator_keyword") + }) +} + /// basedpython: whether a function in `file` with no annotations is given the signature its body /// determines. /// @@ -2329,9 +2334,10 @@ pub(crate) fn infers_unannotated_signatures(db: &dyn Db, file: File) -> bool { } /// basedpython: what the method an unannotated method overrides says about its return type. -#[derive(Debug, Copy, Clone, PartialEq, Eq)] -enum OverriddenReturnType<'db> { - /// nothing in the MRO after this class defines the name, so the base has nothing to say +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash, get_size2::GetSize, salsa::SalsaValue)] +pub(crate) enum OverriddenReturnType<'db> { + /// the base has nothing to say: nothing in the MRO after this class defines the name, or what + /// does leaves its return type gradual NotOverridden, /// the base declares this, and it can stand as this method's own return type Declared(Type<'db>), @@ -2342,6 +2348,41 @@ enum OverriddenReturnType<'db> { Inexpressible, } +impl<'db> OverriddenReturnType<'db> { + /// what a base whose return type is `return_ty` says about the return type of an override + pub(crate) fn of(db: &'db dyn Db, env: &ProgramEnvironment<'db>, return_ty: Type<'db>) -> Self { + if return_ty.is_unknown() { + return Self::NotOverridden; + } + // a type variable in the base's return type is bound to the base method's scope, so + // copying it here would silently rebind it + if any_over_type(db, env, return_ty, false, |ty| { + matches!(ty, Type::TypeVar(_)) + }) { + return Self::Inexpressible; + } + Self::Declared(return_ty) + } +} + +/// basedpython: where a function's return type comes from; see +/// [`OverloadLiteral::return_type_source`] +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) enum ReturnTypeSource<'db> { + /// the function writes it: an annotation, or an `asserts` clause, which returns `None` + Written, + /// the getter of an untyped property is held to the type its initialiser declares + PropertyInitialiser(Type<'db>), + /// the method this one overrides supplies it + Overridden(Type<'db>), + /// the union of what the sibling `@overload`s return + Overloads(Type<'db>), + /// the function's own body + Body, + /// nothing supplies one, so it stays gradual + Gradual, +} + /// Indicates whether a method is explicitly or implicitly abstract. #[derive(Debug, Copy, Clone, PartialEq, Eq, Hash, get_size2::GetSize)] pub(super) enum AbstractMethodKind { @@ -2513,6 +2554,12 @@ impl<'db> FunctionType<'db> { .map_or(&[], |updated| &updated.specializations) } + /// Whether a decorator replaced this function's signature or its implementation's, as opposed + /// to only specializing them. + pub(super) fn has_replaced_signature(self, db: &'db dyn Db) -> bool { + self.updated_signature(db).is_some() || self.updated_implementation_callables(db).is_some() + } + pub(super) fn updated_signature(self, db: &'db dyn Db) -> Option<&'db CallableSignature<'db>> { self.updated_signatures(db) .as_deref() diff --git a/crates/ty_python_semantic/src/types/infer/builder/function.rs b/crates/ty_python_semantic/src/types/infer/builder/function.rs index 91770ec965..64e9c46a1e 100644 --- a/crates/ty_python_semantic/src/types/infer/builder/function.rs +++ b/crates/ty_python_semantic/src/types/infer/builder/function.rs @@ -859,7 +859,7 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { scope_id.to_scope_id(db, self.program_file()), function, scope_id.is_generator_function(self.index), - can_implicitly_return_none(db, self.index.use_def_map(scope_id)), + self.index.use_def_map(scope_id), |expr| self.expression_type(expr), ) }); diff --git a/crates/ty_python_semantic/src/types/inferred_signature.rs b/crates/ty_python_semantic/src/types/inferred_signature.rs index c62025c49e..8c5f483864 100644 --- a/crates/ty_python_semantic/src/types/inferred_signature.rs +++ b/crates/ty_python_semantic/src/types/inferred_signature.rs @@ -89,7 +89,7 @@ pub(crate) fn inferred_return_type<'db>( body_scope, node, file_scope_id.is_generator_function(index), - can_implicitly_return_none(db, index.use_def_map(file_scope_id)), + index.use_def_map(file_scope_id), |expr| inference.expression_type(expr), ) } @@ -136,19 +136,25 @@ fn divergence_bounded<'db>( /// runs from inside the very scope that would be re-entered. The two share this so they cannot /// drift: the lint advises deleting an annotation, and a body type it disagreed with would make /// that advice silently change the function's type. +/// +/// A `return` that can never be reached hands nothing back, so its value is left out, as the end +/// of the body is when control cannot fall off it. A body whose every `return` and whose end are +/// unreachable returns `Never`: a call to it cannot return, and the checker ends the flow there. pub(crate) fn return_type_from_body<'db>( db: &'db dyn Db, env: &ProgramEnvironment<'db>, body_scope: ScopeId<'db>, node: &ast::StmtFunctionDef, is_generator: bool, - can_implicitly_return_none: bool, + use_def: &UseDefMap<'db>, expression_type: impl Fn(&Expr) -> Type<'db>, ) -> Type<'db> { + let can_implicitly_return_none = can_implicitly_return_none(db, use_def); let mut collector = BodyValueCollector { db, env: env.clone(), body_scope, + use_def, // a `type def` hands back a type rather than a value, so there is no value whose members // a caller could read and nothing for a structural claim about them to describe carries_member_narrowing: !ast::helpers::is_type_def(node), @@ -2396,17 +2402,18 @@ pub(crate) fn can_implicitly_return_none<'db>(db: &'db dyn Db, use_def: &UseDefM /// /// A nested function or lambda has its own body scope and its own returns, so /// this never descends into one. A class body is skipped for the same reason. -struct BodyValueCollector<'db, F> { +struct BodyValueCollector<'a, 'db, F> { db: &'db dyn Db, env: ProgramEnvironment<'db>, body_scope: ScopeId<'db>, + use_def: &'a UseDefMap<'db>, carries_member_narrowing: bool, expression_type: F, returns: Vec>, yields: Vec>, } -impl<'db, F> Visitor<'_> for BodyValueCollector<'db, F> +impl<'db, F> Visitor<'_> for BodyValueCollector<'_, 'db, F> where F: Fn(&Expr) -> Type<'db>, { @@ -2419,6 +2426,15 @@ where Stmt::FunctionDef(_) | Stmt::ClassDef(_) => {} Stmt::Return(ret) => { + let reachability = self.use_def.return_reachability(ret.range()); + if self + .use_def + .reachability_constraints() + .evaluate(self.db, self.use_def.predicates(), reachability) + .is_always_false() + { + return; + } let returned = match ret.value.as_deref() { Some(value) => { self.visit_expr(value); diff --git a/crates/ty_python_semantic/src/types/normal_completion.rs b/crates/ty_python_semantic/src/types/normal_completion.rs new file mode 100644 index 0000000000..b5862bac78 --- /dev/null +++ b/crates/ty_python_semantic/src/types/normal_completion.rs @@ -0,0 +1,912 @@ +//! basedpython: whether a call to a function whose return type nobody wrote can return, read off +//! the function's control flow before its recovered return type +//! +//! a statement-level call ends the flow when the function it calls cannot return: when its return +//! type is `Never`. a function annotated `-> NoReturn` says so in its signature. a function whose +//! return type is recovered from its body (`infer-unannotated-signatures`, which `sound-types` +//! implies) returns `Never` when no `return` in the body and not the end of the body can be +//! reached — but recovering that type means inferring the whole body, and everything the body +//! reads. for a call made at module level, what the body reads includes the module's own globals, +//! whose visibility at the end of the module depends on whether that very call returns. every body +//! involved then joins one salsa cycle, which is iterated as a whole until it settles +//! +//! so the question is first answered here from the body's control flow: the reachability decision +//! diagram the semantic index already builds for the body +//! ([`UseDefMap::normal_exit_reachability`]), decided without inferring any expression: +//! +//! - `raise`, `return`, `break` and `continue` are already part of the diagram's shape +//! - a condition is known only when it is a literal the type of which would be known too +//! (`while True`), and ambiguous otherwise +//! - a statement-level call in the body is decided only when its callee resolves through the +//! semantic index alone to a single function, by that function's written return type, or by its +//! body under this same reading. resolution follows a name to the one definition its scope has +//! for it, an import to the module it names, and `self`, `cls` or a class's own name to a method +//! of a class with no bases. every other call is ambiguous +//! +//! the only types read are written return annotations: a callee's, and those of the methods it +//! overrides, which [`OverloadLiteral::return_type_source`] consults before the body. no module +//! global is looked up by type, and no reachability is decided with types, so none of this depends +//! on the visibility of a module's globals at its end: the cycle has nothing to close through +//! +//! the reading has three answers. a body no `return` and no end of which can be reached cannot +//! complete, and a call to it does not return. a body that always reaches one completes, and a +//! call to it returns. a body whose answer depends on something only types decide leaves the +//! question open, and the checker reads the recovered return type instead, as it reads any other +//! callee's. only then is the body inferred, and a cycle through the module's globals is one the +//! answer really depends on +//! +//! wherever types would be needed to say more, the reading is ambiguous, and ternary logic is +//! monotone in it, so its two definite answers are ones the types agree with. a body it finds +//! cannot complete has no reachable `return` and no reachable end by types either, and the +//! recovered return type leaves out what cannot be reached, so it is `Never`. a body it finds +//! always completes reaches a `return` or its end by types too. the one place the two can still +//! differ is the value a reached `return` hands back, which would need the value's type: +//! `return sys.exit()` is a `return` that is reached, and the call is read as returning although +//! its recovered return type is `Never` +//! +//! recursion between bodies is resolved as a least fixed point: a body's answer starts as "cannot +//! complete" and can only grow towards "completes". that is the reading execution agrees with — a +//! call that returns is a finite one, so `def f(): f()` never returns, it ends in `RecursionError` +//! — and it never calls a body terminal that has a completing path, because such a path is found +//! without assuming anything about the body it is in +//! +//! a cycle can also pass through the checker's own inference, when a callee's written annotation +//! names a global whose reachability depends on the call. there too, every call the least fixed +//! point calls terminal is one whose body cannot complete, or one whose annotation declares +//! `NoReturn` and whose body the checker holds to that. what the fixed point leaves out is only +//! code that runs after such a call returns, so if one of them did return, the first to do so +//! would have to have returned through a path that is blocked by an earlier one + +use ruff_db::files::File; +use ruff_db::parsed::parsed_module; +use ruff_python_ast as ast; +use ruff_python_ast::name::Name; +use std::cell::Cell; + +use rustc_hash::{FxHashMap, FxHashSet}; +use ty_module_resolver::{ + ImportingFile, Module, ModuleName, resolve_module, resolve_module_for_import_from, +}; +use ty_python_core::definition::{Definition, DefinitionKind, ParameterDefinitionNodeKind}; +use ty_python_core::predicate::CallableAndCallExpr; +use ty_python_core::reachability_constraints::ScopedReachabilityConstraintId; +use ty_python_core::scope::{NodeWithScopeRef, ScopeId}; +use ty_python_core::{ + ProgramFile, Truthiness, UseDefMap, global_scope, semantic_index, use_def_map, +}; + +use crate::place::implicit_builtins_symbol_scope; +use crate::reachability::{PredicateReading, evaluate_with_reading}; +use crate::types::Type; +use crate::types::definition_resolution::{find_symbol_in_scope, visible_definitions_for_name}; +use crate::types::function::{ + FunctionForm, FunctionHeader, FunctionType, OverloadLiteral, OverriddenReturnType, + ReturnTypeSource, infers_unannotated_signatures, +}; +use crate::types::signatures::function_signature_expression_type; +use crate::{Db, ProgramEnvironment, attribute_assignments, attribute_declarations}; + +/// whether a statement-level call to `function` can return, when the function's return type is +/// recovered from a body: `AlwaysFalse` when it cannot, `AlwaysTrue` when it does, and `Ambiguous` +/// when only the recovered return type can tell +/// +/// `None` when the return type is not recovered from a body: it is written down, or supplied by an +/// overload group, a replaced signature or a `decorator def`. the signature answers those without +/// reading a body, and the caller reads it +pub(crate) fn body_recovered_call_returns<'db>( + db: &'db dyn Db, + function: FunctionType<'db>, + is_await: bool, +) -> Option { + if function.has_replaced_signature(db) { + return None; + } + let (overloads, implementation) = function.overloads_and_implementation(db); + let implementation = implementation.filter(|_| overloads.is_empty())?; + unannotated_call_returns(db, implementation, is_await) +} + +/// [`body_recovered_call_returns`] for the one definition of a function +/// +/// a cycle through it starts from the answer that the call returns, which is what the checker +/// answers for any call it cannot see through +#[salsa::tracked( + returns(copy), + cycle_initial = |_, _, _, _| Some(Truthiness::AlwaysTrue), + heap_size = ruff_memory_usage::heap_size, +)] +fn unannotated_call_returns<'db>( + db: &'db dyn Db, + overload: OverloadLiteral<'db>, + is_await: bool, +) -> Option { + let header = overload.header(db); + if header.form == FunctionForm::DecoratorKeyword { + return None; + } + // an `async def` called without `await` hands back a coroutine, whatever its body does + if header.is_async && !is_await { + return Some(Truthiness::AlwaysTrue); + } + let env = ProgramEnvironment::from_scope(overload.body_scope(db)); + let base_is_undecided = Cell::new(false); + let source = overload.return_type_source(db, &env, false, |base| { + overridden_return_type(db, base).unwrap_or_else(|| { + base_is_undecided.set(true); + OverriddenReturnType::NotOverridden + }) + }); + // what the base supplies is not known without its recovered return type, so neither is where + // this function's return type comes from + if base_is_undecided.get() { + return Some(Truthiness::Ambiguous); + } + match source { + ReturnTypeSource::Body => Some(body_call_returns(db, overload.body_scope(db))), + ReturnTypeSource::Overridden(return_ty) => Some(Truthiness::from( + !return_ty.is_equivalent_to(db, &env, Type::Never), + )), + ReturnTypeSource::Written + | ReturnTypeSource::PropertyInitialiser(_) + | ReturnTypeSource::Overloads(_) + | ReturnTypeSource::Gradual => None, + } +} + +/// what the method `base` supplies as the return type of a method that overrides it, told without +/// recovering any return type; `None` when that cannot be told +fn overridden_return_type<'db>( + db: &'db dyn Db, + base: FunctionType<'db>, +) -> Option> { + let (overloads, implementation) = base.overloads_and_implementation(db); + // which type an overloaded base returns depends on the arguments, so the override's return + // type stays gradual + match implementation.filter(|_| overloads.is_empty()) { + Some(base) => return_type_seen_by_overrides(db, base), + None => Some(OverriddenReturnType::Inexpressible), + } +} + +/// [`overridden_return_type`] for the one definition of a base method +/// +/// this follows [`OverloadLiteral::overridden_return_type`], which reads the base's signature and +/// so recovers the base's own return type from its body when it writes none. here that body is +/// read the same way as any other body. one that cannot complete returns `Never`. one that +/// reaches a `return` or its end returns a type this reading does not recover, which stands as +/// the gradual type: it is only used to tell whether a call returns, and a call returning the +/// gradual type is taken to. one that only types can read leaves the answer open +/// +/// a cycle through it starts from the open answer, which sends the caller to the types +#[salsa::tracked( + returns(copy), + cycle_initial = |_, _, _| None, + heap_size = ruff_memory_usage::heap_size, +)] +fn return_type_seen_by_overrides<'db>( + db: &'db dyn Db, + base: OverloadLiteral<'db>, +) -> Option> { + let env = ProgramEnvironment::from_scope(base.body_scope(db)); + let base_is_undecided = Cell::new(false); + let source = base.return_type_source(db, &env, false, |base| { + overridden_return_type(db, base).unwrap_or_else(|| { + base_is_undecided.set(true); + OverriddenReturnType::NotOverridden + }) + }); + if base_is_undecided.get() { + return None; + } + Some(match source { + ReturnTypeSource::Written => match written_return_type(db, base.definition(db)) { + Some(return_ty) => OverriddenReturnType::of(db, &env, return_ty), + None => OverriddenReturnType::NotOverridden, + }, + ReturnTypeSource::PropertyInitialiser(return_ty) + | ReturnTypeSource::Overridden(return_ty) + | ReturnTypeSource::Overloads(return_ty) => OverriddenReturnType::of(db, &env, return_ty), + ReturnTypeSource::Body => match body_call_returns(db, base.body_scope(db)) { + Truthiness::AlwaysTrue => OverriddenReturnType::Declared(Type::unknown()), + Truthiness::AlwaysFalse => OverriddenReturnType::Declared(Type::Never), + Truthiness::Ambiguous => return None, + }, + ReturnTypeSource::Gradual => OverriddenReturnType::NotOverridden, + }) +} + +/// the type the return annotation of the function `definition` declares, `None` when it has none +/// +/// an `asserts` clause names a place rather than a type, and such a function returns `None` +#[salsa::tracked( + returns(copy), + cycle_initial = |_, _, _| Some(Type::unknown()), + heap_size = ruff_memory_usage::heap_size, +)] +fn written_return_type<'db>(db: &'db dyn Db, definition: Definition<'db>) -> Option> { + let module = parsed_module(db, definition.python_file(db)).load(db); + let function = definition.kind(db).as_function()?.node(&module); + let returns = function.returns.as_deref()?; + Some(if function.is_asserts_return { + Type::none(db, &ProgramEnvironment::from_scope(definition.scope(db))) + } else { + function_signature_expression_type(db, definition, returns) + }) +} + +/// whether a call to the function whose body is `body_scope` returns, given that the call is not +/// an `async def` left unawaited: a generator function's call builds the generator without running +/// the body, and any other call returns when the body completes +fn body_call_returns<'db>(db: &'db dyn Db, body_scope: ScopeId<'db>) -> Truthiness { + let index = semantic_index(db, body_scope.program_file(db)); + if body_scope.file_scope_id(db).is_generator_function(index) { + return Truthiness::AlwaysTrue; + } + function_body_completion(db, body_scope) +} + +/// whether control leaves the function body `body_scope` normally — reaches a `return` or the end +/// of the body — as read off its control flow, without inferring any expression in it +/// +/// `AlwaysFalse` when it cannot and `AlwaysTrue` when it always can. `Ambiguous` when that depends +/// on something only types can decide, such as a condition that is not a literal +/// +/// recursion is solved as a least fixed point: the provisional answer is `AlwaysFalse`, and an +/// iteration can only move it towards `AlwaysTrue`. see the module documentation for why that is +/// sound +#[salsa::tracked( + returns(copy), + cycle_initial = |_, _, _| Truthiness::AlwaysFalse, + cycle_fn = |_, _, previous: &Truthiness, result: Truthiness, _| previous.or(result), + heap_size = ruff_memory_usage::heap_size, +)] +pub(crate) fn function_body_completion<'db>( + db: &'db dyn Db, + body_scope: ScopeId<'db>, +) -> Truthiness { + let use_def = use_def_map(db, body_scope); + let mut reading = ControlFlowReading { + db, + env: ProgramEnvironment::from_scope(body_scope), + scope: body_scope, + use_def, + continuations: FxHashMap::default(), + }; + evaluate_with_reading( + db, + use_def.reachability_constraints(), + use_def.predicates(), + use_def.normal_exit_reachability(), + &mut reading, + ) +} + +/// decides a scope's reachability predicates from control flow alone +struct ControlFlowReading<'a, 'db> { + db: &'db dyn Db, + env: ProgramEnvironment<'db>, + scope: ScopeId<'db>, + use_def: &'a UseDefMap<'db>, + /// a sequence of `finally` suites nests each one's continuation in the next, so each is + /// evaluated once + continuations: FxHashMap, +} + +impl<'db> PredicateReading<'db> for ControlFlowReading<'_, 'db> { + fn types(&self) -> Option<&ProgramEnvironment<'db>> { + None + } + + fn statement_call(&mut self, call: CallableAndCallExpr<'db>) -> Truthiness { + let db = self.db; + let module = parsed_module(db, self.scope.python_file(db)).load(db); + let callee = call.callable.node_ref(db).node(&module); + let mut resolver = CalleeResolver { + db, + env: &self.env, + visited: FxHashSet::default(), + }; + match resolver.resolve(call.callable.scope(db), callee) { + Some(Resolved::Function(definition)) => { + call_truthiness(db, &self.env, definition, false, call.is_await) + } + Some(Resolved::Method(definition)) => { + call_truthiness(db, &self.env, definition, true, call.is_await) + } + Some(Resolved::Module(_) | Resolved::Class(_) | Resolved::Receiver(_)) | None => { + Truthiness::Ambiguous + } + } + } + + fn continuation( + &mut self, + scope: ScopeId<'db>, + continuation: ScopedReachabilityConstraintId, + ) -> Truthiness { + if scope != self.scope { + return Truthiness::Ambiguous; + } + if let Some(reachability) = self.continuations.get(&continuation) { + return *reachability; + } + let use_def = self.use_def; + let reachability = evaluate_with_reading( + self.db, + use_def.reachability_constraints(), + use_def.predicates(), + continuation, + self, + ); + self.continuations.insert(continuation, reachability); + reachability + } +} + +/// what the `def` line of a function says, and which body it runs +#[derive(Debug, Clone, PartialEq, Eq, Hash, get_size2::GetSize, salsa::SalsaValue)] +struct CalleeFacts<'db> { + header: FunctionHeader, + body_scope: ScopeId<'db>, + /// its return annotation, if it writes one + writes_return_type: bool, + name: Name, + /// it has decorators, and each is the builtin `staticmethod` or `classmethod` + only_method_kind_decorators: bool, + has_decorators: bool, +} + +/// [`CalleeFacts`] of the function `definition`, `None` when it is not a function +#[salsa::tracked(returns(ref), heap_size = ruff_memory_usage::heap_size)] +fn callee_facts<'db>(db: &'db dyn Db, definition: Definition<'db>) -> Option> { + let file = definition.program_file(db); + let module = parsed_module(db, file.python_file(db)).load(db); + let node = definition.kind(db).as_function()?.node(&module); + let env = ProgramEnvironment::from_scope(definition.scope(db)); + let only_method_kind_decorators = node.decorator_list.iter().all(|decorator| { + ["staticmethod", "classmethod"].into_iter().any(|name| { + is_builtin_name(db, &env, definition.scope(db), &decorator.expression, name) + }) + }); + Some(CalleeFacts { + header: FunctionHeader::of(node), + body_scope: semantic_index(db, file) + .node_scope(NodeWithScopeRef::Function(node)) + .to_scope_id(db, file), + writes_return_type: node.returns.is_some(), + name: node.name.id.clone(), + only_method_kind_decorators, + has_decorators: !node.decorator_list.is_empty(), + }) +} + +/// whether a call to the function `definition` returns: `AlwaysFalse` when it cannot, `AlwaysTrue` +/// when it does, and `Ambiguous` when only types can tell +/// +/// `in_class` is whether the function was found in a class body, where `staticmethod` and +/// `classmethod` leave it the same function to call; any other decorator could replace it +fn call_truthiness<'db>( + db: &'db dyn Db, + env: &ProgramEnvironment<'db>, + definition: Definition<'db>, + in_class: bool, + is_await: bool, +) -> Truthiness { + let Some(facts) = callee_facts(db, definition).as_ref() else { + return Truthiness::Ambiguous; + }; + let decorators_keep_the_function = + !facts.has_decorators || (in_class && facts.only_method_kind_decorators); + if !decorators_keep_the_function { + return Truthiness::Ambiguous; + } + // an `async def` called without `await` hands back a coroutine, whatever its body does + if facts.header.is_async && !is_await { + return Truthiness::AlwaysTrue; + } + + if facts.writes_return_type { + // what awaiting something other than a coroutine hands back is a question for its type + if is_await && !facts.header.is_async { + return Truthiness::Ambiguous; + } + let Some(return_ty) = written_return_type(db, definition) else { + return Truthiness::Ambiguous; + }; + // a type variable in the return type is solved by the call's arguments, which can solve + // it to `Never` + return if return_ty.is_equivalent_to(db, env, Type::Never) { + Truthiness::AlwaysFalse + } else if return_ty.has_typevar(db, env) { + Truthiness::Ambiguous + } else { + Truthiness::AlwaysTrue + }; + } + + // without the setting, what an unannotated function returns is the gradual type + if !infers_unannotated_signatures(db, definition.file(db)) { + return Truthiness::AlwaysTrue; + } + // `__new__`'s return type is never read off its body, and what a method of a class with no + // bases returns when it overrides something `object` defines comes from `object` + if facts.name == "__new__" + || facts.header.form == FunctionForm::DecoratorKeyword + || (in_class && defined_on_object(db, env, &facts.name)) + { + return Truthiness::Ambiguous; + } + body_call_returns(db, facts.body_scope) +} + +/// what a callee expression resolves to through the semantic index +#[derive(Debug, Clone, Copy)] +enum Resolved<'db> { + Module(ProgramFile<'db>), + Class(Definition<'db>), + Function(Definition<'db>), + /// a function found in the body of a class + Method(Definition<'db>), + /// the first parameter of a method of the class `Definition`: its instance or, in a + /// `classmethod`, the class itself + Receiver(Definition<'db>), +} + +/// resolves callee expressions through the semantic index alone: no expression's type is inferred +/// +/// a name, or a member of a module or class, resolves only when every definition the scope has for +/// it is one definition. the type of a name the scope binds more than once may be a union, which a +/// call is never terminal through +struct CalleeResolver<'a, 'db> { + db: &'db dyn Db, + env: &'a ProgramEnvironment<'db>, + /// imports already followed, which ends a cycle of re-exports + visited: FxHashSet>, +} + +impl<'db> CalleeResolver<'_, 'db> { + fn resolve(&mut self, scope: ScopeId<'db>, expr: &ast::Expr) -> Option> { + match expr { + ast::Expr::Name(name) => self.resolve_name(scope, name.id.as_str()), + ast::Expr::Attribute(attribute) => match self.resolve(scope, &attribute.value)? { + Resolved::Module(file) => self.module_member(file, attribute.attr.as_str()), + Resolved::Class(class) | Resolved::Receiver(class) => { + self.class_member(class, attribute.attr.as_str()) + } + Resolved::Function(_) | Resolved::Method(_) => None, + }, + _ => None, + } + } + + fn resolve_name(&mut self, scope: ScopeId<'db>, name: &str) -> Option> { + let db = self.db; + let mut definitions = visible_definitions_for_name(db, scope, name); + if definitions.is_empty() { + let builtins = implicit_builtins_symbol_scope(db, self.env, name)?; + definitions = find_symbol_in_scope(db, builtins, name) + .into_iter() + .filter(|definition| definition.is_reexported(db)) + .collect(); + } + let definition = single(definitions)?; + self.resolve_definition(definition, name) + } + + fn resolve_definition( + &mut self, + definition: Definition<'db>, + name: &str, + ) -> Option> { + let db = self.db; + if !self.visited.insert(definition) { + return None; + } + match definition.kind(db) { + DefinitionKind::Function(_) => Some(Resolved::Function(definition)), + DefinitionKind::Class(_) => Some(Resolved::Class(definition)), + DefinitionKind::Parameter(parameter) => self.receiver(definition, parameter), + DefinitionKind::Import(_) + | DefinitionKind::ImportFrom(_) + | DefinitionKind::StarImport(_) => match import_target(db, definition).as_ref()? { + ImportTarget::Module(file) => Some(self.module(*file)), + ImportTarget::Member { + module, + name, + importing_file, + } => self.imported_member(*module, *importing_file, name), + // a star import binds every name it finds, so the one looked up is its own + ImportTarget::Members { + module, + importing_file, + } => self.imported_member(*module, *importing_file, name), + }, + _ => None, + } + } + + fn module(&self, file: File) -> Resolved<'db> { + Resolved::Module(ProgramFile::new(self.db, file, self.env.program(self.db))) + } + + /// `from module import name`: the module's own `name`, or its submodule of that name + fn imported_member( + &mut self, + module: Module<'db>, + importing_file: File, + name: &str, + ) -> Option> { + let db = self.db; + if let Some(file) = module.file(db) { + let file = ProgramFile::new(db, file, self.env.program(db)); + if !self.module_members(file, name).is_empty() { + return self.module_member(file, name); + } + } + let mut submodule = module.name(db).clone(); + submodule.extend(&ModuleName::new(name)?); + let importing = ImportingFile::File(importing_file, self.env.resolver_environment(db)); + Some(self.module(resolve_module(db, importing, &submodule)?.file(db)?)) + } + + /// the definitions of `name` in `file`'s global scope that an import can see: in a stub, only + /// the ones it re-exports + fn module_members(&self, file: ProgramFile<'db>, name: &str) -> Vec> { + let db = self.db; + let is_stub = file.file(db).is_stub(db); + find_symbol_in_scope(db, global_scope(db, file), name) + .into_iter() + .filter(|definition| !is_stub || definition.is_reexported(db)) + .collect() + } + + fn module_member(&mut self, file: ProgramFile<'db>, name: &str) -> Option> { + let definition = single(self.module_members(file, name))?; + self.resolve_definition(definition, name) + } + + /// a function defined in the body of `class`, when nothing else can supply the member: the + /// class has no bases, no decorators and no metaclass, and no method assigns the name on an + /// instance + fn class_member(&mut self, class: Definition<'db>, name: &str) -> Option> { + let db = self.db; + let body = class_body(db, class)?; + if !body.is_closed { + return None; + } + let assigned_on_instance = + attribute_assignments(db, body.scope, name).any(|(mut bindings, _)| { + bindings.any(|binding| binding.binding.definition().is_some()) + }) || attribute_declarations(db, body.scope, name).any(|(mut declarations, _)| { + declarations.any(|declaration| declaration.declaration.definition().is_some()) + }); + if assigned_on_instance { + return None; + } + let definition = single(find_symbol_in_scope(db, body.scope, name))?; + definition + .kind(db) + .as_function() + .map(|_| Resolved::Method(definition)) + } + + /// the first parameter of a method: `self`, or `cls` in a `classmethod` + /// + /// a name resolves within its own file, so the method is in the file being read + fn receiver( + &self, + definition: Definition<'db>, + parameter: &ParameterDefinitionNodeKind, + ) -> Option> { + let db = self.db; + let ParameterDefinitionNodeKind::Parameter(parameter) = parameter else { + return None; + }; + let file = definition.program_file(db); + let module = parsed_module(db, file.python_file(db)).load(db); + let index = semantic_index(db, file); + let function_scope = definition.scope(db).file_scope_id(db); + let function = index + .scope(function_scope) + .node() + .as_function()? + .node(&module); + let first = function + .parameters + .posonlyargs + .iter() + .chain(&function.parameters.args) + .next()?; + if first.parameter.name.as_str() != parameter.node(&module).parameter.name.as_str() { + return None; + } + let class_scope = index.parent_scope_id(function_scope)?; + let class = index.scope(class_scope).node().as_class()?; + // a `staticmethod` has no receiver, and any other decorator could have changed what the + // first parameter is bound to + let class_body = class_scope.to_scope_id(db, file); + let receives = match function.decorator_list.as_slice() { + [] => true, + [decorator] => is_builtin_name( + db, + self.env, + class_body, + &decorator.expression, + "classmethod", + ), + _ => false, + }; + if !receives { + return None; + } + index.try_definition(class).map(Resolved::Receiver) + } +} + +/// what an import definition binds, read once so that following an import into another module +/// does not tie the reader to that module's syntax tree +#[derive(Debug, Clone, PartialEq, Eq, Hash, salsa::SalsaValue)] +enum ImportTarget<'db> { + /// `import a.b` binds `a`, and `import a.b as c` binds `a.b` + Module(File), + /// `from module import name`, in `importing_file` + Member { + module: Module<'db>, + name: Name, + importing_file: File, + }, + /// `from module import *`, in `importing_file` + Members { + module: Module<'db>, + importing_file: File, + }, +} + +#[salsa::tracked(returns(ref))] +fn import_target<'db>(db: &'db dyn Db, definition: Definition<'db>) -> Option> { + let file = definition.program_file(db); + let env = ProgramEnvironment::from_scope(definition.scope(db)); + let importing_file = file.file(db); + let importing = ImportingFile::File(importing_file, env.resolver_environment(db)); + let module = parsed_module(db, file.python_file(db)).load(db); + match definition.kind(db) { + DefinitionKind::Import(import) => { + let alias = import.alias(&module); + let module_name = if alias.asname.is_some() { + ModuleName::new(&alias.name)? + } else { + ModuleName::new(alias.name.split('.').next()?)? + }; + Some(ImportTarget::Module( + resolve_module(db, importing, &module_name)?.file(db)?, + )) + } + DefinitionKind::ImportFrom(import) => Some(ImportTarget::Member { + module: resolve_module_for_import_from(db, importing, import.import(&module))?, + name: import.alias(&module).name.id.clone(), + importing_file, + }), + DefinitionKind::StarImport(import) => Some(ImportTarget::Members { + module: resolve_module_for_import_from(db, importing, import.import(&module))?, + importing_file, + }), + _ => None, + } +} + +/// the body of a class, and whether nothing but it can supply a member: the class has no +/// decorators and no bases but `object`, and passes no keyword such as `metaclass` +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, get_size2::GetSize, salsa::SalsaValue)] +struct ClassBody<'db> { + scope: ScopeId<'db>, + is_closed: bool, +} + +#[salsa::tracked(returns(copy), heap_size = ruff_memory_usage::heap_size)] +fn class_body<'db>(db: &'db dyn Db, class: Definition<'db>) -> Option> { + let file = class.program_file(db); + let module = parsed_module(db, file.python_file(db)).load(db); + let node = class.kind(db).as_class()?.node(&module); + let env = ProgramEnvironment::from_scope(class.scope(db)); + let is_closed = node.decorator_list.is_empty() + && node.arguments.as_deref().is_none_or(|arguments| { + arguments.keywords.is_empty() + && arguments + .args + .iter() + .all(|base| is_builtin_name(db, &env, class.scope(db), base, "object")) + }); + Some(ClassBody { + scope: semantic_index(db, file) + .node_scope(NodeWithScopeRef::Class(node)) + .to_scope_id(db, file), + is_closed, + }) +} + +/// the one element of `definitions`, if it has exactly one +fn single<'db>(definitions: impl IntoIterator>) -> Option> { + let mut definitions = definitions.into_iter(); + let definition = definitions.next()?; + definitions.next().is_none().then_some(definition) +} + +/// whether `expr`, written in `scope`, names the builtin class `name`: a name no enclosing scope +/// defines, which the builtins module defines as a class +fn is_builtin_name<'db>( + db: &'db dyn Db, + env: &ProgramEnvironment<'db>, + scope: ScopeId<'db>, + expr: &ast::Expr, + name: &str, +) -> bool { + let ast::Expr::Name(expr) = expr else { + return false; + }; + expr.id.as_str() == name + && visible_definitions_for_name(db, scope, name).is_empty() + && builtin_class(db, env, name).is_some() +} + +/// the builtins module's definition of the class `name` +fn builtin_class<'db>( + db: &'db dyn Db, + env: &ProgramEnvironment<'db>, + name: &str, +) -> Option> { + let builtins = implicit_builtins_symbol_scope(db, env, name)?; + let definition = single(find_symbol_in_scope(db, builtins, name))?; + definition + .kind(db) + .as_class() + .is_some() + .then_some(definition) +} + +/// whether `object` defines a member `name`, which every class inherits +fn defined_on_object<'db>(db: &'db dyn Db, env: &ProgramEnvironment<'db>, name: &str) -> bool { + let Some(object) = builtin_class(db, env, "object").and_then(|object| class_body(db, object)) + else { + return true; + }; + !find_symbol_in_scope(db, object.scope, name).is_empty() +} + +#[cfg(test)] +mod tests { + use std::fmt::Write as _; + + use ruff_db::files::system_path_to_file; + use ruff_db::system::DbWithWritableSystem as _; + use ruff_db::testing::assert_function_query_was_not_run_by_name; + use ty_python_core::ProgramFile; + + use crate::db::tests::setup_db; + use crate::place::{DefinedPlace, Place, global_symbol}; + + /// A package that, at module level and behind a condition, calls the first of `length` + /// unannotated functions in a submodule, each of which calls the next both as a statement and + /// in its `return`. The last one ends with `last`. Returns whether `AFTER`, bound after the + /// call, is reachable. + /// + /// Recovering the chain's return types infers every body in it, and the last one reads the + /// package back, so all of them joined one cycle with the package's globals: at 500 links + /// checking the package took 45 CPU seconds in a release build, and far longer in a debug one. + fn chain(length: usize, last: &str) -> anyhow::Result<(crate::db::tests::TestDb, bool)> { + let mut db = setup_db(); + db.write_dedented( + "/src/pkg/__init__.py", + r#" + import os + + VALUE = os.environ.get("VALUE") + + if "FLAG" in os.environ: + import pkg.sub as sub + + sub.f0() + AFTER = 1 + "#, + )?; + let mut sub = String::from("import sys\n\nimport pkg\n\n"); + for link in 0..length { + if link + 1 < length { + let next = link + 1; + writeln!( + sub, + "def f{link}():\n f{next}()\n return f{next}()\n\n" + )?; + } else { + writeln!(sub, "def f{link}():\n {last}\n")?; + } + } + db.write_file("/src/pkg/sub.py", sub)?; + + let file = system_path_to_file(&db, "/src/pkg/__init__.py")?; + let file = ProgramFile::new(&db, file, db.program_environment().program(&db)); + // a binding the flow never reaches has the type `Never` + let Place::Defined(DefinedPlace { ty: after, .. }) = + global_symbol(&db, file, "AFTER").place + else { + anyhow::bail!("`AFTER` is always bound in the source"); + }; + let after = !after.is_never(); + Ok((db, after)) + } + + /// runs `test` on a thread with the stack ty's checking threads have + /// + /// each link of a chain is read by a query the reading of the link before it waits on, so how + /// deep the reading goes grows with the chain, and a chain of 500 links needs more than the + /// stack a test thread starts with + fn on_a_checking_stack( + test: impl FnOnce() -> anyhow::Result<()> + Send + 'static, + ) -> anyhow::Result<()> { + std::thread::Builder::new() + .stack_size(ruff_db::STACK_SIZE) + .spawn(test)? + .join() + .unwrap_or_else(|panic| std::panic::resume_unwind(panic)) + } + + #[test] + fn a_long_chain_is_read_without_recovering_its_return_types() -> anyhow::Result<()> { + on_a_checking_stack(|| { + let (mut db, after) = chain(500, "return pkg.VALUE")?; + assert!(after, "the chain returns, so `AFTER` is reachable"); + let events = db.take_salsa_events(); + assert_function_query_was_not_run_by_name(&db, "inferred_return_type", None, &events); + assert_reads_each_link_once(&db, &events, 500); + Ok(()) + }) + } + + #[test] + fn a_long_chain_that_exits_never_returns() -> anyhow::Result<()> { + on_a_checking_stack(|| { + let (mut db, after) = chain(500, "sys.exit(1)")?; + assert!(!after, "the chain never returns, so `AFTER` is unreachable"); + let events = db.take_salsa_events(); + assert_function_query_was_not_run_by_name(&db, "inferred_return_type", None, &events); + assert_reads_each_link_once(&db, &events, 500); + Ok(()) + }) + } + + /// Each link's body is read once, and neither the reading nor the module-level call it + /// answers is part of a cycle that has to be iterated: the cost is linear in the chain. + fn assert_reads_each_link_once( + db: &crate::db::tests::TestDb, + events: &[salsa::Event], + length: usize, + ) { + salsa::Database::attach(db, |_| { + let executions = |query: &str| { + events + .iter() + .filter(|event| match event.kind { + salsa::EventKind::WillExecute { database_key } => { + format!("{database_key:?}").starts_with(query) + } + _ => false, + }) + .count() + }; + assert_eq!(executions("function_body_completion("), length); + let iterated = events + .iter() + .filter_map(|event| match event.kind { + salsa::EventKind::WillIterateCycle { database_key, .. } => { + Some(format!("{database_key:?}")) + } + _ => None, + }) + .filter(|head| { + head.starts_with("function_body_completion(") + || head.starts_with("analyze_non_terminal_call(") + }) + .collect::>(); + assert_eq!(iterated, Vec::::new()); + }); + } +} diff --git a/docs/basedpython/features/sound-types.md b/docs/basedpython/features/sound-types.md index 47e275e16b..f020ea825d 100644 --- a/docs/basedpython/features/sound-types.md +++ b/docs/basedpython/features/sound-types.md @@ -344,7 +344,7 @@ contradiction at every call site and never where it lives ## unannotated return types a function with no return annotation returns what its body returns: the union of every `return` -expression, plus `None` when control can also fall off the end +expression that can be reached, plus `None` when control can also fall off the end ```python def f(): @@ -361,6 +361,25 @@ class X(Protocol): def f(self): ... # (self) -> None ``` +a statement that calls a function which returns `Never` ends the flow, as a call to a `NoReturn` +function does, and the code after it is unreachable. a function returns `Never` when no path through +its body reaches a `return` or its end without passing a `raise`, a loop that never exits, or a +call that cannot return either. a function that only ever calls itself never returns, since each +call makes another until `RecursionError` + +```python +import sys + +def die(): + sys.exit(1) + +def f() -> int: + die() # no `invalid-return-type`: the end of `f` is unreachable +``` + +a `return` whose value never finishes evaluating, such as `return sys.exit(1)`, is the exception: the +function returns `Never`, but a call to it is still treated as one that can return + a generator returns a generator: its `yield` expressions supply the yield type and its `return` statements the third type argument. the send type is the one thing the body does not determine, so it stays gradual From e8c52068b5809920986a220a8a8bbd53bf274a77 Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Sat, 26 Sep 2026 04:21:24 +1000 Subject: [PATCH 11/11] refuse textDocument/rename for a symbol prepareRename refuses, instead of renaming every use of a builtin across the project --- .../src/server/api/requests/rename.rs | 27 +++- crates/ty_server/tests/e2e/rename.rs | 138 +++++++++++++++++- 2 files changed, 159 insertions(+), 6 deletions(-) diff --git a/crates/ty_server/src/server/api/requests/rename.rs b/crates/ty_server/src/server/api/requests/rename.rs index c90e6ddcae..6e2760f53f 100644 --- a/crates/ty_server/src/server/api/requests/rename.rs +++ b/crates/ty_server/src/server/api/requests/rename.rs @@ -5,7 +5,10 @@ use anyhow::anyhow; use lsp_server::ErrorCode; use lsp_types::RenameRequest; use lsp_types::{RenameParams, TextEdit, Uri, WorkspaceEdit}; -use ty_ide::{TemplateRename, TemplateRenameOutcome, django_rename, rename}; +use ty_ide::{ + PreparedRename, TemplateRename, TemplateRenameOutcome, django_rename, invalid_new_name, + prepare_rename, rename, +}; use ty_project::{ProjectDatabase, SemanticDb as _}; use crate::document::{FileRangeExt, LspRange, PositionExt, ToLink}; @@ -55,9 +58,18 @@ impl BackgroundDocumentRequestHandler for RenameRequestHandler { let template = snapshot.is_django_template(); - // a python symbol is renamed exactly as it was; the django names a module - // writes as plain strings are what is left over once it declines - if !template + // a rename is checked as `textDocument/prepareRename` checks it, since a + // client need not ask that first. the django names a module writes as + // plain strings are what is left over once the python symbol is not one + let python = (!template).then(|| prepare_rename(db, db.program_file(file), offset)); + // only a python symbol's new name must be an identifier: a django rename + // may be naming a template file + if let Some(PreparedRename::Ready { .. }) = python + && let Some(why) = invalid_new_name(¶ms.new_name) + { + return Err(Error::new(anyhow!(why), ErrorCode::RequestFailed)); + } + if let Some(PreparedRename::Ready { .. }) = python && let Some(rename_results) = rename(db, db.program_file(file), offset, ¶ms.new_name) { @@ -87,7 +99,12 @@ impl BackgroundDocumentRequestHandler for RenameRequestHandler { } match django_rename(db, file, offset, ¶ms.new_name, template) { - None => Ok(None), + None => match python { + Some(PreparedRename::Refused(why)) => { + Err(Error::new(anyhow!(why), ErrorCode::RequestFailed)) + } + Some(PreparedRename::Ready { .. } | PreparedRename::NoSymbol) | None => Ok(None), + }, Some(TemplateRenameOutcome::Refused(why)) => { Err(Error::new(anyhow!(why), ErrorCode::RequestFailed)) } diff --git a/crates/ty_server/tests/e2e/rename.rs b/crates/ty_server/tests/e2e/rename.rs index 8d7f1b209f..9ea093b514 100644 --- a/crates/ty_server/tests/e2e/rename.rs +++ b/crates/ty_server/tests/e2e/rename.rs @@ -3,7 +3,7 @@ use std::time::Duration; use lsp_types::ShowMessageNotification; use crate::notebook::NotebookBuilder; -use crate::{AwaitResponseError, TestServerBuilder}; +use crate::{AwaitResponseError, TestServer, TestServerBuilder}; use insta::assert_json_snapshot; #[test] @@ -117,6 +117,32 @@ fn a_builtin_is_refused_with_the_reason() -> anyhow::Result<()> { Ok(()) } +/// a client that renames without asking first is refused just the same, rather +/// than handed an edit renaming every use of `print` in the project +#[test] +fn a_rename_of_a_builtin_is_refused_without_asking_first() -> anyhow::Result<()> { + let mut server = TestServerBuilder::new()? + .with_file("foo.py", "")? + .build() + .wait_until_workspaces_are_initialized(); + + server.open_text_document("foo.py", "print(1)\n", 1); + + let response = rename_at( + &mut server, + "foo.py", + lsp_types::Position::new(0, 2), + "show", + ); + + assert_eq!( + refusal(response), + "`print` is declared outside this project" + ); + + Ok(()) +} + /// where there is no name at all there is nothing to explain, and the answer /// stays `null` #[test] @@ -182,3 +208,113 @@ fn prepare_rename_params( work_done_progress_params: lsp_types::WorkDoneProgressParams::default(), } } + +/// the quotes are part of what is replaced, so the argument is left written +/// the plain way, matching the parameter +#[test] +fn a_quoted_keyword_argument_is_renamed_to_a_bare_name() -> anyhow::Result<()> { + let mut server = TestServerBuilder::new()? + .with_file("foo.by", QUOTED_KEYWORD)? + .build() + .wait_until_workspaces_are_initialized(); + + server.open_text_document("foo.by", QUOTED_KEYWORD, 1); + + let edit = rename_at( + &mut server, + "foo.by", + lsp_types::Position::new(2, 4), + "retries", + ) + .expect("the rename is carried out") + .expect("an edit"); + + let edits = edit + .changes + .expect("edits by file") + .remove(&server.file_uri("foo.by")) + .expect("edits to foo.by"); + + assert_eq!( + apply(QUOTED_KEYWORD, edits), + "def f(retries: int): ...\n\nf(retries=1)\n" + ); + + Ok(()) +} + +/// a new name that is not a name, or is a keyword, would leave the code unable +/// to parse, so the rename is refused rather than written +#[test] +fn a_new_name_that_is_not_a_name_is_refused() -> anyhow::Result<()> { + let mut server = TestServerBuilder::new()? + .with_file("foo.by", QUOTED_KEYWORD)? + .build() + .wait_until_workspaces_are_initialized(); + + server.open_text_document("foo.by", QUOTED_KEYWORD, 1); + + let quoted = rename_at( + &mut server, + "foo.by", + lsp_types::Position::new(2, 4), + "\"retries\"", + ); + assert_eq!(refusal(quoted), "`\"retries\"` is not a valid name"); + + let keyword = rename_at( + &mut server, + "foo.by", + lsp_types::Position::new(2, 4), + "class", + ); + assert_eq!(refusal(keyword), "`class` is a keyword"); + + Ok(()) +} + +fn rename_at( + server: &mut TestServer, + path: &str, + position: lsp_types::Position, + new_name: &str, +) -> Result, AwaitResponseError> { + let id = server.send_request::(lsp_types::RenameParams { + text_document_position_params: lsp_types::TextDocumentPositionParams { + text_document: lsp_types::TextDocumentIdentifier { + uri: server.file_uri(path), + }, + position, + }, + new_name: new_name.to_string(), + work_done_progress_params: lsp_types::WorkDoneProgressParams::default(), + }); + server.try_await_response::(&id, None) +} + +/// the reason a request was refused, asserting that it was +#[track_caller] +fn refusal(response: Result) -> String { + let Err(AwaitResponseError::RequestFailed(failure)) = response else { + panic!("expected a refusal, got {response:?}"); + }; + assert_eq!(failure.code, lsp_server::ErrorCode::RequestFailed as i32); + failure.message +} + +/// `source` with `edits` applied, for edits that each stay on one line of +/// ascii text +fn apply(source: &str, mut edits: Vec) -> String { + let mut lines: Vec = source.split_inclusive('\n').map(str::to_string).collect(); + edits + .sort_by_key(|edit| std::cmp::Reverse((edit.range.start.line, edit.range.start.character))); + for edit in edits { + assert_eq!(edit.range.start.line, edit.range.end.line); + let line = &mut lines[edit.range.start.line as usize]; + line.replace_range( + edit.range.start.character as usize..edit.range.end.character as usize, + &edit.new_text, + ); + } + lines.concat() +}