diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 939717d2d9..cde89207ec 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -87,7 +87,7 @@ repos: types: [rust] language: system pass_filenames: false - stages: [pre-push] + stages: [manual] # Prettier - repo: https://github.com/rbubley/mirrors-prettier diff --git a/crates/by_stage/src/project.rs b/crates/by_stage/src/project.rs index 326bd1e454..50cc973ec4 100644 --- a/crates/by_stage/src/project.rs +++ b/crates/by_stage/src/project.rs @@ -5,8 +5,10 @@ //! project reports errors for files the project deliberately excludes — and, when //! this was first written, wrote nothing for them either. So the file set is the //! project's own (`Project::files`), narrowed only by things a *build* has to -//! exclude and a check does not: a hidden directory holding copies, and the last -//! build's own output. +//! exclude and a check does not: a hidden directory holding copies, and the +//! directory this build writes to. Every tree a build already wrote carries a +//! manifest, and the project leaves those out itself — see +//! [`ty_project::build_output`]. //! //! It lives here rather than in the `by` binary because the language server needs //! the same answers. A single file re-staged into a build tree has to land where @@ -14,13 +16,11 @@ //! build would have transpiled it, and the only way to guarantee that is for there //! to be one implementation of each. -use std::collections::HashMap; use std::ffi::OsStr; use std::path::{Path, PathBuf}; use anyhow::Context; -use ruff_db::Db as _; -use ruff_db::system::{OsSystem, System, SystemPath, SystemPathBuf}; +use ruff_db::system::{OsSystem, SystemPath, SystemPathBuf}; use ty_project::{Db, ProjectDatabase, ProjectMetadata}; /// Everything needed to build a project db a second time. @@ -310,8 +310,7 @@ pub(crate) fn project_sources( root: &Path, output: Option<&Path>, ) -> Vec<(PathBuf, ruff_db::files::File)> { - let candidates: Vec<(PathBuf, ruff_db::files::File)> = db - .project() + db.project() .files(db) .into_iter() .filter(|file| { @@ -327,79 +326,16 @@ pub(crate) fn project_sources( // dependencies, not this project's sources — emitting them would write // a parallel tree nobody asked for .filter(|(path, _)| !is_hidden_within(path, root)) - // nor is the last build's output. it holds a copy of every `.by` source - // this build is about to read, and reading those instead would build the - // project into itself, one directory deeper each time + // nor is the directory this build writes to. it holds a copy of every + // `.by` source this build is about to read, and reading those instead + // would build the project into itself, one directory deeper each time. + // a tree a build finished carries a manifest and is never among the + // project's files to begin with, whatever `--out` called it; this is for + // the one this run was told to write to, which may not have one yet .filter(|(path, _)| output.is_none_or(|output| !path.starts_with(output))) - .collect(); - // and nor is any *other* build's output. `--out` can name any directory, and - // a project can have several, so the one this run was given is not the only - // one that has to be turned away - let mut known = HashMap::new(); - candidates - .into_iter() - .filter(|(path, _)| !inside_build_output(db.system(), path, root, &mut known)) .collect() } -/// Whether `path` sits inside a tree that some build already wrote. -/// -/// A build output is self-describing: it holds a `.by-manifest` naming everything -/// that build put there. That marker is what identifies it, rather than the -/// directory's name. [`NON_SOURCE_DIRS`] already turns away the two default names, -/// but `--out` can say anything, and only the tree this run was *given* is known -/// to be an output from the arguments — a project that builds into two of them has -/// to have the other recognised on sight. -/// -/// It has to be turned away because the tree holds a *copy* of the project: every -/// hand-written `.py`, and every `.by` too when `build.sources` is on. Reading one -/// back means each of those modules is claimed twice — once where it was written -/// and once where it was copied — and the build stops, telling the author that two -/// files they never put side by side would compile to the same module. -/// -/// The project root is never itself an output, whatever it holds: a `.by-manifest` -/// left at the root would otherwise empty the project of every file it has. -fn inside_build_output( - system: &dyn System, - path: &Path, - root: &Path, - known: &mut HashMap, -) -> bool { - let Some(parent) = path.parent() else { - return false; - }; - let mut current = parent; - while current != root && current.starts_with(root) { - let is_output = match known.get(current) { - Some(answer) => *answer, - None => { - let answer = holds_manifest(system, current); - known.insert(current.to_path_buf(), answer); - answer - } - }; - if is_output { - return true; - } - match current.parent() { - Some(next) => current = next, - None => return false, - } - } - false -} - -/// Whether `directory` carries the manifest that marks it a build output. -/// -/// Asked of the db's [`System`] rather than of `std::fs`, so that it is the same -/// file system every other question in a build is asked — the language server's -/// re-stage runs this on a warm db, and an in-memory test system has to be able -/// to answer it. -pub(crate) fn holds_manifest(system: &dyn System, directory: &Path) -> bool { - SystemPath::from_std_path(directory) - .is_some_and(|path| system.is_file(&path.join(crate::staging::MANIFEST_FILENAME))) -} - /// The project's first-party module roots, longest first, as absolute paths. /// /// These are the directories a module name is resolved against — for a @@ -420,60 +356,9 @@ pub fn module_roots(db: &ProjectDatabase, cwd: &Path) -> Vec { #[cfg(test)] mod tests { - use super::{inside_build_output, is_hidden_within}; - use ruff_db::system::{OsSystem, SystemPath}; - use std::collections::HashMap; + use super::is_hidden_within; use std::path::Path; - /// `--out` can name any directory and a project can hold several, so a build - /// output is recognised by the manifest it carries rather than by its name. - #[test] - fn a_directory_holding_a_manifest_is_a_build_output() { - let directory = tempfile::tempdir().expect("tempdir"); - let root = directory.path(); - let output = root.join("anything"); - std::fs::create_dir_all(output.join("pkg")).expect("create"); - std::fs::write(output.join(".by-manifest"), "").expect("write"); - let plain = root.join("src"); - std::fs::create_dir_all(&plain).expect("create"); - - let system = - OsSystem::new(SystemPath::from_std_path(root).expect("the temp directory is utf-8")); - let mut known = HashMap::new(); - assert!(inside_build_output( - &system, - &output.join("a.by"), - root, - &mut known - )); - assert!( - inside_build_output(&system, &output.join("pkg").join("a.by"), root, &mut known), - "a file deeper inside the output is inside it too" - ); - assert!( - !inside_build_output(&system, &plain.join("a.by"), root, &mut known), - "a directory with no manifest is ordinary source" - ); - } - - /// A manifest at the root would otherwise empty the project of every file. - #[test] - fn the_project_root_is_never_itself_an_output() { - let directory = tempfile::tempdir().expect("tempdir"); - let root = directory.path(); - std::fs::write(root.join(".by-manifest"), "").expect("write"); - - let system = - OsSystem::new(SystemPath::from_std_path(root).expect("the temp directory is utf-8")); - let mut known = HashMap::new(); - assert!(!inside_build_output( - &system, - &root.join("a.by"), - root, - &mut known - )); - } - #[test] fn a_hidden_directory_is_not_project_source() { let root = Path::new("/p"); diff --git a/crates/by_stage/src/staging.rs b/crates/by_stage/src/staging.rs index ea08eee095..7413b94a9d 100644 --- a/crates/by_stage/src/staging.rs +++ b/crates/by_stage/src/staging.rs @@ -8,6 +8,7 @@ use std::collections::{BTreeMap, BTreeSet}; use std::fs; +use std::io::Write as _; use std::path::{Path, PathBuf}; use anyhow::Context; @@ -20,7 +21,13 @@ use anyhow::Context; /// until one shadows a module that moved. The manifest is what makes the output a /// mirror rather than a pile: what the previous build wrote and this one did not /// is deleted. -pub(crate) const MANIFEST_FILENAME: &str = ".by-manifest"; +/// +/// Being there is also what marks the tree as a build output: the project the build +/// was made from leaves it out of its own files — see [`ty_project::build_output`]. +const MANIFEST_FILENAME: &str = ty_project::build_output::BUILD_MANIFEST; + +/// the line a manifest starts with, before the paths it records +const MANIFEST_HEADER: &str = "# written by `by build`; delete it and stale output stays\n"; /// An output tree being written. pub struct Staging { @@ -30,6 +37,8 @@ pub struct Staging { /// destinations another writer produced, held apart from `written` so a /// source that would land on one is reported rather than overwriting it recorded: BTreeSet, + /// whether the tree is known to carry a manifest already — see [`Self::mark_as_output`] + marked: bool, } impl Staging { @@ -38,9 +47,46 @@ impl Staging { out: out.to_path_buf(), written: BTreeMap::new(), recorded: BTreeSet::new(), + marked: false, } } + /// mark the tree as a build output before the first file lands in it + /// + /// the manifest [`Self::finish`] writes is also what tells the project this tree is not + /// part of it. a first build writes into a directory that has none yet, and a language + /// server watching the project would otherwise see a copy of every source appear, take them + /// for the project's own, and check each one until the build finished and the manifest + /// arrived. an empty manifest says the same as a missing one to the next build (nothing + /// was written here before), and one already there is left for `finish` to read + /// + /// writing and copying through the staging mark it themselves, and so does [`Self::record`]. + /// a writer that lays files out itself, as `by compile` does, calls this before it writes + pub fn mark_as_output(&mut self) -> anyhow::Result<()> { + if self.marked { + return Ok(()); + } + let manifest = self.out.join(MANIFEST_FILENAME); + create_parent(&manifest)?; + match fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(&manifest) + { + Ok(mut file) => { + file.write_all(MANIFEST_HEADER.as_bytes()) + .with_context(|| format!("could not write {}", manifest.display()))?; + } + Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => {} + Err(error) => { + return Err(error) + .with_context(|| format!("could not write {}", manifest.display())); + } + } + self.marked = true; + Ok(()) + } + pub(crate) fn out(&self) -> &Path { &self.out } @@ -76,6 +122,7 @@ impl Staging { contents: &str, ) -> anyhow::Result<()> { self.claim(relative, source)?; + self.mark_as_output()?; let destination = self.out.join(relative); create_parent(&destination)?; fs::write(&destination, contents) @@ -104,6 +151,7 @@ impl Staging { claimant(source.as_deref()), ); } + self.mark_as_output()?; self.recorded.insert(relative.to_path_buf()); Ok(()) } @@ -111,6 +159,7 @@ impl Staging { /// Copy `source` to `relative` verbatim. pub(crate) fn copy(&mut self, relative: &Path, source: &Path) -> anyhow::Result<()> { self.claim(relative, Some(source))?; + self.mark_as_output()?; let destination = self.out.join(relative); create_parent(&destination)?; fs::copy(source, &destination).with_context(|| { @@ -190,8 +239,7 @@ impl Staging { } create_parent(&manifest)?; - let mut rendered = - String::from("# written by `by build`; delete it and stale output stays\n"); + let mut rendered = String::from(MANIFEST_HEADER); for path in current { rendered.push_str(&portable(path)); rendered.push('\n'); @@ -426,6 +474,69 @@ mod tests { assert!(!directory.path().join("mine.py").exists()); } + /// the manifest is what keeps the tree out of the project it was built from, so it is + /// there before the first file is, not only once the build is done + #[test] + fn a_first_build_marks_its_output_before_writing_into_it() { + let directory = tempfile::tempdir().expect("tempdir"); + let out = directory.path().join("build"); + let mut staging = Staging::new(&out); + staging + .write(Path::new("a.py"), None, "x = 1\n") + .expect("write"); + assert!( + ty_project::build_output::is_build_output( + &ruff_db::system::OsSystem::new( + ruff_db::system::SystemPath::from_std_path(directory.path()).expect("utf-8") + ), + ruff_db::system::SystemPath::from_std_path(&out).expect("utf-8"), + ), + "a tree being written is an output before the build finishes" + ); + staging.finish().expect("finish"); + assert!(read_manifest(&out.join(MANIFEST_FILENAME)).contains(Path::new("a.py"))); + } + + /// a file another writer put into the tree marks it too + #[test] + fn recording_an_artifact_marks_the_output() { + let directory = tempfile::tempdir().expect("tempdir"); + let out = directory.path().join("build"); + let mut staging = Staging::new(&out); + staging.record(Path::new("a.so")).expect("record"); + assert!( + ty_project::build_output::is_build_output( + &ruff_db::system::OsSystem::new( + ruff_db::system::SystemPath::from_std_path(directory.path()).expect("utf-8") + ), + ruff_db::system::SystemPath::from_std_path(&out).expect("utf-8"), + ), + "a tree with an artifact recorded in it is an output before the build finishes" + ); + } + + /// marking a tree the last build left does not forget what that build wrote + #[test] + fn marking_an_output_keeps_the_previous_manifest() { + let directory = tempfile::tempdir().expect("tempdir"); + let mut first = Staging::new(directory.path()); + first + .write(Path::new("gone.py"), None, "x = 1\n") + .expect("write"); + first.finish().expect("finish"); + + let mut second = Staging::new(directory.path()); + second + .write(Path::new("kept.py"), None, "x = 1\n") + .expect("write"); + assert!( + read_manifest(&directory.path().join(MANIFEST_FILENAME)).contains(Path::new("gone.py")), + "the previous build's record survives until this one replaces it" + ); + second.finish().expect("finish"); + assert!(!directory.path().join("gone.py").exists()); + } + #[test] fn a_manifest_round_trips_through_its_portable_form() { let directory = tempfile::tempdir().expect("tempdir"); diff --git a/crates/by_stage/src/verbatim.rs b/crates/by_stage/src/verbatim.rs index cab6374dcd..7285ca3256 100644 --- a/crates/by_stage/src/verbatim.rs +++ b/crates/by_stage/src/verbatim.rs @@ -18,10 +18,11 @@ use std::path::{Path, PathBuf}; use ruff_db::Db as _; use ruff_db::system::SystemPath; +use ty_project::build_output::is_build_output; use ty_project::{Db, ProjectDatabase}; use walkdir::WalkDir; -use crate::project::{holds_manifest, may_hold_build_content}; +use crate::project::may_hold_build_content; use crate::staging::{Staging, relative_destination}; /// One file carried into the output unchanged. @@ -62,11 +63,14 @@ fn verbatim_files( // copy of every file this walk carries, so one that was not turned // away would be copied into the next wholesale, a tree deep in a // tree. it is recognised by the manifest it carries rather than by - // its name, because `--out` can say anything. the root is exempt: - // a manifest there would otherwise carry nothing over at all + // its name, because `--out` can say anything, and asked of the db's + // file system, which the language server's re-stage and an in-memory + // test system answer too. the root is exempt: a manifest there would + // otherwise carry nothing over at all if entry.file_type().is_dir() && entry.path() != root - && holds_manifest(db.system(), entry.path()) + && SystemPath::from_std_path(entry.path()) + .is_some_and(|path| is_build_output(db.system(), path)) { return false; } diff --git a/crates/ruff_db/src/files.rs b/crates/ruff_db/src/files.rs index 1f8d3ec542..c643eb0e06 100644 --- a/crates/ruff_db/src/files.rs +++ b/crates/ruff_db/src/files.rs @@ -4,7 +4,9 @@ use std::sync::Arc; use std::sync::atomic::{AtomicBool, Ordering}; use dashmap::mapref::entry::Entry; -pub use directory::{DirectoryListing, DirectoryListingError, directory_listing}; +pub use directory::{ + DirectoryListing, DirectoryListingError, directory_listing, system_path_to_directory, +}; pub use file_root::{FileRoot, FileRootKind}; pub use path::FilePath; use ruff_notebook::{Notebook, NotebookError}; @@ -270,7 +272,9 @@ impl Files { /// Refreshing the state of files recursively is expensive. It requires iterating over all known files /// and making system calls to get the latest status of matching files. /// That's why [`File::sync_path`] is preferred if it is known that the path is a file. - pub fn sync_all_recursive(db: &mut dyn Db, paths: I) + /// + /// Returns whether it changed anything in the database. + pub fn sync_all_recursive(db: &mut dyn Db, paths: I) -> bool where P: AsRef, I: IntoIterator, @@ -284,14 +288,10 @@ impl Files { .collect::>(); if paths.is_empty() { - return; + return false; } - let parents = paths - .iter() - .filter_map(|path| path.parent().map(SystemPath::to_path_buf)) - .collect::>(); - + let mut wrote = false; let inner = Arc::clone(&db.files().inner); for entry in inner.system_by_path.iter_mut() { let path = entry.key(); @@ -299,11 +299,25 @@ impl Files { .range(..=path.to_path_buf()) .next_back() .is_some_and(|candidate| path.starts_with(candidate.as_path())) - || parents.contains(path) { - File::sync_system_path(db, path, Some(*entry.value())); + wrote |= File::sync_system_path(db, path, Some(*entry.value())).wrote; } } + + // each of `paths` appeared or went away, which changed its parent's entries the way a + // file's appearing does + for path in &paths { + wrote |= File::touch_parent_directory_after_sync( + db, + path, + SyncPathResult { + status_changed: true, + wrote: false, + }, + ); + } + + wrote } /// Refreshes the state of all known files. @@ -314,12 +328,16 @@ impl Files { /// # Performance /// Refreshing the state of every file is expensive. It requires iterating over all known files and /// issuing a system call to get the latest status of each file. - pub fn sync_all(db: &mut dyn Db) { + /// + /// Returns whether it changed anything in the database. + pub fn sync_all(db: &mut dyn Db) -> bool { tracing::debug!("Syncing all files"); + let mut wrote = false; let inner = Arc::clone(&db.files().inner); for entry in inner.system_by_path.iter_mut() { - File::sync_system_path(db, entry.key(), Some(*entry.value())); + wrote |= File::sync_system_path(db, entry.key(), Some(*entry.value())).wrote; } + wrote } } @@ -395,6 +413,8 @@ impl get_size2::GetSize for File {} struct SyncPathResult { status_changed: bool, + /// whether the sync set any of the file's fields + wrote: bool, } impl File { @@ -453,25 +473,33 @@ impl File { /// /// Directory listings are invalidated if the path's file status changed, its prior status is /// unknown, or if `path` is itself a directory. - pub fn sync_path(db: &mut dyn Db, path: &SystemPath) { + /// + /// Returns whether it changed anything in the database. + pub fn sync_path(db: &mut dyn Db, path: &SystemPath) -> bool { let absolute = SystemPath::absolute(path, db.system().current_directory()); let result = Self::sync_system_path(db, &absolute, None); - Self::touch_parent_directory_after_sync(db, &absolute, result); + let wrote = result.wrote; + Self::touch_parent_directory_after_sync(db, &absolute, result) || wrote } /// Refreshes *only* the file metadata by querying the file system if needed. /// - /// This specifically does not invalidate any directory listings. - pub fn sync_path_only(db: &mut dyn Db, path: &SystemPath) { + /// This specifically does not invalidate any directory listings. Returns whether it changed + /// anything in the database. + pub fn sync_path_only(db: &mut dyn Db, path: &SystemPath) -> bool { let absolute = SystemPath::absolute(path, db.system().current_directory()); - Self::sync_system_path(db, &absolute, None); + Self::sync_system_path(db, &absolute, None).wrote } /// Increments the revision for the virtual file at `path`. - pub fn sync_virtual_path(db: &mut dyn Db, path: &SystemVirtualPath) { - if let Some(virtual_file) = db.files().try_virtual_file(path) { - virtual_file.sync(db); - } + /// + /// Returns whether there is a virtual file at `path`. + pub fn sync_virtual_path(db: &mut dyn Db, path: &SystemVirtualPath) -> bool { + let Some(virtual_file) = db.files().try_virtual_file(path) else { + return false; + }; + virtual_file.sync(db); + true } /// Syncs the [`File`]'s state with the state of the file on the system. @@ -498,6 +526,7 @@ impl File { let Some(file) = file.or_else(|| db.files().try_system(db, path)) else { return SyncPathResult { status_changed: true, + wrote: false, }; }; @@ -532,28 +561,49 @@ impl File { clear_override = true; } + let mut permissions_changed = false; if file.permissions(db) != permission { tracing::debug!("Updating the permissions of `{}`", file.path(db)); file.set_permissions(db).to(permission); + permissions_changed = true; } if clear_override && file.source_text_override(db).is_some() { file.set_source_text_override(db).to(None); } - SyncPathResult { status_changed } + SyncPathResult { + status_changed, + wrote: clear_override || permissions_changed, + } } + /// syncs the directories whose entries a sync of `path` shows to have changed, and returns + /// whether that changed anything in the database + /// + /// a path that appeared or went away changed its parent's entries. so did the parent, if it + /// appeared or went away with it (a file written into a directory made for it), or if + /// nothing had read it, when nothing can say it did not. up from there, to the first + /// directory that was there before and after: the only one whose entries changed without + /// its own existence changing + /// + /// all the way, and not only to the parent, because a change is often reported for the + /// path alone: a client watching `**/*.py` says nothing of the directories a new file + /// brought with it, and a query that listed the directory above them would never be told fn touch_parent_directory_after_sync( db: &mut dyn Db, - path: &SystemPath, - result: SyncPathResult, - ) { - if result.status_changed + mut path: &SystemPath, + mut result: SyncPathResult, + ) -> bool { + let mut wrote = false; + while result.status_changed && let Some(parent) = path.parent() { - Self::sync_system_path(db, parent, None); + result = Self::sync_system_path(db, parent, None); + wrote |= result.wrote; + path = parent; } + wrote } /// Returns `true` if the file exists. diff --git a/crates/ruff_db/src/files/directory.rs b/crates/ruff_db/src/files/directory.rs index ba2f445a6d..fde306708e 100644 --- a/crates/ruff_db/src/files/directory.rs +++ b/crates/ruff_db/src/files/directory.rs @@ -136,10 +136,54 @@ fn directory_listing_query( #[cfg(test)] mod tests { - use crate::files::directory_listing; - use crate::system::{DbWithWritableSystem as _, SystemPath}; + use crate::files::{File, Files, directory_listing}; + use crate::system::{DbWithWritableSystem as _, SystemPath, WritableSystem as _}; use crate::tests::TestDb; + fn names(db: &TestDb, directory: &str) -> Vec { + directory_listing(db, SystemPath::new(directory)) + .unwrap() + .iter() + .map(|(name, _)| name.to_string()) + .collect() + } + + /// a change is often reported for a file alone, as a client watching `**/*.py` reports it, + /// and not for the directory that was made for it: the directory above the new one is still + /// the one whose entries changed + #[test] + fn a_file_written_into_a_new_directory_changes_the_listing_above_it() -> std::io::Result<()> { + let mut db = TestDb::new(); + db.write_file("src/a.py", "")?; + assert_eq!(names(&db, "src"), ["a.py"]); + + let file = SystemPath::new("src/pkg/module.py"); + db.writable_system() + .create_directory_all(SystemPath::new("src/pkg"))?; + db.writable_system().write_file(file, "")?; + File::sync_path(&mut db, file); + + assert_eq!(names(&db, "src"), ["a.py", "pkg"]); + + Ok(()) + } + + /// the same holds of a directory reported alone, without the new directory it was made in + #[test] + fn a_directory_made_in_a_new_directory_changes_the_listing_above_it() -> std::io::Result<()> { + let mut db = TestDb::new(); + db.write_file("src/a.py", "")?; + assert_eq!(names(&db, "src"), ["a.py"]); + + let directory = SystemPath::new("src/pkg/sub"); + db.writable_system().create_directory_all(directory)?; + Files::sync_all_recursive(&mut db, [directory]); + + assert_eq!(names(&db, "src"), ["a.py", "pkg"]); + + Ok(()) + } + #[test] fn listing_is_sorted() -> std::io::Result<()> { let mut db = TestDb::new(); diff --git a/crates/ty/src/by_commands.rs b/crates/ty/src/by_commands.rs index 55628c3d0f..eabf23b338 100644 --- a/crates/ty/src/by_commands.rs +++ b/crates/ty/src/by_commands.rs @@ -1325,6 +1325,10 @@ pub(crate) fn cmd_compile( // manifest left describing the run before it would have the *next* run prune // against a tree that no longer exists let compiling: anyhow::Result<()> = 'compiling: { + // `by_build` writes each artifact before it is recorded, so the tree is marked first + if let Err(error) = staging.mark_as_output() { + break 'compiling Err(error); + } for ((path, file), name) in planned { let source = match fs::read_to_string(path) .with_context(|| format!("could not read {}", path.display())) diff --git a/crates/ty/tests/by_e2e.rs b/crates/ty/tests/by_e2e.rs index 07dfd7b554..833d244324 100644 --- a/crates/ty/tests/by_e2e.rs +++ b/crates/ty/tests/by_e2e.rs @@ -6847,3 +6847,101 @@ fn a_restaged_module_finds_a_helper_the_build_never_used() { String::from_utf8_lossy(&restaged.stderr) ); } + +/// `by check` in `dir`, as the path, relative to `dir`, of each diagnostic it reports +fn checked_paths(dir: &Path, extra: &[&str]) -> Vec { + let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") + .args(["check", "--output-format", "concise"]) + .args(extra) + .current_dir(dir) + .output() + .expect("failed to spawn by"); + let stdout = String::from_utf8_lossy(&output.stdout); + let mut paths: Vec = stdout + .lines() + .filter(|line| !line.starts_with("Found ") && !line.trim().is_empty()) + .filter_map(|line| { + line.split_once(':') + .map(|(path, _)| path.replace('\\', "/")) + }) + .collect(); + paths.sort(); + paths +} + +/// a project whose one `.by` module and one hand-written `.py` module each assign a `str` to an +/// `int`, so that a copy of either in a build output would be reported again +fn project_with_an_error_in_each_module(name: &str) -> Scratch { + let dir = Scratch::new(name); + fs::create_dir_all(dir.join("pkg")).unwrap(); + fs::write( + dir.join("pyproject.toml"), + "[project]\nname=\"s\"\nversion=\"0\"\nrequires-python=\">=3.13\"\n", + ) + .unwrap(); + fs::write(dir.join("pkg").join("__init__.py"), "").unwrap(); + fs::write(dir.join("pkg").join("main.by"), "x: int = \"by\"\n").unwrap(); + fs::write(dir.join("pkg").join("helper.py"), "y: int = \"py\"\n").unwrap(); + dir +} + +/// `by build` in `dir` with `args`, which has to write `pkg/main.py` into `out` +/// +/// the build reports the project's type errors and exits non-zero for them, and writes the tree +/// all the same, which is the tree these tests are about +fn build_into(dir: &Path, out: &str, args: &[&str]) { + let build = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") + .arg("build") + .args(args) + .current_dir(dir) + .output() + .expect("failed to spawn by"); + assert!( + dir.join(out).join("pkg").join("main.py").exists(), + "by build wrote no tree:\n{}", + String::from_utf8_lossy(&build.stderr) + ); +} + +#[test] +fn check_does_not_check_the_tree_a_build_wrote() { + // the build output is a copy of the project: a `.py` for every `.by` and every hand-written + // `.py` carried over. checked as part of the project, each copy is checked again and every + // diagnostic is reported twice, once at a path nobody edits — in a project that imports + // torch that was most of what a check cost + let dir = project_with_an_error_in_each_module("by_cli_check_skips_build"); + build_into(&dir, "build", &[]); + + assert_eq!( + checked_paths(&dir, &[]), + ["pkg/helper.py", "pkg/main.by"], + "only the sources are checked, not their copies under build/" + ); + + // a tree asked for by name is still checked: naming it is asking for it + let named = checked_paths(&dir, &["build"]); + assert!( + named.iter().any(|path| path.starts_with("build/")), + "the output named on the command line was not checked: {named:?}" + ); +} + +#[test] +fn check_follows_the_build_to_another_directory_and_keeps_a_package_called_build() { + // the output is recognised by the manifest the build leaves, not by its name: `--out` can + // write anywhere, and a directory of the author's own that happens to be called `build` is + // part of the project when the build writes somewhere else + let dir = project_with_an_error_in_each_module("by_cli_check_skips_out"); + fs::create_dir_all(dir.join("build")).unwrap(); + fs::write(dir.join("build").join("tool.py"), "z: int = \"mine\"\n").unwrap(); + + build_into(&dir, "elsewhere", &["--out", "elsewhere"]); + + assert_eq!( + checked_paths(&dir, &[]), + ["build/tool.py", "pkg/helper.py", "pkg/main.by"], + "the sources and the author's own `build/`, and nothing the build wrote" + ); +} diff --git a/crates/ty/tests/file_watching.rs b/crates/ty/tests/file_watching.rs index 3cf0b1e711..7df019b28b 100644 --- a/crates/ty/tests/file_watching.rs +++ b/crates/ty/tests/file_watching.rs @@ -1,5 +1,7 @@ use std::collections::HashSet; use std::io::Write; +#[cfg(unix)] +use std::os::unix::fs::PermissionsExt as _; use std::time::{Duration, Instant}; use anyhow::{Context, anyhow}; @@ -18,7 +20,9 @@ use ty_project::metadata::options::{EnvironmentOptions, Options, SrcOptions}; use ty_project::metadata::pyproject::{PyProject, Tool}; use ty_project::metadata::python_version::SupportedPythonVersion; use ty_project::metadata::value::{RelativeGlobPattern, RelativePathBuf}; -use ty_project::watch::{ChangeEvent, ProjectWatcher, directory_watcher}; +use ty_project::watch::{ + ChangeEvent, ChangedKind, CreatedKind, DeletedKind, ProjectWatcher, directory_watcher, +}; use ty_project::{ChangeResult, Db, ProjectDatabase, ProjectMetadata}; use ty_python_core::platform::PythonPlatform; use ty_static::EnvVars; @@ -975,6 +979,118 @@ fn changed_file() -> anyhow::Result<()> { Ok(()) } +/// a change reported for a file whose content, metadata and place in the project are all as they +/// were changes nothing, however many times it is reported, and a real change to the file does +#[test] +fn a_change_that_changes_nothing_leaves_the_database_as_it_was() -> anyhow::Result<()> { + let mut case = setup([("foo.py", "x = 1\n")])?; + let foo_path = case.project_path("foo.py"); + let foo = case.system_file(&foo_path)?; + case.assert_indexed_project_files([foo]); + + let unchanged = [ChangeEvent::Changed { + path: foo_path.clone(), + kind: ChangedKind::Any, + }]; + // applying a batch cancels the database's other readers, and salsa counts cancellations in a + // `u8`, starting a new revision with no write in it when the count wraps: enough batches for + // it to wrap several times + for batch in 0..1024 { + let result = case.apply_changes(&unchanged); + assert!( + !result.database_changed(), + "batch {batch} changed the database" + ); + } + + update_file(&foo_path, "x = 2\n")?; + assert!(case.apply_changes(&unchanged).database_changed()); + + Ok(()) +} + +/// a batch whose only effect is to clear a diagnostic the walk reported still changes what a +/// check of the project says +#[cfg(unix)] +#[test] +fn clearing_a_walk_diagnostic_changes_the_database() -> anyhow::Result<()> { + let mut case = setup([("foo.py", ""), ("locked/bar.py", "")])?; + let foo_path = case.project_path("foo.py"); + let locked = case.project_path("locked"); + let unreadable = |case: &TestCase| { + case.db() + .check() + .iter() + .any(|diagnostic| diagnostic.concise_message().to_string().contains("locked")) + }; + + std::fs::set_permissions(locked.as_std_path(), std::fs::Permissions::from_mode(0o000))?; + let foo = case.system_file(&foo_path)?; + case.assert_indexed_project_files([foo]); + std::fs::set_permissions(locked.as_std_path(), std::fs::Permissions::from_mode(0o755))?; + assert!( + unreadable(&case), + "the walk reports the directory it could not read" + ); + + let result = case.apply_changes(&[ChangeEvent::Changed { + path: foo_path, + kind: ChangedKind::Any, + }]); + assert!(result.database_changed()); + assert!(!unreadable(&case)); + + Ok(()) +} + +/// a manifest written into a directory of the project makes the directory a build's output, +/// which is not part of the project +#[test] +fn a_build_manifest_created_takes_its_directory_out_of_the_project() -> anyhow::Result<()> { + let mut case = setup([("app.py", ""), ("build/app.py", "")])?; + let app = case.system_file(case.project_path("app.py"))?; + let built = case.system_file(case.project_path("build/app.py"))?; + case.assert_indexed_project_files([app, built]); + + let manifest = case.project_path("build/.by-manifest"); + std::fs::write(manifest.as_std_path(), "app.py\n")?; + let result = case.apply_changes(&[ChangeEvent::Created { + path: manifest, + kind: CreatedKind::File, + }]); + + assert!(result.database_changed()); + case.assert_indexed_project_files([app]); + + Ok(()) +} + +/// a build's output whose manifest is deleted is a directory like any other, and part of the +/// project again +#[test] +fn a_build_manifest_deleted_returns_its_directory_to_the_project() -> anyhow::Result<()> { + let mut case = setup([ + ("app.py", ""), + ("build/.by-manifest", "app.py\n"), + ("build/app.py", ""), + ])?; + let app = case.system_file(case.project_path("app.py"))?; + case.assert_indexed_project_files([app]); + + let manifest = case.project_path("build/.by-manifest"); + std::fs::remove_file(manifest.as_std_path())?; + let result = case.apply_changes(&[ChangeEvent::Deleted { + path: manifest, + kind: DeletedKind::File, + }]); + + assert!(result.database_changed()); + let built = case.system_file(case.project_path("build/app.py"))?; + case.assert_indexed_project_files([app, built]); + + Ok(()) +} + #[test] fn scripts_to_synchronize_after_file_and_directory_changes() -> anyhow::Result<()> { let script = dedent( diff --git a/crates/ty_ide/src/django_template/inlay_hints.rs b/crates/ty_ide/src/django_template/inlay_hints.rs index 91b65ac178..232f48b9ef 100644 --- a/crates/ty_ide/src/django_template/inlay_hints.rs +++ b/crates/ty_ide/src/django_template/inlay_hints.rs @@ -11,6 +11,8 @@ use ruff_text_size::{TextRange, TextSize}; use ty_project::Db; use crate::InlayHintSettings; +use crate::inlay_hints::hint_switches; +use strum_macros::EnumIter; use super::index::{BindingOrigin, TemplateIndex, TemplateReference}; use super::lexer::TokenKind; @@ -30,7 +32,7 @@ pub struct TemplateInlayHint { } /// what a template hint says -#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, EnumIter)] pub enum TemplateInlayHintKind { /// the type of a name the template binds Type, @@ -38,6 +40,11 @@ pub enum TemplateInlayHintKind { Template, } +hint_switches!(TemplateInlayHintKind, template_enabled, set_template_enabled, { + Type => template_binding_types = "templateBindingTypes", + Template => resolved_templates = "resolvedTemplates", +}); + /// every hint `range` of the template `file` shows pub(crate) fn inlay_hints( db: &dyn Db, @@ -50,11 +57,11 @@ pub(crate) fn inlay_hints( ) -> Vec { let mut hints = Vec::new(); - if settings.template_binding_types { + if settings.template_enabled(TemplateInlayHintKind::Type) { binding_types(db, env, file, index, source, range, &mut hints); } - if settings.resolved_templates { + if settings.template_enabled(TemplateInlayHintKind::Template) { for reference in index.extends().into_iter().chain(index.includes()) { resolved_template(db, index, range, reference, &mut hints); } diff --git a/crates/ty_ide/src/django_template/project.rs b/crates/ty_ide/src/django_template/project.rs index 2c471f40cc..6bccf69ff6 100644 --- a/crates/ty_ide/src/django_template/project.rs +++ b/crates/ty_ide/src/django_template/project.rs @@ -12,10 +12,10 @@ //! type checker has already populated for every project file, the ast work is //! shared rather than duplicated. -use std::sync::Mutex; +use std::sync::{Mutex, PoisonError}; use compact_str::{CompactString, ToCompactString}; -use ruff_db::files::{File, FilePath, system_path_to_file}; +use ruff_db::files::{File, FilePath, system_path_to_directory, system_path_to_file}; use ruff_db::parsed::parsed_module; use ruff_db::source::source_text; use ruff_db::system::walk_directory::WalkState; @@ -28,6 +28,7 @@ use ty_module_resolver::ImportingFile; use ty_module_resolver::{ Module, ModuleName, file_to_module, resolve_module_confident, resolve_real_module, }; +use ty_project::build_output::is_build_output; use ty_project::{Db, Project}; use ty_python_core::definition::DefinitionKind; use ty_python_semantic::ProgramEnvironment; @@ -1634,11 +1635,6 @@ fn discover( directory: &str, order: &SearchOrder, ) -> Box<[DiscoveredFile]> { - // walking the file system is invisible to salsa, so the callers' queries would - // hand back their first answer forever. reading the revision the project bumps - // on every create and delete is what makes a template added mid-session show up - let _ = project.file_system_revision(db); - let mut found = discover_by_convention(db, project, directory, order); // a directory the settings name is under no obligation to be called @@ -1692,13 +1688,18 @@ fn discover_by_convention( ) -> Vec { let root = project.root(db); let found = Mutex::new(Vec::new()); + let listed = Mutex::new(Vec::new()); db.system() .walk_directory(root) .standard_filters(project.settings(db).src().respect_ignore_files) .ignore_hidden(true) .run(|| { - Box::new(|entry| { + // a walk visits from threads of its own, which each need a handle on the database + let db = Db::dyn_clone(db); + let found = &found; + let listed = &listed; + Box::new(move |entry| { let Ok(entry) = entry else { return WalkState::Continue; }; @@ -1715,6 +1716,14 @@ fn discover_by_convention( if !inside && entry.depth() >= DISCOVERY_DEPTH { return WalkState::Skip; } + list(&*db, listed, entry.path()); + // a build's output is a copy of the project, templates and all, and + // not the project's own (see `ty_project::build_output`). its manifest + // is one of the entries of the directory just listed, so writing or + // deleting it is a change to what the walk read + if entry.depth() > 0 && is_build_output(db.system(), entry.path()) { + return WalkState::Skip; + } WalkState::Continue } FileType::File => { @@ -1722,15 +1731,17 @@ fn discover_by_convention( && let Some((under, name)) = relative_to_directory(entry.path(), directory) { - found.lock().unwrap().push(DiscoveredFile { - name, - path: entry.path().to_path_buf(), - precedence: order.rank(under), - // the walk starts at the project root and - // respects its ignore rules, so whatever it - // reaches is the project's own - own: true, - }); + found.lock().unwrap_or_else(PoisonError::into_inner).push( + DiscoveredFile { + name, + path: entry.path().to_path_buf(), + precedence: order.rank(under), + // the walk starts at the project root and + // respects its ignore rules, so whatever it + // reaches is the project's own + own: true, + }, + ); } WalkState::Continue } @@ -1739,9 +1750,45 @@ fn discover_by_convention( }) }); - found - .into_inner() - .unwrap_or_else(std::sync::PoisonError::into_inner) + depend_on_entries( + db, + listed.into_inner().unwrap_or_else(PoisonError::into_inner), + ); + + found.into_inner().unwrap_or_else(PoisonError::into_inner) +} + +/// note that a walk is about to read the entries of `directory`, for [`depend_on_entries`] +/// +/// the database reads a directory's revision the first time it is asked about the directory, +/// and it is asked here, before the walk reads the entries. a revision first read after them +/// could already take in an entry created in between that the walk never saw, and the change +/// the watcher then reports would find the revision as it was and leave the answer missing it +fn list(db: &dyn Db, listed: &Mutex>, directory: &SystemPath) { + let _ = system_path_to_directory(db, directory); + listed + .lock() + .unwrap_or_else(PoisonError::into_inner) + .push(directory.to_path_buf()); +} + +/// make the calling query depend on which entries each of `directories` holds +/// +/// walking the file system is invisible to salsa, so a query that walks would hand back its +/// first answer forever. a directory's revision changes when an entry is created in it, deleted +/// from it or renamed, and the file watcher's changes sync the directory holding each path they +/// name, so depending on the revision of every directory a walk read, and on whether a root it +/// was asked to walk exists, is what makes a template added mid-session show up. and only that: +/// a write anywhere the walk did not go, inside a build's output or an ignored directory, +/// changes nothing the query read +fn depend_on_entries(db: &dyn Db, directories: Vec) { + for directory in directories { + // an `Err` still read the path's status: a root that is not there yet is depended on + // being absent + if let Ok(directory) = system_path_to_directory(db, &directory) { + let _ = directory.revision(db); + } + } } /// add every file under `root` to `found`, named the way django would name it @@ -1754,36 +1801,56 @@ fn collect_under( found: &mut Vec, ) { let collected = Mutex::new(Vec::new()); + let listed = Mutex::new(Vec::new()); + // before the walk, and whether or not it is there: a root that does not exist yet is + // depended on being absent + list(db, &listed, root); db.system() .walk_directory(root) .standard_filters(project.settings(db).src().respect_ignore_files) .ignore_hidden(true) .run(|| { - Box::new(|entry| { + // a walk visits from threads of its own, which each need a handle on the database + let db = Db::dyn_clone(db); + let collected = &collected; + let listed = &listed; + Box::new(move |entry| { let Ok(entry) = entry else { return WalkState::Continue; }; + if entry.file_type().is_directory() && entry.depth() > 0 { + list(&*db, listed, entry.path()); + } + if matches!(entry.file_type(), FileType::File) && let Some(name) = relative_to_root(entry.path(), root) { - collected.lock().unwrap().push(DiscoveredFile { - name, - path: entry.path().to_path_buf(), - precedence, - own, - }); + collected + .lock() + .unwrap_or_else(PoisonError::into_inner) + .push(DiscoveredFile { + name, + path: entry.path().to_path_buf(), + precedence, + own, + }); } WalkState::Continue }) }); + depend_on_entries( + db, + listed.into_inner().unwrap_or_else(PoisonError::into_inner), + ); + found.extend( collected .into_inner() - .unwrap_or_else(std::sync::PoisonError::into_inner), + .unwrap_or_else(PoisonError::into_inner), ); } diff --git a/crates/ty_ide/src/inlay_hints.rs b/crates/ty_ide/src/inlay_hints.rs index 2d653371b4..ebd6bf52c5 100644 --- a/crates/ty_ide/src/inlay_hints.rs +++ b/crates/ty_ide/src/inlay_hints.rs @@ -3,6 +3,7 @@ use ty_python_semantic::ProgramEnvironment; use itertools::{Either, Itertools}; use rustc_hash::FxHashMap; +use strum_macros::EnumIter; use crate::importer::{ImportAction, ImportRequest, Importer, MembersInScope}; use crate::{Db, HasNavigationTargets, NavigationTarget}; @@ -87,7 +88,7 @@ impl InlayHint { Some(Self { position, - kind: InlayHintKind::Type, + kind: InlayHintKind::VariableType, label: InlayHintLabel { parts: label_parts }, padding_left: false, padding_right: false, @@ -285,7 +286,7 @@ impl InlayHint { Self { position, - kind: InlayHintKind::TypeArgument, + kind: InlayHintKind::CallTypeArguments, label: InlayHintLabel { parts }, padding_left: false, padding_right: false, @@ -324,7 +325,7 @@ impl InlayHint { fn type_argument_name(position: TextSize, name: &str) -> Self { Self { position, - kind: InlayHintKind::TypeArgument, + kind: InlayHintKind::TypeArgumentName, label: InlayHintLabel { parts: vec![InlayHintLabelPart::new(name), "=".into()], }, @@ -436,6 +437,7 @@ impl InlayHint { fn implicit_parameters( db: &dyn Db, env: &ProgramEnvironment<'_>, + kind: InlayHintKind, position: TextSize, parameters: &[(&str, Option)], leading_space: bool, @@ -459,7 +461,7 @@ impl InlayHint { Self { position, - kind: InlayHintKind::ImplicitParameter, + kind, label: InlayHintLabel { parts }, padding_left: leading_space, padding_right: parameter_follows, @@ -477,7 +479,7 @@ impl InlayHint { ) -> Self { Self { position, - kind: InlayHintKind::Type, + kind: InlayHintKind::InferredReturnType, label: InlayHintLabel { parts: vec![format!("-> {}", returned.display(db, env)).into()], }, @@ -493,12 +495,13 @@ impl InlayHint { fn inferred_annotation( db: &dyn Db, env: &ProgramEnvironment<'_>, + kind: InlayHintKind, position: TextSize, ty: Type, ) -> Self { Self { position, - kind: InlayHintKind::Type, + kind, label: InlayHintLabel { parts: vec![format!(": {}", ty.display(db, env)).into()], }, @@ -513,9 +516,25 @@ impl InlayHint { } } -#[derive(Debug, Clone)] +/// what a hint in a python or basedpython file is +/// +/// each kind is switched by a setting of [`InlayHintSettings`] of its own, so a +/// client can tell every kind apart by [`InlayHintKind::setting`] rather than +/// by reading its label. the django template hints have kinds of their own, in +/// `TemplateInlayHintKind` +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, EnumIter)] pub enum InlayHintKind { - Type, + /// The type of an unannotated binding + VariableType, + /// The type of an unannotated lambda parameter + LambdaParameterType, + /// basedpython: the type a parameter takes from the method it overrides or + /// the overloads it implements + InheritedParameterType, + /// basedpython: the type a property declaration leaves to its accessors + PropertyType, + /// basedpython: the return type of a `def` that leaves its annotation out + InferredReturnType, CallArgumentName, /// basedpython: a function's inferred exception set Raises, @@ -523,9 +542,10 @@ pub enum InlayHintKind { Variance, /// basedpython: a type parameter reified without saying so Reification, - /// The type arguments inferred for a generic call, or the name of the type - /// parameter a positional type argument fills - TypeArgument, + /// The type arguments inferred for a generic call + CallTypeArguments, + /// The name of the type parameter a positional type argument fills + TypeArgumentName, /// basedpython: a method that overrides a superclass member without saying so Override, /// basedpython-ui: the observables a function reads while composing @@ -542,14 +562,123 @@ pub enum InlayHintKind { RevealedType, /// The value an enum member takes without the source writing one EnumValue, - /// basedpython: a parameter the source never spells (`it`, `self`) + /// basedpython: the parameters a trailing lambda binds without spelling them: + /// `it`, and the receiver named `self` when the callback declares one ImplicitParameter, + /// basedpython: the `self` an `init(...)` binds without spelling it + ImplicitSelf, /// basedpython: an argument a call site fills from a `context` declaration ImplicitArgument, /// basedpython: the default a parameter takes from the method its `def` overrides InheritedDefault, } +impl InlayHintKind { + /// whether a client shows this kind of hint as a parameter rather than as a + /// type, which are the only two kinds LSP has + pub const fn is_parameter(self) -> bool { + match self { + InlayHintKind::VariableType + | InlayHintKind::LambdaParameterType + | InlayHintKind::InheritedParameterType + | InlayHintKind::PropertyType + | InlayHintKind::InferredReturnType + // basedpython: an inferred exception set is a type, like a return type + | InlayHintKind::Raises + | InlayHintKind::Variance + | InlayHintKind::Reification + | InlayHintKind::CallTypeArguments + | InlayHintKind::TypeArgumentName + | InlayHintKind::Override + // basedpython-ui: a read set, a dependency set and an invalidation set + // are typing facts about the function, and `unstable` is a modifier + // like `override` + | InlayHintKind::Reads + | InlayHintKind::Stability + | InlayHintKind::DerivedDeps + | InlayHintKind::Invalidates + | InlayHintKind::NumericPromotion + | InlayHintKind::RevealedType + | InlayHintKind::EnumValue => false, + InlayHintKind::CallArgumentName + | InlayHintKind::ImplicitParameter + | InlayHintKind::ImplicitSelf + | InlayHintKind::ImplicitArgument + | InlayHintKind::InheritedDefault => true, + } + } +} + +/// the one table saying which setting switches each kind of hint +/// +/// from it come the kind's `setting()`, the `inlayHints` option a client knows +/// the setting by, and the `InlayHintSettings` methods that read and write the +/// field holding it, so a kind cannot be named by one setting and switched by +/// another. a client that lets its user treat each kind differently needs to +/// know which kind a hint is, and the label is what a hint says rather than +/// what it is: a revealed `list[int]` and a call's `[int]` differ by a bracket +macro_rules! hint_switches { + ( + $kind:ident, + $enabled:ident, + $set_enabled:ident, + { $($variant:ident => $field:ident = $setting:literal,)* } + ) => { + impl $kind { + /// the name of the `inlayHints` option that switches this kind of hint + pub const fn setting(self) -> &'static str { + match self { + $($kind::$variant => $setting,)* + } + } + } + + impl $crate::InlayHintSettings { + /// whether hints of `kind` are shown + pub const fn $enabled(&self, kind: $kind) -> bool { + match kind { + $($kind::$variant => self.$field,)* + } + } + + /// show or hide hints of `kind` + pub const fn $set_enabled(&mut self, kind: $kind, enabled: bool) { + match kind { + $($kind::$variant => self.$field = enabled,)* + } + } + } + }; +} + +pub(crate) use hint_switches; + +hint_switches!(InlayHintKind, enabled, set_enabled, { + VariableType => variable_types = "variableTypes", + LambdaParameterType => lambda_parameter_types = "lambdaParameterTypes", + InheritedParameterType => inherited_parameter_types = "inheritedParameterTypes", + PropertyType => property_types = "propertyTypes", + InferredReturnType => inferred_return_types = "inferredReturnTypes", + CallArgumentName => call_argument_names = "callArgumentNames", + Raises => inferred_raises = "inferredRaises", + Variance => inferred_variance = "inferredVariance", + Reification => inferred_reification = "inferredReification", + CallTypeArguments => call_type_arguments = "callTypeArguments", + TypeArgumentName => type_argument_names = "typeArgumentNames", + Override => inferred_override = "inferredOverride", + Reads => inferred_reads = "inferredReads", + Stability => parameter_stability = "parameterStability", + DerivedDeps => derived_dependencies = "derivedDependencies", + Invalidates => inferred_invalidations = "inferredInvalidations", + NumericPromotion => numeric_promotions = "numericPromotions", + RevealedType => revealed_types = "revealedTypes", + EnumValue => enum_values = "enumValues", + ImplicitParameter => implicit_parameters = "implicitParameters", + ImplicitSelf => implicit_self = "implicitSelf", + ImplicitArgument => implicit_arguments = "implicitArguments", + InheritedDefault => inherited_parameter_defaults = "inheritedParameterDefaults", +}); + #[derive(Debug, Clone)] pub struct InlayHintLabel { parts: Vec, @@ -767,6 +896,14 @@ pub struct InlayHintSettings { /// ```python /// x: dict["_KT="str, "_VT="int] /// ``` + /// + /// in a basedpython file it also names the type arguments of the types the + /// variable type and call type argument hints show, which are still shown + /// and hidden by their own settings + /// + /// ```by + /// x": dict[Key=str, Value=int]" = identity({"a": 1}) + /// ``` pub type_argument_names: bool, /// basedpython: whether to show `override` on a method that overrides a @@ -1330,8 +1467,21 @@ impl<'a, 'db> InlayHintVisitor<'a, 'db> { self.source_type.is_basedpython() } + /// add `hint` if the setting of its kind is on, answering whether it was added + /// + /// every hint is added here, so a hint is shown exactly when the one setting + /// its kind names is on. the same check made before a hint is worked out + /// only saves the work of one that would be dropped + fn push(&mut self, hint: InlayHint) -> bool { + if !self.settings.enabled(hint.kind) { + return false; + } + self.hints.push(hint); + true + } + fn add_type_hint(&mut self, expr: &Expr, rhs: &Expr, ty: Type<'db>, allow_edits: bool) { - if !self.settings.variable_types { + if !self.settings.enabled(InlayHintKind::VariableType) { return; } @@ -1357,7 +1507,7 @@ impl<'a, 'db> InlayHintVisitor<'a, 'db> { if let Some(inlay_hint) = InlayHint::variable_type(context, expr, rhs, ty, allow_edits, named_type_arguments) { - self.hints.push(inlay_hint); + self.push(inlay_hint); } } @@ -1367,7 +1517,7 @@ impl<'a, 'db> InlayHintVisitor<'a, 'db> { /// annotation, before the `:` — so accepting it reads as ordinary source. fn add_inferred_raises(&mut self, function: &ast::StmtFunctionDef) { let env = &self.model.program_environment(); - if !self.settings.inferred_raises || function.raises.is_some() { + if !self.settings.enabled(InlayHintKind::Raises) || function.raises.is_some() { return; } @@ -1383,8 +1533,7 @@ impl<'a, 'db> InlayHintVisitor<'a, 'db> { .as_deref() .map_or_else(|| function.parameters.end(), Ranged::end); - self.hints - .push(InlayHint::inferred_raises(self.db, env, position, raised)); + self.push(InlayHint::inferred_raises(self.db, env, position, raised)); } /// basedpython: hint the variance ty infers for each type parameter of @@ -1394,7 +1543,7 @@ impl<'a, 'db> InlayHintVisitor<'a, 'db> { type_params: Option<&ast::TypeParams>, owner: impl FnOnce(&SemanticModel<'db>) -> Option>, ) { - if !self.settings.inferred_variance || !self.is_basedpython() { + if !self.settings.enabled(InlayHintKind::Variance) || !self.is_basedpython() { return; } @@ -1427,7 +1576,7 @@ impl<'a, 'db> InlayHintVisitor<'a, 'db> { continue; }; - self.hints.push(InlayHint::inferred_variance( + self.push(InlayHint::inferred_variance( type_param.range().start(), variance, )); @@ -1455,7 +1604,7 @@ impl<'a, 'db> InlayHintVisitor<'a, 'db> { type_params: Option<&ast::TypeParams>, inferred: impl FnOnce(&mut Self) -> Vec, ) { - if !self.settings.inferred_reification || !self.is_basedpython() { + if !self.settings.enabled(InlayHintKind::Reification) || !self.is_basedpython() { return; } @@ -1469,8 +1618,7 @@ impl<'a, 'db> InlayHintVisitor<'a, 'db> { // everything the parameter's own declaration spells for type_param in type_params { if inferred.contains(&type_param.name().id) { - self.hints - .push(InlayHint::inferred_reification(type_param.range().start())); + self.push(InlayHint::inferred_reification(type_param.range().start())); } } } @@ -1478,7 +1626,7 @@ impl<'a, 'db> InlayHintVisitor<'a, 'db> { /// Hint the value an enum member takes when its declaration does not write /// one, at `position` — after `auto()`, or after a `case` variant's name. fn add_enum_member_value(&mut self, name: &str, position: TextSize) { - if !self.settings.enum_values { + if !self.settings.enabled(InlayHintKind::EnumValue) { return; } @@ -1497,7 +1645,7 @@ impl<'a, 'db> InlayHintVisitor<'a, 'db> { return; }; - self.hints.push(InlayHint::enum_member_value( + self.push(InlayHint::enum_member_value( position, &rendered.to_string(), )); @@ -1507,24 +1655,17 @@ impl<'a, 'db> InlayHintVisitor<'a, 'db> { /// member without saying so. fn add_inferred_override(&mut self, function: &ast::StmtFunctionDef) { let env = &self.model.program_environment(); - if !self.settings.inferred_override || !self.is_basedpython() { + if !self.settings.enabled(InlayHintKind::Override) || !self.is_basedpython() { return; } - let Some(class_ty) = self.enclosing_class else { - return; - }; - - let Some(superclass) = function - .inferred_type(&self.model) - .and_then(|ty| inferred_override(self.db, env, class_ty, ty, &function.name)) - else { + let Some(superclass) = inferred_override(&self.model, function) else { return; }; // the range of a `def` excludes its decorators, so this is the modifier // position even on a decorated method - self.hints.push(InlayHint::inferred_override( + self.push(InlayHint::inferred_override( function.range().start(), superclass .navigation_targets(self.db, env) @@ -1539,7 +1680,7 @@ impl<'a, 'db> InlayHintVisitor<'a, 'db> { /// annotation and the `raises` clause — and before the `:`, so it reads /// as one more clause of the header. fn add_inferred_reads(&mut self, function: &ast::StmtFunctionDef) { - if !self.settings.inferred_reads || !self.is_basedpython() { + if !self.settings.enabled(InlayHintKind::Reads) || !self.is_basedpython() { return; } @@ -1556,13 +1697,13 @@ impl<'a, 'db> InlayHintVisitor<'a, 'db> { .or(function.returns.as_deref()) .map_or_else(|| function.parameters.end(), Ranged::end); - self.hints.push(InlayHint::inferred_reads(position, &reads)); + self.push(InlayHint::inferred_reads(position, &reads)); } /// basedpython-ui: hint `unstable` before each parameter of the composable /// `function` whose declared type the runtime cannot compare. fn add_parameter_stability(&mut self, function: &ast::StmtFunctionDef) { - if !self.settings.parameter_stability || !self.is_basedpython() { + if !self.settings.enabled(InlayHintKind::Stability) || !self.is_basedpython() { return; } if !function @@ -1584,7 +1725,7 @@ impl<'a, 'db> InlayHintVisitor<'a, 'db> { continue; }; if parameter_stability(self.db, env, ty) == Some(false) { - self.hints.push(InlayHint::unstable_parameter( + self.push(InlayHint::unstable_parameter( parameter.name.range().start(), )); } @@ -1594,7 +1735,7 @@ impl<'a, 'db> InlayHintVisitor<'a, 'db> { /// basedpython-ui: hint what a `derived(...)` / `remember(...)` call's /// computation depends on, at the end of its line. fn add_derived_dependencies(&mut self, call: &ast::ExprCall) { - if !self.settings.derived_dependencies || !self.is_basedpython() { + if !self.settings.enabled(InlayHintKind::DerivedDeps) || !self.is_basedpython() { return; } @@ -1602,7 +1743,7 @@ impl<'a, 'db> InlayHintVisitor<'a, 'db> { return; }; - self.hints.push(InlayHint::derived_dependencies( + self.push(InlayHint::derived_dependencies( self.source.line_end(call.range().end()), &reads, )); @@ -1619,7 +1760,7 @@ impl<'a, 'db> InlayHintVisitor<'a, 'db> { /// after, so a write in one (`if todos.pop():`) is hinted at the end of /// the line it is spelled on. fn add_invalidations(&mut self, site: WriteSite<'_>) { - if !self.settings.inferred_invalidations || !self.is_basedpython() { + if !self.settings.enabled(InlayHintKind::Invalidates) || !self.is_basedpython() { return; } @@ -1634,14 +1775,13 @@ impl<'a, 'db> InlayHintVisitor<'a, 'db> { WriteSite::Statement(stmt) => stmt.end(), WriteSite::Lambda(lambda) => lambda.end(), }; - self.hints - .push(InlayHint::invalidations(position, &invalidations)); + self.push(InlayHint::invalidations(position, &invalidations)); } /// Hint the type arguments inferred for a generic call. fn add_call_type_arguments(&mut self, call: &ast::ExprCall, arguments: &[(Name, Type<'db>)]) { let env = &self.model.program_environment(); - if !self.settings.call_type_arguments || arguments.is_empty() { + if !self.settings.enabled(InlayHintKind::CallTypeArguments) || arguments.is_empty() { return; } @@ -1653,7 +1793,7 @@ impl<'a, 'db> InlayHintVisitor<'a, 'db> { return; } - self.hints.push(InlayHint::call_type_arguments( + self.push(InlayHint::call_type_arguments( self.db, env, call.func.range().end(), @@ -1676,7 +1816,7 @@ impl<'a, 'db> InlayHintVisitor<'a, 'db> { /// — after the explicit arguments, by keyword. fn add_implicit_context_arguments(&mut self, call: &ast::ExprCall, callee: Option>) { let env = &self.model.program_environment(); - if !self.settings.implicit_arguments || !self.is_basedpython() { + if !self.settings.enabled(InlayHintKind::ImplicitArgument) || !self.is_basedpython() { return; } @@ -1709,7 +1849,7 @@ impl<'a, 'db> InlayHintVisitor<'a, 'db> { |argument| argument.range().end(), ); - self.hints.push(InlayHint::implicit_context_arguments( + self.push(InlayHint::implicit_context_arguments( position, last_explicit.is_some(), &labelled, @@ -1719,7 +1859,7 @@ impl<'a, 'db> InlayHintVisitor<'a, 'db> { /// Hint the name of the type parameter each positional argument of a /// subscripted generic fills. fn add_type_argument_names(&mut self, subscript: &ast::ExprSubscript) { - if !self.settings.type_argument_names { + if !self.settings.enabled(InlayHintKind::TypeArgumentName) { return; } @@ -1738,7 +1878,7 @@ impl<'a, 'db> InlayHintVisitor<'a, 'db> { } for (name, argument) in names.iter().zip(subscript_arguments(&subscript.slice)) { - self.hints.push(InlayHint::type_argument_name( + self.push(InlayHint::type_argument_name( argument.range().start(), name, )); @@ -1749,7 +1889,7 @@ impl<'a, 'db> InlayHintVisitor<'a, 'db> { /// `float` / `complex` type expression. fn add_numeric_promotion(&mut self, expr: &Expr) { let env = &self.model.program_environment(); - if !self.settings.numeric_promotions || !self.in_type_expression { + if !self.settings.enabled(InlayHintKind::NumericPromotion) || !self.in_type_expression { return; } @@ -1769,8 +1909,7 @@ impl<'a, 'db> InlayHintVisitor<'a, 'db> { return; }; - self.hints - .push(InlayHint::numeric_promotion(expr.range().end(), arms)); + self.push(InlayHint::numeric_promotion(expr.range().end(), arms)); } /// Visit the operands of a union written in a type expression, each knowing what the @@ -1804,7 +1943,7 @@ impl<'a, 'db> InlayHintVisitor<'a, 'db> { /// Hint the type a `reveal_type` call reveals, at the end of its line. fn add_revealed_type(&mut self, call: &ast::ExprCall) { let env = &self.model.program_environment(); - if !self.settings.revealed_types { + if !self.settings.enabled(InlayHintKind::RevealedType) { return; } @@ -1815,7 +1954,7 @@ impl<'a, 'db> InlayHintVisitor<'a, 'db> { return; }; - self.hints.push(InlayHint::revealed_type( + self.push(InlayHint::revealed_type( self.db, env, self.source.line_end(call.range().end()), @@ -1834,16 +1973,20 @@ impl<'a, 'db> InlayHintVisitor<'a, 'db> { /// head instead. fn add_implicit_self(&mut self, parameter: &ast::Parameter) { let env = &self.model.program_environment(); - if !self.settings.implicit_self || !self.is_basedpython() || !parameter.range().is_empty() { + if !self.settings.enabled(InlayHintKind::ImplicitSelf) + || !self.is_basedpython() + || !parameter.range().is_empty() + { return; } let position = parameter.range().start(); let ty = hintable_parameter_type(&self.model, parameter); - self.hints.push(InlayHint::implicit_parameters( + self.push(InlayHint::implicit_parameters( self.db, env, + InlayHintKind::ImplicitSelf, position, &[(parameter.name.as_str(), ty)], false, @@ -1859,7 +2002,7 @@ impl<'a, 'db> InlayHintVisitor<'a, 'db> { /// the colon rather than between the callee and it. fn add_trailing_lambda_parameter(&mut self, function: &ast::StmtFunctionDef) { let env = &self.model.program_environment(); - if !self.settings.implicit_parameters { + if !self.settings.enabled(InlayHintKind::ImplicitParameter) { return; } @@ -1877,9 +2020,10 @@ impl<'a, 'db> InlayHintVisitor<'a, 'db> { return; } - self.hints.push(InlayHint::implicit_parameters( + self.push(InlayHint::implicit_parameters( self.db, env, + InlayHintKind::ImplicitParameter, colon + TextSize::from(1), ¶meters, true, @@ -1898,7 +2042,7 @@ impl<'a, 'db> InlayHintVisitor<'a, 'db> { /// Hint the inferred type of an unannotated lambda parameter. fn add_lambda_parameter_type(&mut self, parameter: &ast::Parameter) { let env = &self.model.program_environment(); - if !self.settings.lambda_parameter_types + if !self.settings.enabled(InlayHintKind::LambdaParameterType) || !self.in_lambda || parameter.annotation.is_some() || parameter.range().is_empty() @@ -1910,9 +2054,10 @@ impl<'a, 'db> InlayHintVisitor<'a, 'db> { return; }; - self.hints.push(InlayHint::inferred_annotation( + self.push(InlayHint::inferred_annotation( self.db, env, + InlayHintKind::LambdaParameterType, parameter.name.range().end(), ty, )); @@ -1925,7 +2070,7 @@ impl<'a, 'db> InlayHintVisitor<'a, 'db> { /// under their own setting. fn add_inherited_parameter_type(&mut self, parameter: &ast::Parameter) { let env = &self.model.program_environment(); - if !self.settings.inherited_parameter_types + if !self.settings.enabled(InlayHintKind::InheritedParameterType) || self.in_lambda || parameter.annotation.is_some() || parameter.range().is_empty() @@ -1937,9 +2082,10 @@ impl<'a, 'db> InlayHintVisitor<'a, 'db> { return; }; - self.hints.push(InlayHint::inferred_annotation( + self.push(InlayHint::inferred_annotation( self.db, env, + InlayHintKind::InheritedParameterType, parameter.name.range().end(), ty, )); @@ -1948,7 +2094,7 @@ impl<'a, 'db> InlayHintVisitor<'a, 'db> { /// basedpython: hint the default a parameter takes from the method its `def` /// overrides, where the parameter's own default would be written. fn add_inherited_parameter_default(&mut self, parameter: &ast::ParameterWithDefault) { - if !self.settings.inherited_parameter_defaults + if !self.settings.enabled(InlayHintKind::InheritedDefault) || self.in_lambda || parameter.parameter.range().is_empty() { @@ -1959,7 +2105,7 @@ impl<'a, 'db> InlayHintVisitor<'a, 'db> { return; }; - self.hints.push(InlayHint::inherited_default( + self.push(InlayHint::inherited_default( parameter.parameter.range().end(), &value, parameter.parameter.annotation.is_some(), @@ -1974,7 +2120,7 @@ impl<'a, 'db> InlayHintVisitor<'a, 'db> { /// `def f() -> int raises TypeError`. fn add_inferred_return(&mut self, function: &ast::StmtFunctionDef) { let env = &self.model.program_environment(); - if !self.settings.inferred_return_types + if !self.settings.enabled(InlayHintKind::InferredReturnType) || function.returns.is_some() || function.is_asserts_return { @@ -1988,7 +2134,7 @@ impl<'a, 'db> InlayHintVisitor<'a, 'db> { return; }; - self.hints.push(InlayHint::inferred_return( + self.push(InlayHint::inferred_return( self.db, env, function.parameters.end(), @@ -2002,7 +2148,7 @@ impl<'a, 'db> InlayHintVisitor<'a, 'db> { /// so the type goes where the declaration would have written it — after the /// name, exactly as it does for an unannotated variable fn add_property_type(&mut self, getter: &ast::StmtFunctionDef) { - if !self.settings.property_types + if !self.settings.enabled(InlayHintKind::PropertyType) || !self.is_basedpython() || getter.property_construct_range().is_none() // the declaration named a type; the getter carries it as its return @@ -2020,9 +2166,10 @@ impl<'a, 'db> InlayHintVisitor<'a, 'db> { }; let env = &self.model.program_environment(); - self.hints.push(InlayHint::inferred_annotation( + self.push(InlayHint::inferred_annotation( self.db, env, + InlayHintKind::PropertyType, getter.name.range().end(), ty, )); @@ -2041,7 +2188,7 @@ impl<'a, 'db> InlayHintVisitor<'a, 'db> { name: &str, navigation_target: Option, ) -> bool { - if !self.settings.call_argument_names { + if !self.settings.enabled(InlayHintKind::CallArgumentName) { return false; } @@ -2051,8 +2198,7 @@ impl<'a, 'db> InlayHintVisitor<'a, 'db> { let inlay_hint = InlayHint::call_argument_name(position, name, navigation_target); - self.hints.push(inlay_hint); - true + self.push(inlay_hint) } } @@ -2722,6 +2868,8 @@ mod tests { use ruff_python_parser::parse_unchecked_source; use ruff_python_trivia::textwrap::dedent; use ruff_text_size::{TextLen, TextSize}; + use rustc_hash::FxHashSet; + use strum::IntoEnumIterator; use ruff_db::system::{DbWithWritableSystem, SystemPathBuf}; use ty_project::ProjectMetadata; @@ -3043,6 +3191,119 @@ Source with applied edits: "); } + /// every kind of hint is shown with only its own setting on, and nothing + /// else is. a hint is only ever added when its kind's setting is on, so this + /// is what catches a hint whose work is skipped by the wrong setting before + /// it is ever made + #[test] + fn every_kind_is_switched_by_the_setting_it_names() { + fn only(kind: InlayHintKind) -> InlayHintSettings { + let mut settings = InlayHintSettings::none(); + settings.set_enabled(kind, true); + settings + } + + let mut by = basedpython_ui_inlay_hint_test( + r#" + from basedpython_ui import composable, derived, state, Button, Text + + def identity[T](x: T) -> T: + return x + + def two(a: int, b: str) -> None: ... + + x = identity(1) + two(1, 'b') + y = map(lambda v: v + 1, [1]) + z: dict[str, int] = {} + + def raiser(): + raise TypeError + + class Box[T]: + def get(self) -> T: ... + + def reify[T](): + print(T) + + class Base: + def f(self, a: int = 1) -> None: ... + + class Child(Base): + def f(self, a) -> None: ... + + reveal_type(x) + + def apply(fn: (int) -> None) -> None: + fn(1) + + apply: + print(it) + + class Init: + init(a: int) + + class Prop: + let a + get() = 1 + + def ctx(context c: int) -> None: ... + context cc = 1 + ctx() + + enum class Color: + case Red, Green + + @composable + def Counter(items: list[int]): + let count = state(0) + let doubled = derived(lambda: count.value * 2) + Text(f"{doubled.value}") + Button("+"): + count.value += 1 + "#, + ); + let mut python = inlay_hint_test( + " + def f(x: float | None) -> None: + reveal_type(x) + ", + ); + + let mut produced = FxHashSet::default(); + for kind in InlayHintKind::iter() { + let settings = only(kind); + for test in [&mut by, &mut python] { + let hints = inlay_hints( + &test.db, + ProgramFile::new( + &test.db, + test.file, + test.db.program_environment().program(&test.db), + ), + test.range, + &settings, + ); + for hint in hints { + assert_eq!( + hint.kind, + kind, + "`{}` produced a {:?} hint", + kind.setting(), + hint.kind, + ); + produced.insert(kind); + } + } + } + + let missing: Vec<_> = InlayHintKind::iter() + .filter(|kind| !produced.contains(kind)) + .map(InlayHintKind::setting) + .collect(); + assert!(missing.is_empty(), "no source produced {missing:?}"); + } + /// The same, for the hints a `.py` file gets: one shown at the end of a line /// and one shown between the operands of a union. #[test] @@ -10983,6 +11244,24 @@ Source with applied edits: })); } + #[test] + fn basedpython_inferred_override_of_a_private_member() { + let mut test = basedpython_inlay_hint_test( + " + class A: + private def f(self) -> None: ... + + class B(A): + private def f(self) -> None: ... + ", + ); + + assert_snapshot!(test.inlay_hints_with_settings(&InlayHintSettings { + inferred_override: true, + ..InlayHintSettings::none() + })); + } + /// The mock of the `basedpython_ui` framework the ui hints are tested /// against: its observables, slot functions and the builders the examples /// use, developed in place as a first-party package — which is how the @@ -12095,6 +12374,26 @@ def each(fn: (int) -> None) -> None: })); } + /// with `typeArgumentNames` off, the types other hints show leave their type + /// arguments unnamed, while those hints are still shown + #[test] + fn basedpython_type_arguments_unnamed_with_the_setting_off() { + let mut test = basedpython_inlay_hint_test( + " + class Pair[Key, Value]: + init(key: Key, value: Value) + + a = Pair(1, 'x') + ", + ); + + assert_snapshot!(test.inlay_hints_with_settings(&InlayHintSettings { + variable_types: true, + call_type_arguments: true, + ..InlayHintSettings::none() + })); + } + /// A `some` parameter opens an anonymous hole, which is not a position a call /// site can supply, so there is no specialization to offer for it. #[test] diff --git a/crates/ty_ide/src/lib.rs b/crates/ty_ide/src/lib.rs index 27b2adc4be..9964c2ed86 100644 --- a/crates/ty_ide/src/lib.rs +++ b/crates/ty_ide/src/lib.rs @@ -35,6 +35,7 @@ mod selection_range; mod semantic_tokens; mod signature_help; mod stub_mapping; +mod super_members; mod symbols; mod syntax_outline; mod type_hierarchy; @@ -80,13 +81,14 @@ pub use module_rename::{ }; pub use refactor::{Refactor, RefactorKind, RefactorOffer, refactor, refactors}; pub use references::ReferencesMode; -pub use rename::{can_rename, rename}; +pub use rename::{PreparedRename, can_rename, invalid_new_name, prepare_rename, rename}; pub use selection_range::selection_range; pub use semantic_tokens::{ Fragments, SemanticToken, SemanticTokenModifier, SemanticTokenType, SemanticTokens, semantic_tokens, }; pub use signature_help::{ParameterDetails, SignatureDetails, SignatureHelpInfo, signature_help}; +pub use super_members::{OverridingMember, SuperMember, document_super_members, super_members}; pub use symbols::{FlatSymbols, HierarchicalSymbols, SymbolId, SymbolInfo, SymbolKind}; pub use syntax_outline::{ OutlineCall, OutlineClause, OutlineModifier, OutlineStatement, OutlineString, SyntaxOutline, diff --git a/crates/ty_ide/src/rename.rs b/crates/ty_ide/src/rename.rs index d3923482cf..fe514b7d5f 100644 --- a/crates/ty_ide/src/rename.rs +++ b/crates/ty_ide/src/rename.rs @@ -1,55 +1,148 @@ -use crate::goto::find_goto_target; +use crate::goto::{GotoTarget, find_goto_target}; use crate::references::{ReferencesMode, references}; use crate::{Db, ReferenceTarget}; use ruff_db::files::File; -use ruff_text_size::{Ranged, TextSize}; +use ruff_python_ast as ast; +use ruff_python_stdlib::identifiers::is_identifier; +use ruff_python_stdlib::keyword::is_keyword; +use ruff_text_size::{Ranged, TextRange, TextSize}; use ty_python_core::ProgramFile; -use ty_python_semantic::SemanticModel; +use ty_python_semantic::{ResolvedDefinition, SemanticModel}; /// Returns the range of the symbol if it can be renamed, None if not. -pub fn can_rename( - db: &dyn Db, - file: ProgramFile<'_>, - offset: TextSize, -) -> Option { +pub fn can_rename(db: &dyn Db, file: ProgramFile<'_>, offset: TextSize) -> Option { + match prepare_rename(db, file, offset) { + PreparedRename::Ready { range, .. } => Some(range), + PreparedRename::Refused(_) | PreparedRename::NoSymbol => None, + } +} + +/// what a rename at an offset would be, worked out before an editor offers one +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum PreparedRename { + /// the name the editor offers to rename + Ready { + /// what the rename replaces + range: TextRange, + /// the name as the editor offers it for editing, which is not always the + /// text at `range`: a `.by` keyword argument named with a string, as in + /// `f("timeout"=1)`, is offered as `timeout`, and the rename replaces the + /// quotes along with it + placeholder: String, + }, + /// there is a name here, and this is why it cannot be renamed + /// + /// said rather than kept quiet: an editor asked to rename `print` that + /// hears nothing back does nothing, and leaves the user to guess whether + /// the key was pressed at all + Refused(String), + /// there is nothing here a rename could be about: whitespace, punctuation, + /// a literal + NoSymbol, +} + +/// whether the symbol at `offset` can be renamed, and if it cannot, why not +pub fn prepare_rename(db: &dyn Db, file: ProgramFile<'_>, offset: TextSize) -> PreparedRename { let parsed = ruff_db::parsed::parsed_module(db, file.python_file(db)); let module = parsed.load(db); let source_file = file.file(db); let model = SemanticModel::new(db, file); // Get the definitions for the symbol at the offset - let goto_target = find_goto_target(&model, &module, offset)?; + let Some(goto_target) = find_goto_target(&model, &module, offset) else { + return PreparedRename::NoSymbol; + }; + + // `None`, `True` and `False` are goto targets like any name, but nothing + // declares them that a rename could reach + if let GotoTarget::Expression(expression) + | GotoTarget::Call { + callable: expression, + .. + } = &goto_target + && let Some(keyword) = keyword_literal(*expression) + { + return PreparedRename::Refused(format!("`{keyword}` is a keyword")); + } + + // this is the name `references` searches for, so a target without one is + // nothing a rename could be about: a literal, an operator. a string may yet + // be one of the django names a module writes as plain text, which is + // answered once this declines + let Some(name) = goto_target.to_string() else { + return PreparedRename::NoSymbol; + }; // Don't allow renaming of import module components - if matches!( - goto_target, - crate::goto::GotoTarget::ImportModuleComponent { .. } - ) { - return None; + if matches!(goto_target, GotoTarget::ImportModuleComponent { .. }) { + return PreparedRename::Refused(format!( + "`{name}` names a module, which is renamed by renaming its file" + )); } let current_file_in_project = is_file_in_project(db, source_file); - let declaration_targets = goto_target - .definitions(&model, ReferencesMode::Rename.to_import_alias_resolution())? - .goto_declaration(&model, &goto_target)? - .into_navigation_targets(model.db()); + let Some(declarations) = goto_target + .definitions(&model, ReferencesMode::Rename.to_import_alias_resolution()) + .and_then(|definitions| definitions.goto_declaration(&model, &goto_target)) + else { + // with no declaration there is nothing to rename, and renaming every + // other use of the same spelling would be a guess + return PreparedRename::Refused(format!("cannot find where `{name}` is declared")); + }; - for target in &declaration_targets { - let target_file = target.file(); + for declaration in &declarations { + let target_file = match declaration { + ResolvedDefinition::Definition(definition) => definition.file(db), + ResolvedDefinition::Module(module) => module.file(db), + ResolvedDefinition::FileWithRange(range) => range.file(), + }; // If definition is outside the project, refuse rename if !is_file_in_project(db, target_file) { - return None; + return PreparedRename::Refused(match declaration { + ResolvedDefinition::Module(_) => { + format!("`{name}` names a module outside this project") + } + ResolvedDefinition::Definition(_) | ResolvedDefinition::FileWithRange(_) => { + format!("`{name}` is declared outside this project") + } + }); } // If current file is not in project and any definition is outside current file, refuse rename if !current_file_in_project && target_file != source_file { - return None; + return PreparedRename::Refused(format!( + "`{name}` is declared in another file, and this file is not part of the project" + )); } } - Some(goto_target.range()) + PreparedRename::Ready { + range: goto_target.range(), + placeholder: name.into_owned(), + } +} + +/// why a symbol cannot be renamed to `new_name`, if it cannot: a name that is +/// not an identifier, or is a keyword, would leave the code unable to parse +pub fn invalid_new_name(new_name: &str) -> Option { + if is_keyword(new_name) { + Some(format!("`{new_name}` is a keyword")) + } else if is_identifier(new_name) { + None + } else { + Some(format!("`{new_name}` is not a valid name")) + } +} + +/// the keyword a `None`, `True` or `False` literal is spelled with +fn keyword_literal(expression: ast::ExprRef<'_>) -> Option<&'static str> { + match expression { + ast::ExprRef::NoneLiteral(_) => Some("None"), + ast::ExprRef::BooleanLiteral(literal) => Some(if literal.value { "True" } else { "False" }), + _ => None, + } } /// Perform a rename operation on the symbol at the given position. @@ -101,21 +194,27 @@ mod tests { use insta::assert_snapshot; use ruff_db::diagnostic::{Annotation, Diagnostic, DiagnosticId, LintName, Severity, Span}; use ruff_db::files::FileRange; + use ruff_ranged_value::ValueSource; use ruff_text_size::Ranged; + use ty_project::metadata::options::Options; impl CursorTest { fn prepare_rename(&self) -> String { - let Some(range) = salsa::attach(&self.db, || { - can_rename( + let prepared = salsa::attach(&self.db, || { + prepare_rename( &self.db, self.program_file(self.cursor.file), self.cursor.offset, ) - }) else { - return "Cannot rename".to_string(); - }; + }); - format!("Can rename symbol at range {range:?}") + match prepared { + PreparedRename::Ready { range, placeholder } => { + format!("Can rename symbol `{placeholder}` at range {range:?}") + } + PreparedRename::Refused(why) => format!("Cannot rename: {why}"), + PreparedRename::NoSymbol => "Cannot rename".to_string(), + } } fn rename(&self, new_name: &str) -> String { @@ -352,7 +451,7 @@ value = 0 ", ); - assert_snapshot!(test.prepare_rename(), @"Can rename symbol at range 10..15"); + assert_snapshot!(test.prepare_rename(), @"Can rename symbol `value` at range 10..15"); } #[test] @@ -1051,7 +1150,7 @@ x = os.path.join('a', 'b') ", ); - assert_snapshot!(test.prepare_rename(), @"Cannot rename"); + assert_snapshot!(test.prepare_rename(), @"Cannot rename: `os` names a module, which is renamed by renaming its file"); } #[test] @@ -1064,7 +1163,7 @@ result = join('a', 'b') ", ); - assert_snapshot!(test.prepare_rename(), @"Cannot rename"); + assert_snapshot!(test.prepare_rename(), @"Cannot rename: `path` names a module, which is renamed by renaming its file"); } #[test] @@ -1078,7 +1177,7 @@ x = os.path.join('a', 'b') ", ); - assert_snapshot!(test.prepare_rename(), @"Cannot rename"); + assert_snapshot!(test.prepare_rename(), @"Cannot rename: `os` names a module outside this project"); } #[test] @@ -1267,7 +1366,7 @@ def process_value(value): ", ); - assert_snapshot!(test.prepare_rename(), @"Cannot rename"); + assert_snapshot!(test.prepare_rename(), @"Cannot rename: `None` is a keyword"); } #[test] @@ -1280,7 +1379,7 @@ def convert_to_number(value): ", ); - assert_snapshot!(test.prepare_rename(), @"Cannot rename"); + assert_snapshot!(test.prepare_rename(), @"Cannot rename: `int` is declared outside this project"); } /// basedpython: a name that means a `typing` member without an import is a @@ -1292,7 +1391,51 @@ def convert_to_number(value): .source("main.by", "a: Mapping\n") .build(); - assert_snapshot!(test.prepare_rename(), @"Cannot rename"); + assert_snapshot!(test.prepare_rename(), @"Cannot rename: `Mapping` is declared outside this project"); + } + + /// a file the project leaves out is renamed in on its own, so a name it takes + /// from another file cannot be renamed from it + #[test] + fn cannot_rename_from_a_file_outside_the_project() { + let test = CursorTest::builder() + .options( + Options::from_toml_str("[src]\nexclude = [\"main.py\"]\n", ValueSource::Cli) + .expect("valid options"), + ) + .source("lib.py", "def f(): ...\n") + .source("main.py", "from lib import f\n\nf()\n") + .build(); + + assert_snapshot!(test.prepare_rename(), @"Cannot rename: `f` is declared in another file, and this file is not part of the project"); + } + + /// basedpython: a keyword argument may be named with a string, and that name is + /// renamed like any other keyword argument's, quotes and all + #[test] + fn rename_quoted_keyword_argument() { + let test = CursorTest::builder() + .source( + "main.by", + " +def f(timeout: int): ... + +f(\"timeout\"=1) +", + ) + .build(); + + assert_snapshot!(test.prepare_rename(), @"Can rename symbol `timeout` at range 29..38"); + assert_snapshot!(test.rename("retries"), @r#" + info[rename]: Rename symbol (found 2 locations) + --> main.by:2:7 + | + 2 | def f(timeout: int): ... + | ^^^^^^^ + 3 | + 4 | f("timeout"=1) + | --------- + "#); } #[test] @@ -1304,7 +1447,7 @@ def convert_to_number(value): .source("main.py", "_T_co\n") .build(); - assert_snapshot!(test.prepare_rename(), @"Cannot rename"); + assert_snapshot!(test.prepare_rename(), @"Cannot rename: cannot find where `_T_co` is declared"); } #[test] diff --git a/crates/ty_ide/src/snapshots/ty_ide__inlay_hints__tests__basedpython_inferred_override_of_a_private_member.snap b/crates/ty_ide/src/snapshots/ty_ide__inlay_hints__tests__basedpython_inferred_override_of_a_private_member.snap new file mode 100644 index 0000000000..c4e63c82d3 --- /dev/null +++ b/crates/ty_ide/src/snapshots/ty_ide__inlay_hints__tests__basedpython_inferred_override_of_a_private_member.snap @@ -0,0 +1,10 @@ +--- +source: crates/ty_ide/src/inlay_hints.rs +expression: "test.inlay_hints_with_settings(&InlayHintSettings\n{ inferred_override: true, ..InlayHintSettings::none() })" +--- + +class A: + private def f(self) -> None: ... + +class B(A): + private def f(self) -> None: ... diff --git a/crates/ty_ide/src/snapshots/ty_ide__inlay_hints__tests__basedpython_type_arguments_unnamed_with_the_setting_off.snap b/crates/ty_ide/src/snapshots/ty_ide__inlay_hints__tests__basedpython_type_arguments_unnamed_with_the_setting_off.snap new file mode 100644 index 0000000000..011f4556d6 --- /dev/null +++ b/crates/ty_ide/src/snapshots/ty_ide__inlay_hints__tests__basedpython_type_arguments_unnamed_with_the_setting_off.snap @@ -0,0 +1,74 @@ +--- +source: crates/ty_ide/src/inlay_hints.rs +expression: "test.inlay_hints_with_settings(&InlayHintSettings\n{\n variable_types: true, call_type_arguments: true,\n ..InlayHintSettings::none()\n})" +--- + +class Pair[Key, Value]: + init(key: Key, value: Value) + +a[: Pair[1, "x"]] = Pair[[1, "x"]](1, 'x') + +--------------------------------------------- +info[inlay-hint-location]: Inlay Hint Target + --> main.by:2:7 + | +2 | class Pair[Key, Value]: + | ^^^^ +info: Source + --> main2.py:5:5 + | +5 | a[: Pair[1, "x"]] = Pair[[1, "x"]](1, 'x') + | ^^^^ + +info[inlay-hint-location]: Inlay Hint Target + --> stdlib/builtins.byi:LL:7 + | +LL | class int: + | ^^^ +info: Source + --> main2.py:LL:10 + | +LL | a[: Pair[1, "x"]] = Pair[[1, "x"]](1, 'x') + | ^ + +info[inlay-hint-location]: Inlay Hint Target + --> stdlib/builtins.byi:LL:7 + | +LL | class str(Sequence[str]): + | ^^^ +info: Source + --> main2.py:LL:13 + | +LL | a[: Pair[1, "x"]] = Pair[[1, "x"]](1, 'x') + | ^^^ + +info[inlay-hint-location]: Inlay Hint Target + --> stdlib/builtins.byi:LL:7 + | +LL | class int: + | ^^^ +info: Source + --> main2.py:LL:27 + | +LL | a[: Pair[1, "x"]] = Pair[[1, "x"]](1, 'x') + | ^ + +info[inlay-hint-location]: Inlay Hint Target + --> stdlib/builtins.byi:LL:7 + | +LL | class str(Sequence[str]): + | ^^^ +info: Source + --> main2.py:LL:30 + | +LL | a[: Pair[1, "x"]] = Pair[[1, "x"]](1, 'x') + | ^^^ + +--------------------------------------------- +info[inlay-hint-edit]: Inlay hint edits +--> main.by:1:1 + | +4 | + - a = Pair(1, 'x') +5 + a: Pair[1, "x"] = Pair(1, 'x') + | diff --git a/crates/ty_ide/src/super_members.rs b/crates/ty_ide/src/super_members.rs new file mode 100644 index 0000000000..30324f17c4 --- /dev/null +++ b/crates/ty_ide/src/super_members.rs @@ -0,0 +1,869 @@ +//! finds the superclass members a class member overrides: the direction +//! [`crate::goto_implementation()`] does not go, which finds the members that +//! override one +//! +//! what counts as an override is the override checks' own walk up the MRO, +//! through [`overridden_members`]. of the declarations that walk finds, a member +//! goes to the nearest along each branch of the MRO: the first of them is what +//! `missing-override-decorator` names, and `invalid-method-override` compares +//! the member with each declaration the walk finds, these and the ones they +//! override in turn +//! +//! asked two ways, from the same code: of one member, at a position on its name +//! ([`super_members`]), which is what *Go to Super* asks; and of every member a +//! document declares at once ([`document_super_members`]), which is what an +//! editor drawing an "overrides" marker beside each one asks, on every pass over +//! the document + +use ruff_python_ast::name::Name; +use ruff_text_size::{TextRange, TextSize}; +use ty_python_core::ProgramFile; +use ty_python_semantic::types::Type; +use ty_python_semantic::types::ide_support::{ + ClassMemberDefinition, OverriddenMember, class_member_definitions, is_constructor_like_method, + overridden_members, +}; +use ty_python_semantic::{ProgramEnvironment, SemanticModel}; + +use crate::goto::Definitions; +use crate::{Db, HasNavigationTargets, NavigationTarget}; + +/// a superclass member that a class member overrides +#[derive(Debug, Clone)] +pub struct SuperMember { + /// the member's name + pub name: Name, + /// the superclass that declares it + pub superclass: Name, + /// where the superclass declares it, or, for a member the superclass + /// synthesizes rather than writes, the superclass itself + pub target: NavigationTarget, + /// whether the superclass synthesizes the member, as a dataclass does its + /// `__init__`, so that [`Self::target`] is the class + pub synthesized: bool, + /// whether the member is abstract where the superclass declares it: an + /// `@abstractmethod`, or a protocol method with no implementation + pub is_abstract: bool, +} + +/// a class member a document declares that overrides something, and what it +/// overrides +#[derive(Debug, Clone)] +pub struct OverridingMember { + /// the member's name + pub name: Name, + /// the class whose body declares it + pub class: Name, + /// its name where it is declared: the name of a `def` or a nested class, a + /// name a class body assigns, annotates or captures, or an import's alias + pub name_range: TextRange, + /// whether the member is itself abstract, in the sense of + /// [`SuperMember::is_abstract`] + pub is_abstract: bool, + /// what it overrides directly, in MRO order. never empty + pub super_members: Vec, +} + +/// the superclass members the class member declared at `offset` overrides +/// directly, in MRO order +/// +/// `offset` is on the member's name where it is declared, as +/// [`OverridingMember::name_range`] has it. `None` when there is no such member +/// there; empty when there is one and it overrides nothing +pub fn super_members( + db: &dyn Db, + file: ProgramFile<'_>, + offset: TextSize, +) -> Option> { + let declared: Vec<_> = class_member_definitions(db, file) + .into_iter() + .filter(|member| { + !member.name_range.is_empty() && member.name_range.contains_inclusive(offset) + }) + .collect(); + if declared.is_empty() { + return None; + } + let env = SemanticModel::new(db, file).program_environment(); + Some(overriding_member(db, &env, &declared)?.super_members) +} + +/// every class member `file` declares that overrides something, with what it +/// overrides, in source order: [`super_members`] of each, from one pass over the +/// document +/// +/// a member declared more than once is listed at each declaration that names it +/// in the source, as each is answered: a python property's getter and its setter +/// each override their own accessor. a declaration that stands for several, as a +/// basedpython property with `get` and `set` blocks stands for its getter and its +/// setter under the one name, is listed once, with what any of them overrides +/// +/// constructors and the methods they call (`__init__`, `__new__`, +/// `__post_init__`, `__init_subclass__`) are left out: every class's `__init__` +/// overrides `object.__init__`, which says nothing about any of them, and +/// neither `invalid-method-override` nor `missing-override-decorator` holds one +/// to what it overrides. [`super_members`] still answers one when asked +pub fn document_super_members(db: &dyn Db, file: ProgramFile<'_>) -> Vec { + let env = SemanticModel::new(db, file).program_environment(); + class_member_definitions(db, file) + // the definitions come in source order, so the ones naming the same text + // are next to each other + .chunk_by(|left, right| left.name_range == right.name_range) + .filter(|declared| { + declared.first().is_some_and(|member| { + !member.name_range.is_empty() && !is_constructor_like_method(&member.name) + }) + }) + .filter_map(|declared| overriding_member(db, &env, declared)) + .filter(|member| !member.super_members.is_empty()) + .collect() +} + +/// what the definitions `declared`, which all name the same member at the same +/// place in the source, override between them. `None` when `declared` is empty +fn overriding_member( + db: &dyn Db, + env: &ProgramEnvironment<'_>, + declared: &[ClassMemberDefinition<'_>], +) -> Option { + let first = declared.first()?; + let mut is_abstract = false; + let mut overridden: Vec> = Vec::new(); + for member in declared { + let overrides = overridden_members(db, member); + is_abstract |= overrides.is_abstract; + for member in overrides.overridden { + if !overridden + .iter() + .any(|found| found.superclass == member.superclass) + { + overridden.push(member); + } + } + } + Some(OverridingMember { + name: first.name.clone(), + class: first.class_name.clone(), + name_range: first.name_range, + is_abstract, + super_members: targets(db, env, &first.name, overridden), + }) +} + +/// where each of `overridden`, the members named `name` a member overrides, is +/// declared, or the superclass itself for one with no declaration to go to. one +/// with neither is left out +fn targets( + db: &dyn Db, + env: &ProgramEnvironment<'_>, + name: &Name, + overridden: Vec>, +) -> Vec { + overridden + .into_iter() + .filter_map(|member| { + let target = Definitions::new(member.definitions) + .into_navigation_targets(db) + .into_iter() + .next() + .or_else(|| { + Type::ClassLiteral(member.superclass) + .navigation_targets(db, env) + .into_iter() + .next() + })?; + Some(SuperMember { + name: name.clone(), + superclass: member.superclass_name, + target, + synthesized: member.synthesized, + is_abstract: member.is_abstract, + }) + }) + .collect() +} + +#[cfg(test)] +mod tests { + use crate::tests::{CursorTest, cursor_test}; + use insta::assert_snapshot; + use ruff_db::source::source_text; + use ruff_source_file::LineIndex; + + impl CursorTest { + /// one line per member: `Superclass.name` and the line it is declared + /// on, `(synthesized)` when the superclass synthesizes it, `(abstract)` + /// when the member is abstract there + fn super_members(&self) -> String { + let Some(members) = super::super_members( + &self.db, + self.program_file(self.cursor.file), + self.cursor.offset, + ) else { + return "None".to_string(); + }; + if members.is_empty() { + return "[]".to_string(); + } + members + .iter() + .map(|member| self.super_member(member)) + .collect::>() + .join("\n") + } + + fn super_member(&self, member: &super::SuperMember) -> String { + let file = member.target.file(); + let source = source_text(&self.db, file); + let line = LineIndex::from_source_text(&source) + .line_index(member.target.focus_range().start()); + format!( + "{}.{} {}:{}{}{}", + member.superclass, + member.name, + file.path(&self.db).as_str(), + line, + if member.synthesized { + " (synthesized)" + } else { + "" + }, + if member.is_abstract { + " (abstract)" + } else { + "" + }, + ) + } + + /// one line per overriding member of the cursor's file: its name, the + /// line and column its name is on, `(abstract)` when it is abstract + /// itself, and after `->` what it overrides, as [`Self::super_members`] + /// writes each + fn document_super_members(&self) -> String { + let file = self.cursor.file; + let source = source_text(&self.db, file); + let index = LineIndex::from_source_text(&source); + let members = + super::document_super_members(&self.db, self.program_file(self.cursor.file)); + if members.is_empty() { + return "[]".to_string(); + } + members + .iter() + .map(|member| { + let start = member.name_range.start(); + let line = index.line_index(start); + let column = start - index.line_start(line, &source); + format!( + "{}.{} {}:{}{} -> {}", + member.class, + member.name, + line, + column.to_u32(), + if member.is_abstract { + " (abstract)" + } else { + "" + }, + member + .super_members + .iter() + .map(|member| self.super_member(member)) + .collect::>() + .join(", "), + ) + }) + .collect::>() + .join("\n") + } + } + + #[test] + fn an_abstract_method_is_said_to_be_abstract() { + let test = cursor_test( + " +from abc import ABC, abstractmethod + +class A(ABC): + @abstractmethod + def f(self) -> None: ... + +class B(A): + def f(self) -> None: + pass +", + ); + assert_snapshot!(test.super_members(), @"A.f /main.py:6 (abstract)"); + } + + #[test] + fn a_document_lists_each_member_that_overrides_something() { + let test = cursor_test( + " +from typing import Protocol + +class Greeter(Protocol): + def greet(self) -> str: ... + +class A: + size: int = 1 + + def f(self) -> None: ... + + @property + def x(self) -> int: + return 1 + + @x.setter + def x(self, value: int) -> None: ... + +class B(A): + size = 2 + + def __init__(self) -> None: ... + + def f(self) -> None: ... + + def g(self) -> None: ... + + @property + def x(self) -> int: + return 2 + + @x.setter + def x(self, value: int) -> None: ... + +class C(B, Greeter): + def f(self) -> None: + f = 1 + class Inner(A): + def f(self) -> None: ... + + def greet(self) -> str: + return 'hi' + + names = [f for f in 'ab'] +", + ); + assert_snapshot!(test.document_super_members(), @" + B.size 20:4 -> A.size /main.py:8 + B.f 24:8 -> A.f /main.py:10 + B.x 29:8 -> A.x /main.py:13 + B.x 33:8 -> A.x /main.py:17 + C.f 36:8 -> B.f /main.py:24 + Inner.f 39:16 -> A.f /main.py:10 + C.greet 41:8 -> Greeter.greet /main.py:5 (abstract) + "); + } + + #[test] + fn a_document_member_that_overrides_through_two_bases_lists_both() { + let test = cursor_test( + " +class Root: + def f(self) -> None: ... + +class Left(Root): + def f(self) -> None: ... + +class Right(Root): + def f(self) -> None: ... + +class Both(Left, Right): + def f(self) -> None: ... +", + ); + assert_snapshot!(test.document_super_members(), @" + Left.f 6:8 -> Root.f /main.py:3 + Right.f 9:8 -> Root.f /main.py:3 + Both.f 12:8 -> Left.f /main.py:6, Right.f /main.py:9 + "); + } + + #[test] + fn a_protocol_member_redeclared_in_a_protocol_is_abstract_itself() { + let test = cursor_test( + " +from typing import Protocol + +class Shape(Protocol): + def area(self) -> float: ... + +class Polygon(Shape, Protocol): + def area(self) -> float: ... +", + ); + assert_snapshot!( + test.document_super_members(), + @"Polygon.area 8:8 (abstract) -> Shape.area /main.py:5 (abstract)" + ); + } + + #[test] + fn a_document_with_no_overrides_is_empty() { + let test = cursor_test( + " +class A: + def __init__(self) -> None: ... + + def f(self) -> None: ... + +def g() -> None: ... +", + ); + assert_snapshot!(test.document_super_members(), @"[]"); + } + + #[test] + fn a_method_goes_to_the_method_it_overrides() { + let test = cursor_test( + " +class A: + def f(self) -> None: ... + +class B(A): + def f(self) -> None: ... +", + ); + assert_snapshot!(test.super_members(), @"A.f /main.py:3"); + } + + #[test] + fn two_levels_up_goes_to_the_nearer() { + let test = cursor_test( + " +class A: + def f(self) -> None: ... + +class B(A): + def f(self) -> None: ... + +class C(B): + def f(self) -> None: ... +", + ); + assert_snapshot!(test.super_members(), @"B.f /main.py:6"); + } + + #[test] + fn two_levels_up_past_a_class_that_does_not_declare_it() { + let test = cursor_test( + " +class A: + def f(self) -> None: ... + +class B(A): + pass + +class C(B): + def f(self) -> None: ... +", + ); + assert_snapshot!(test.super_members(), @"A.f /main.py:3"); + } + + #[test] + fn several_bases_that_each_declare_it_are_each_overridden_in_mro_order() { + let test = cursor_test( + " +class Root: + def f(self) -> None: ... + +class Left(Root): + def f(self) -> None: ... + +class Right(Root): + def f(self) -> None: ... + +class Both(Left, Right): + def f(self) -> None: ... +", + ); + assert_snapshot!(test.super_members(), @" + Left.f /main.py:6 + Right.f /main.py:9 + "); + } + + #[test] + fn a_diamond_whose_sides_do_not_declare_it_goes_to_the_root_once() { + let test = cursor_test( + " +class Root: + def f(self) -> None: ... + +class Left(Root): ... + +class Right(Root): ... + +class Both(Left, Right): + def f(self) -> None: ... +", + ); + assert_snapshot!(test.super_members(), @"Root.f /main.py:3"); + } + + #[test] + fn a_protocol_member() { + let test = cursor_test( + " +from typing import Protocol + +class Greeter(Protocol): + def greet(self) -> str: ... + +class English(Greeter): + def greet(self) -> str: + return 'hello' +", + ); + assert_snapshot!(test.super_members(), @"Greeter.greet /main.py:5 (abstract)"); + } + + #[test] + fn a_property_getter_goes_to_the_getter() { + let test = cursor_test( + " +class A: + @property + def x(self) -> int: + return 1 + + @x.setter + def x(self, value: int) -> None: ... + +class B(A): + @property + def x(self) -> int: + return 2 +", + ); + assert_snapshot!(test.super_members(), @"A.x /main.py:4"); + } + + #[test] + fn a_class_attribute() { + let test = cursor_test( + " +class A: + size: int = 1 + +class B(A): + size = 2 +", + ); + assert_snapshot!(test.super_members(), @"A.size /main.py:3"); + } + + #[test] + fn a_member_the_superclass_synthesizes_goes_to_the_superclass() { + let test = cursor_test( + " +from dataclasses import dataclass + +@dataclass +class A: + x: int + +class B(A): + def __init__(self) -> None: + super().__init__(1) +", + ); + assert_snapshot!(test.super_members(), @"A.__init__ /main.py:5 (synthesized)"); + } + + #[test] + fn a_method_that_overrides_nothing() { + let test = cursor_test( + " +class A: + def f(self) -> None: ... + +class B(A): + def g(self) -> None: ... +", + ); + assert_snapshot!(test.super_members(), @"[]"); + } + + #[test] + fn a_function_that_is_not_a_member() { + let test = cursor_test( + " +def f() -> None: ... +", + ); + assert_snapshot!(test.super_members(), @"None"); + } + + #[test] + fn basedpython_override_and_protocol() { + let test = CursorTest::builder() + .source( + "main.by", + " +protocol Shape: + def area(self) -> float: ... + +class Square(Shape): + def __init__(self, side: float) -> None: + self.side = side + + override def area(self) -> float: + return self.side * self.side +", + ) + .build(); + assert_snapshot!(test.super_members(), @"Shape.area /main.by:3 (abstract)"); + } + + #[test] + fn a_private_member_overrides_nothing() { + let test = CursorTest::builder() + .source( + "main.by", + " +class A: + private def f(self) -> None: ... + +class B(A): + private def f(self) -> None: ... +", + ) + .build(); + assert_snapshot!(test.super_members(), @"[]"); + } + + /// a decorator can replace a class with anything, and the members are the + /// class's own all the same: those of the class as written, not of what the + /// decorator returns + #[test] + fn a_member_of_a_decorated_class() { + let test = cursor_test( + " +class Base: + def f(self) -> None: ... + +class Other(Base): ... + +def to_int(cls: type) -> int: + return 1 + +def to_other(cls: type) -> Other: + return Other() + +class A: + def f(self) -> None: ... + +@to_int +class C(A): + def f(self) -> None: ... + +@to_other +class D(A): + def f(self) -> None: ... +", + ); + assert_snapshot!(test.document_super_members(), @" + C.f 18:8 -> A.f /main.py:14 + D.f 22:8 -> A.f /main.py:14 + "); + } + + #[test] + fn an_accessor_block_is_listed_once() { + let test = CursorTest::builder() + .source( + "main.by", + " +class A: + var age: int = 0 + get() = field + set(value): + field = value + +class B(A): + override var age: int = 0 + get() = field + set(value): + field = value +", + ) + .build(); + assert_snapshot!(test.document_super_members(), @"B.age 9:17 -> A.age /main.by:3"); + } + + #[test] + fn an_accessor_block_is_asked_about_once() { + let test = CursorTest::builder() + .source( + "main.by", + " +class A: + var age: int = 0 + get() = field + set(value): + field = value + +class B(A): + override var age: int = 0 + get() = field + set(value): + field = value +", + ) + .build(); + assert_snapshot!(test.super_members(), @"A.age /main.by:3"); + } + + #[test] + fn a_setter_overriding_a_property_with_only_a_getter_goes_to_the_getter() { + let test = cursor_test( + " +class A: + @property + def x(self) -> int: + return 1 + +class B(A): + @property + def x(self) -> int: + return 2 + + @x.setter + def x(self, value: int) -> None: ... +", + ); + assert_snapshot!(test.document_super_members(), @" + B.x 9:8 -> A.x /main.py:4 + B.x 13:8 -> A.x /main.py:4 + "); + } + + #[test] + fn a_property_overriding_an_attribute() { + let test = cursor_test( + " +class A: + x: int = 1 + +class B(A): + @property + def x(self) -> int: + return 2 +", + ); + assert_snapshot!(test.document_super_members(), @"B.x 7:8 -> A.x /main.py:3"); + } + + /// a base whose members cannot be known is passed over, and the walk goes + /// on to the bases after it + #[test] + fn a_dynamic_base_is_passed_over() { + let test = cursor_test( + " +from typing import Any + +class A: + def f(self) -> None: ... + +class B(Any, A): + def f(self) -> None: ... +", + ); + assert_snapshot!(test.document_super_members(), @"B.f 8:8 -> A.f /main.py:5"); + } + + /// the keys of a typed dict are not members a class body overrides + #[test] + fn a_typed_dict_key_overrides_nothing() { + let test = cursor_test( + " +from typing import TypedDict + +class A(TypedDict): + x: int + +class B(A): + x: int +", + ); + assert_snapshot!(test.document_super_members(), @"[]"); + } + + #[test] + fn a_member_of_a_generic_class() { + let test = cursor_test( + " +class A[T]: + def f(self, x: T) -> T: + return x + +class B(A[int]): + def f(self, x: int) -> int: + return x + +class C[T](A[T]): + def f(self, x: T) -> T: + return x +", + ); + assert_snapshot!(test.document_super_members(), @" + B.f 7:8 -> A.f /main.py:3 + C.f 11:8 -> A.f /main.py:3 + "); + } + + #[test] + fn a_nested_class_an_import_and_a_walrus_in_a_decorator_are_members() { + let test = cursor_test( + " +import os + +def deco(f: object) -> object: + return f + +class A: + import os + class Meta: ... + d = deco + match 1: + case captured: ... + + @deco + def f(self) -> None: ... + +class B(A): + import os + class Meta: ... + + @(d := deco) + def f(self) -> None: ... + + match 1: + case captured: ... +", + ); + assert_snapshot!(test.document_super_members(), @" + B.os 18:11 -> A.os /main.py:8 + B.Meta 19:10 -> A.Meta /main.py:9 + B.d 21:6 -> A.d /main.py:10 + B.f 22:8 -> A.f /main.py:15 + B.captured 25:13 -> A.captured /main.py:12 + "); + } + + #[test] + fn a_member_of_object() { + let test = cursor_test( + " +class A: + def __repr__(self) -> str: + return 'A' +", + ); + assert_snapshot!(test.super_members(), @"object.__repr__ stdlib/builtins.byi:93"); + } +} diff --git a/crates/ty_project/src/build_output.rs b/crates/ty_project/src/build_output.rs new file mode 100644 index 0000000000..9f55804308 --- /dev/null +++ b/crates/ty_project/src/build_output.rs @@ -0,0 +1,165 @@ +//! the trees `by build` and `by compile` write, which are not part of the project they were built +//! from +//! +//! a build output holds a copy of the project: a `.py` for every `.by`, every hand-written `.py` +//! carried over verbatim, and the `.by` sources too when `build.sources` is on. taken for part of +//! the project, each of those would be checked a second time, and every diagnostic in a source +//! reported again against its copy, at a path the author never edits +//! +//! an output is recognised by the manifest the build leaves at its top, not by its name. the +//! default output is `build/` in the project root, but `--out` can name any directory, and a +//! project can have several; each one describes itself. a directory the author happens to call +//! `build` holds no manifest, so it stays part of the project wherever the build writes +//! +//! the exclusion applies where the project's files are discovered (the walk, and the file +//! watcher's changes) the way an ignore file's does, so that `by check`, the language server's +//! workspace diagnostics, and everything else that reads [`crate::Project::files`] agree. it +//! does not make the output unreadable: module resolution reads the file system, not the +//! project's file list, and a generated file an editor opens is still a file it can ask about + +use ruff_db::system::{System, SystemPath, SystemPathBuf}; + +/// the file a build writes at the top of its output, recording what it wrote there, and, by +/// being there, marking the directory as an output +pub const BUILD_MANIFEST: &str = ".by-manifest"; + +/// whether `directory` is the top of a build output +pub fn is_build_output(system: &dyn System, directory: &SystemPath) -> bool { + system.is_file(&directory.join(BUILD_MANIFEST)) +} + +/// whether `path` lies inside a build output below the walk root that holds it +/// +/// what the walk decides for a path it reaches from one of `walk_roots` (the project root, or +/// the paths a command line named), decided for one path on its own, as the file watcher's +/// changes need. a walk root is never itself skipped: the project root is never an output, +/// whatever it holds, and a directory a command line names is checked because it was asked for, +/// output or not. only the directories between the walk root and `path` are asked +pub(crate) fn is_within_build_output( + system: &dyn System, + path: &SystemPath, + walk_roots: &[SystemPathBuf], +) -> bool { + let Some(walk_root) = innermost_walk_root(path, walk_roots) else { + return false; + }; + path.ancestors() + .skip(1) + .take_while(|directory| *directory != walk_root) + .any(|directory| is_build_output(system, directory)) +} + +/// whether `path` is the manifest of a build output inside one of `walk_roots`: its appearing or +/// disappearing changes which of the files under its directory belong to the project +pub(crate) fn is_build_manifest(path: &SystemPath, walk_roots: &[SystemPathBuf]) -> bool { + path.file_name() == Some(BUILD_MANIFEST) + && path.parent().is_some_and(|directory| { + innermost_walk_root(directory, walk_roots) + .is_some_and(|walk_root| directory != walk_root) + }) +} + +/// the walk root nearest above `path`, or `path` itself if it is one +/// +/// the walk starts again from each of its roots, one nested in another included, so a path is +/// reached from every root above it, and from the innermost one through the fewest directories. +/// a path any of the walks keeps, that one keeps +fn innermost_walk_root<'a>( + path: &SystemPath, + walk_roots: &'a [SystemPathBuf], +) -> Option<&'a SystemPath> { + walk_roots + .iter() + .filter(|walk_root| path.starts_with(walk_root)) + .max_by_key(|walk_root| walk_root.components().count()) + .map(SystemPathBuf::as_path) +} + +#[cfg(test)] +mod tests { + use ruff_db::system::{SystemPath, SystemPathBuf, TestSystem}; + + use super::{BUILD_MANIFEST, is_build_manifest, is_within_build_output}; + + fn system_with(files: &[&str]) -> TestSystem { + let system = TestSystem::default(); + for file in files { + system + .memory_file_system() + .write_file_all(file, "") + .expect("an in-memory write succeeds"); + } + system + } + + fn roots(paths: &[&str]) -> Vec { + paths + .iter() + .map(|path| SystemPathBuf::from(*path)) + .collect() + } + + /// `--out` can name any directory, and a directory the author calls `build` is theirs until + /// a build writes a manifest into it + #[test] + fn an_output_is_recognised_by_its_manifest_not_its_name() { + let system = system_with(&["/p/anything/.by-manifest", "/p/build/tool.py"]); + let walk_roots = roots(&["/p"]); + assert!(is_within_build_output( + &system, + SystemPath::new("/p/anything/a.py"), + &walk_roots + )); + assert!( + is_within_build_output( + &system, + SystemPath::new("/p/anything/pkg/a.py"), + &walk_roots + ), + "a file deeper inside the output is inside it too" + ); + assert!(!is_within_build_output( + &system, + SystemPath::new("/p/build/tool.py"), + &walk_roots + )); + } + + /// a manifest left at the root would otherwise empty the project of every file, and a + /// directory named on the command line is checked because it was asked for + #[test] + fn a_walk_root_is_never_itself_an_output() { + let system = system_with(&["/p/.by-manifest", "/p/build/.by-manifest"]); + assert!(!is_within_build_output( + &system, + SystemPath::new("/p/a.py"), + &roots(&["/p"]) + )); + assert!(!is_within_build_output( + &system, + SystemPath::new("/p/build/a.py"), + &roots(&["/p/build"]) + )); + assert!(!is_build_manifest( + &SystemPath::new("/p").join(BUILD_MANIFEST), + &roots(&["/p"]) + )); + assert!(is_build_manifest( + &SystemPath::new("/p/build").join(BUILD_MANIFEST), + &roots(&["/p"]) + )); + + // named inside another walk root, it is still a root of its own: the walk from it keeps + // what the walk from the root above skips + let nested = roots(&["/p", "/p/build"]); + assert!(!is_within_build_output( + &system, + SystemPath::new("/p/build/a.py"), + &nested + )); + assert!(!is_build_manifest( + &SystemPath::new("/p/build").join(BUILD_MANIFEST), + &nested + )); + } +} diff --git a/crates/ty_project/src/db/changes.rs b/crates/ty_project/src/db/changes.rs index 0c1dc7ed33..519c7e31cd 100644 --- a/crates/ty_project/src/db/changes.rs +++ b/crates/ty_project/src/db/changes.rs @@ -1,3 +1,4 @@ +use crate::build_output::{is_build_manifest, is_within_build_output}; use crate::db::{Db, ProjectDatabase}; use crate::metadata::CONFIG_FILE_NAMES; use crate::script::script_tag; @@ -10,7 +11,6 @@ use ruff_db::Db as _; use ruff_db::files::{File, Files, system_path_to_file}; use ruff_db::system::{SystemPath, SystemPathBuf}; use rustc_hash::FxHashSet; -use salsa::Setter as _; use ty_python_core::program::FallibleStrategy; /// Represents the result of applying changes to the project database. @@ -19,6 +19,7 @@ pub struct ChangeResult { project_sync_path: Option, custom_stdlib_changed: bool, changed_files: ChangedFiles, + database_changed: bool, } impl ChangeResult { @@ -34,6 +35,16 @@ impl ChangeResult { self.project_sync_path.as_deref() } + /// whether applying the changes changed anything in the database + /// + /// when it did not, every query answers as it did before the changes, so nothing a client + /// has been shown can be out of date. most file system events are like that: a write to a + /// file nothing has read, to a directory nothing has listed, to a build's output or to an + /// ignored directory + pub fn database_changed(&self) -> bool { + self.database_changed + } + /// Returns `true` if the custom stdlib's VERSIONS file has changed. pub fn custom_stdlib_changed(&self) -> bool { self.custom_stdlib_changed @@ -115,6 +126,8 @@ impl ProjectDatabase { } else { ChangedFiles::Known(FxHashSet::default()) }, + // every write below reports whether it set anything, and each one that did sets this + database_changed: false, }; // Paths whose project files should be discovered incrementally. let mut added_paths = BTreeSet::default(); @@ -127,16 +140,17 @@ impl ProjectDatabase { let mut removed_paths = BTreeSet::default(); let mut reload_project = false; let mut reload_project_files = false; - // Whether the shape of the tree changed, as opposed to the contents of a - // file that was already there. See `Project::file_system_revision`. - let mut file_system_changed = false; // TODO: This should be removed once the incremental checker is ported // over to the `ignore` crate, since the `ignore` crate will respect // the settings provided in `create_walker`. ---AG let respect_ignore_files = project.settings(self).src().respect_ignore_files; - let ignore_walk_roots = - respect_ignore_files.then(|| project.included_paths_or_root(self).to_vec()); - let mut ignore_files = ignore_walk_roots.as_deref().and_then(|walk_roots| { + // also what decides what the walk would skip as a build's output, for one changed path at + // a time. a build writes a copy of every source it read, and none of it is the project's: + // the writes are synced like any file's, but they neither join the project nor set off a + // walk + let walk_roots = project.included_paths_or_root(self).to_vec(); + let ignore_walk_roots = respect_ignore_files.then_some(walk_roots.as_slice()); + let mut ignore_files = ignore_walk_roots.and_then(|walk_roots| { Some(create_walker_builder(self, walk_roots)?.incremental_matcher()) }); @@ -145,14 +159,44 @@ impl ProjectDatabase { if let Some(path) = change.system_path() { if configuration_paths.is_configuration(path, &project_root) { - File::sync_path(self, path); + result.database_changed |= File::sync_path(self, path); reload_project = true; continue; } + // a build's manifest appearing or going away turns the directory into an output or + // back into part of the project, as an ignore file saying `*` would. walking it + // again settles which: a walk skips an output. a manifest a build only rewrites + // changes nothing + if is_build_manifest(path, &walk_roots) + && matches!( + change, + ChangeEvent::Created { .. } | ChangeEvent::Deleted { .. } + ) + { + result.database_changed |= File::sync_path(self, path); + if let Some(directory) = path.parent() + && !is_within_build_output(self.system(), directory, &walk_roots) + && project.is_directory_included(self, directory) + { + tracing::debug!( + manifest = %path, + directory = %directory, + "Queueing project-file reindex for a build output's manifest" + ); + removed_paths.insert(directory.to_path_buf()); + result.changed_files.mark_unknown(); + if self.system().is_directory(directory) { + added_paths.insert(directory.to_path_buf()); + } + } + + continue; + } + if is_ignore_file(path) && project.settings(self).src().respect_ignore_files { - File::sync_path(self, path); + result.database_changed |= File::sync_path(self, path); if let Some(directory) = path.parent() { if project .included_paths_or_root(self) @@ -166,6 +210,7 @@ impl ProjectDatabase { ); reload_project_files = true; } else if project.is_directory_included(self, directory) + && !is_within_build_output(self.system(), path, &walk_roots) && ignore_files.as_mut().is_none_or(|ignore_files| { !ignore_files.is_ignored(directory, true) }) @@ -203,14 +248,10 @@ impl ProjectDatabase { ChangeEvent::Changed { path, .. } | ChangeEvent::Opened(path) | ChangeEvent::Created { path, .. } => { - if matches!(change, ChangeEvent::Created { .. }) { - file_system_changed = true; - } - match change { ChangeEvent::Changed { .. } => { if synced_files.insert(path.to_path_buf()) { - File::sync_path_only(self, path); + result.database_changed |= File::sync_path_only(self, path); } } ChangeEvent::Opened(_) @@ -219,7 +260,7 @@ impl ProjectDatabase { .. } => { if synced_files.insert(path.to_path_buf()) { - File::sync_path(self, path); + result.database_changed |= File::sync_path(self, path); } } _ => { @@ -242,6 +283,7 @@ impl ProjectDatabase { if !project .is_file_included(self, path) .should_index_file(self.system(), path) + || is_within_build_output(self.system(), path, &walk_roots) { continue; } @@ -258,9 +300,10 @@ impl ProjectDatabase { && !project.is_file_explicitly_included(self, file); if exclude_script { - project.remove_file(self, file); + result.database_changed |= project.remove_file(self, file); } else { - project.add_file(self, file, is_script); + result.database_changed |= + project.add_file(self, file, is_script); } if let ChangedFiles::Known(changed_files) = @@ -271,6 +314,7 @@ impl ProjectDatabase { } } else if change.is_created() && project.is_directory_included(self, path) + && !is_within_build_output(self.system(), path, &walk_roots) && ignore_files .as_mut() .is_none_or(|ignore_files| !ignore_files.is_ignored(path, true)) @@ -284,8 +328,6 @@ impl ProjectDatabase { } ChangeEvent::Deleted { kind, path } => { - file_system_changed = true; - let is_file = match kind { DeletedKind::File => true, DeletedKind::Directory => false, @@ -297,11 +339,11 @@ impl ProjectDatabase { if is_file { if synced_files.insert(path.to_path_buf()) { - File::sync_path(self, path); + result.database_changed |= File::sync_path(self, path); } if let Some(file) = self.files().try_system(self, path) { - project.remove_file(self, file); + result.database_changed |= project.remove_file(self, file); } } else { sync_recursively.insert(path.clone()); @@ -325,20 +367,20 @@ impl ProjectDatabase { } ChangeEvent::CreatedVirtual(path) | ChangeEvent::ChangedVirtual(path) => { - File::sync_virtual_path(self, path); + result.database_changed |= File::sync_virtual_path(self, path); } ChangeEvent::DeletedVirtual(path) => { if let Some(virtual_file) = self.files().try_virtual_file(path) { virtual_file.close(self); + result.database_changed = true; } } ChangeEvent::Rescan => { - file_system_changed = true; reload_project = true; reload_project_files = true; - Files::sync_all(self); + result.database_changed |= Files::sync_all(self); sync_recursively.clear(); removed_paths.clear(); break; @@ -346,15 +388,7 @@ impl ProjectDatabase { } } - Files::sync_all_recursive(self, sync_recursively); - - // Bumped before the reload paths below, some of which return early. - if file_system_changed { - let revision = project.file_system_revision(self); - project - .set_file_system_revision(self) - .to(revision.wrapping_add(1)); - } + result.database_changed |= Files::sync_all_recursive(self, sync_recursively); if reload_project { // The active project root may have been deleted. Start rediscovery from the closest @@ -372,8 +406,12 @@ impl ProjectDatabase { // We're not refreshing uv metadata, so use the existing environment. let environment = metadata.environment().clone(); match project.rediscover(self, path, environment) { - Ok(ProjectReloadResult::Unchanged) => {} + // a rediscovery that leaves the metadata and settings as they were can still + // replace the program settings or the settings diagnostics, and does not say + // whether it did, so it is taken to have + Ok(ProjectReloadResult::Unchanged) => result.database_changed = true, Ok(ProjectReloadResult::Changed { files_changed }) => { + result.database_changed = true; result.project_changed = true; result.changed_files.mark_unknown(); if files_changed { @@ -388,7 +426,7 @@ impl ProjectDatabase { "Failed to load project, keeping old project configuration: {error:#}" ); if reload_project_files { - project.reload_files(self); + result.database_changed |= project.reload_files(self); result.changed_files.mark_unknown(); return result; } @@ -398,7 +436,7 @@ impl ProjectDatabase { } if reload_project_files { - project.reload_files(self); + result.database_changed |= project.reload_files(self); result.changed_files.mark_unknown(); // A full project-file reload supersedes incremental project-file updates. added_paths.clear(); @@ -419,13 +457,14 @@ impl ProjectDatabase { Ok((_, diagnostics)) => diagnostics, Err(error) => vec![error.into_diagnostic()], }; - project.update_program(self, program_settings); + result.database_changed |= project.update_program(self, program_settings); settings_diagnostics.extend( program_settings_diagnostics .into_iter() .map(|diagnostic| diagnostic.into_diagnostic(self)), ); - project.update_settings_diagnostics(self, settings_diagnostics); + result.database_changed |= + project.update_settings_diagnostics(self, settings_diagnostics); } Err(error) => { tracing::error!("Failed to resolve program settings: {error}"); @@ -433,7 +472,7 @@ impl ProjectDatabase { } } - project.remove_files_under(self, removed_paths); + result.database_changed |= project.remove_files_under(self, removed_paths); let diagnostics = if !project.file_set(self).is_lazy() { // Use directory walking to discover newly added files. @@ -441,7 +480,7 @@ impl ProjectDatabase { let (files, diagnostics) = walker.collect_vec(self); for file in files { - project.add_file(self, file.file, file.is_script); + result.database_changed |= project.add_file(self, file.file, file.is_script); } diagnostics @@ -454,7 +493,7 @@ impl ProjectDatabase { // across revisions doesn't feel essential, considering that they're rare. However, we could // implement a `BTreeMap` or similar and only prune the diagnostics from paths that we've // re-scanned (or that were removed etc). - project.replace_index_diagnostics(self, diagnostics); + result.database_changed |= project.replace_index_diagnostics(self, diagnostics); result } diff --git a/crates/ty_project/src/files.rs b/crates/ty_project/src/files.rs index 5638ac0450..8577a92773 100644 --- a/crates/ty_project/src/files.rs +++ b/crates/ty_project/src/files.rs @@ -266,7 +266,16 @@ impl IndexedMut<'_> { } pub(super) fn set_diagnostics(&mut self, diagnostics: Vec) { - self.inner_mut().diagnostics = diagnostics; + if self.indexed.diagnostics != diagnostics { + self.inner_mut().diagnostics = diagnostics; + self.did_change = true; + } + } + + /// whether anything this view was asked to do changed the index, which it then writes back + /// to the database when it is dropped + pub(super) fn did_change(&self) -> bool { + self.did_change } fn inner_mut(&mut self) -> &mut IndexedInner { diff --git a/crates/ty_project/src/lib.rs b/crates/ty_project/src/lib.rs index 3665397971..8f3cd91631 100644 --- a/crates/ty_project/src/lib.rs +++ b/crates/ty_project/src/lib.rs @@ -37,6 +37,7 @@ use ty_python_semantic::lint::RuleSelection; use uv::DependencyMetadataError; pub use uv::{ScriptEnvironmentAvailability, UseUv, UvEnvironments, UvSyncChanges}; +pub mod build_output; mod db; mod files; pub mod glob; @@ -135,17 +136,6 @@ pub struct Project { #[default] #[returns(copy)] force_exclude_flag: bool, - - /// A counter bumped whenever a file or directory is created or deleted. - /// - /// The project's file set holds Python files only, so nothing in the database - /// changes when a file of any other kind appears or disappears. A query that - /// discovers such files by walking the file system — the template and static - /// asset discovery the Django template services do — has nothing else to - /// depend on, and would otherwise serve its first answer forever. - #[default] - #[returns(copy)] - pub file_system_revision: u64, } /// A checker for the part of a project the type checker does not read. @@ -293,7 +283,6 @@ impl Project { .durability(Durability::MEDIUM) .open_fileset_durability(Durability::LOW) .file_set_durability(Durability::LOW) - .file_system_revision_durability(Durability::LOW) .new(db) } @@ -388,11 +377,14 @@ impl Project { .map(|metadata| Some(Box::new(metadata))) } - pub fn update_program(self, db: &mut dyn Db, settings: ProgramSettings) { - if self.program_settings(db) != &settings { - settings.search_paths.try_register_static_roots(db); - self.set_program_settings(db).to(settings); + /// returns whether the settings differ from the ones the program had + pub fn update_program(self, db: &mut dyn Db, settings: ProgramSettings) -> bool { + if self.program_settings(db) == &settings { + return false; } + settings.search_paths.try_register_static_roots(db); + self.set_program_settings(db).to(settings); + true } pub fn root(self, db: &dyn Db) -> &SystemPath { @@ -538,15 +530,17 @@ impl Project { /// Replace stored settings diagnostics after recomputing program settings. /// /// This is used when a change affects [`ty_python_core::program::ProgramSettings`] without - /// reloading the full project. + /// reloading the full project. Returns whether the diagnostics changed. fn update_settings_diagnostics( self, db: &mut dyn Db, settings_diagnostics: Vec, - ) { - if self.settings_diagnostics(db) != settings_diagnostics { - self.set_settings_diagnostics(db).to(settings_diagnostics); + ) -> bool { + if self.settings_diagnostics(db) == settings_diagnostics { + return false; } + self.set_settings_diagnostics(db).to(settings_diagnostics); + true } /// Checks the project and its dependencies according to the project's check mode. @@ -736,8 +730,9 @@ impl Project { self.set_open_fileset(db).to(FxHashSet::default()) } + /// returns whether the file was one of the project's #[tracing::instrument(level = "debug", skip(self, db))] - pub fn remove_file(self, db: &mut dyn Db, file: File) { + pub fn remove_file(self, db: &mut dyn Db, file: File) -> bool { tracing::debug!( "Removing file `{}` from project `{}`", file.path(db), @@ -745,17 +740,17 @@ impl Project { ); let Some(mut index) = IndexedFiles::indexed_mut(db, self) else { - return; + return false; }; - index.remove(file); + index.remove(file) } - /// Removes all indexed project files under `paths`. + /// Removes all indexed project files under `paths`, and returns whether there were any. /// /// This is a no-op if the project files are still lazily indexed. #[tracing::instrument(level = "debug", skip(self, db, paths))] - fn remove_files_under(self, db: &mut dyn Db, paths: I) + fn remove_files_under(self, db: &mut dyn Db, paths: I) -> bool where I: IntoIterator, P: AsRef, @@ -768,11 +763,11 @@ impl Project { .collect::>(); if paths.is_empty() { - return; + return false; } if self.file_set(db).is_lazy() { - return; + return false; } let files_to_remove = { @@ -791,19 +786,22 @@ impl Project { }; if files_to_remove.is_empty() { - return; + return false; } let Some(mut index) = IndexedFiles::indexed_mut(db, self) else { - return; + return false; }; for file in files_to_remove { index.remove(file); } + index.did_change() } - fn add_file(self, db: &mut dyn Db, file: File, is_script: bool) { + /// returns whether the file was not already one of the project's, as a script if `is_script` + /// and as a module if not + fn add_file(self, db: &mut dyn Db, file: File, is_script: bool) -> bool { tracing::debug!( "Adding file `{}` to project `{}`", file.path(db), @@ -811,21 +809,24 @@ impl Project { ); let Some(mut index) = IndexedFiles::indexed_mut(db, self) else { - return; + return false; }; index.insert(file, is_script); + index.did_change() } - /// Replaces the diagnostics from indexing the project files with `diagnostics`. + /// Replaces the diagnostics from indexing the project files with `diagnostics`, and returns + /// whether they differ from the ones it had. /// /// This is a no-op if the project files haven't been indexed yet. - fn replace_index_diagnostics(self, db: &mut dyn Db, diagnostics: Vec) { + fn replace_index_diagnostics(self, db: &mut dyn Db, diagnostics: Vec) -> bool { let Some(mut index) = IndexedFiles::indexed_mut(db, self) else { - return; + return false; }; index.set_diagnostics(diagnostics); + index.did_change() } /// Returns whether `file` itself is an explicit check path. @@ -874,13 +875,16 @@ impl Project { } } - fn reload_files(self, db: &mut dyn Db) { + /// returns whether the files had been indexed, and so are indexed again when next asked for + fn reload_files(self, db: &mut dyn Db) -> bool { tracing::debug!("Reloading files for project `{}`", self.name(db)); - if !self.file_set(db).is_lazy() { - // Force a re-index of the files in the next revision. - self.set_file_set(db).to(IndexedFiles::lazy()); + if self.file_set(db).is_lazy() { + return false; } + // Force a re-index of the files in the next revision. + self.set_file_set(db).to(IndexedFiles::lazy()); + true } /// Check if the project's settings have any issues diff --git a/crates/ty_project/src/walk.rs b/crates/ty_project/src/walk.rs index 4b6d224b9e..85e7d6299c 100644 --- a/crates/ty_project/src/walk.rs +++ b/crates/ty_project/src/walk.rs @@ -1,3 +1,4 @@ +use crate::build_output::is_build_output; use crate::files::IndexedFile; use crate::glob::IncludeExcludeFilter; use crate::script::script_tag; @@ -78,8 +79,8 @@ impl<'a> ProjectFilesFilter<'a> { /// ## Note /// /// This method may return `true` for files that don't end up being included when walking the - /// project tree because it doesn't consider `.gitignore` and other ignore files when deciding - /// if a file's included. + /// project tree because it doesn't consider `.gitignore` and other ignore files, or the build + /// outputs the walk skips (see [`crate::build_output`]), when deciding if a file's included. pub(crate) fn is_file_included( &self, path: &SystemPath, @@ -210,6 +211,21 @@ impl ProjectFilesWalker { GlobFilterCheckMode::TopDown, ); return match directory_included { + // a tree a build wrote is a copy of the project, not part + // of it. a walk root is never skipped for it: the project + // root is never an output, and a directory named on the + // command line was asked for. see `crate::build_output` + IncludeResult::Included { .. } + if entry.depth() > 0 + && is_build_output(db.system(), entry.path()) => + { + tracing::debug!( + "Skipping directory `{path}` because a build \ + wrote it", + path = entry.path() + ); + WalkState::Skip + } IncludeResult::Included { .. } => WalkState::Continue, IncludeResult::Excluded => { tracing::debug!( diff --git a/crates/ty_python_core/src/builder.rs b/crates/ty_python_core/src/builder.rs index 47a533630e..6af822d206 100644 --- a/crates/ty_python_core/src/builder.rs +++ b/crates/ty_python_core/src/builder.rs @@ -3179,6 +3179,21 @@ impl<'db, 'ast> SemanticIndexBuilder<'db, 'ast> { .record_negated_narrowing_constraint_for_places(predicate_id, &possibly_narrowed); } + /// basedpython: visits the `finally` suite of a `try` statement reached with `try_entry` + /// + /// the suite is visited once, in a single entry state, but an exception can enter it from + /// anywhere in the statement; see `UseDefMapBuilder::try_entries_of_finally_suites` + fn visit_finally_suite( + &mut self, + finalbody: &'ast [ast::Stmt], + try_entry: ScopedReachabilityConstraintId, + ) { + self.current_use_def_map_mut() + .enter_finally_suite(try_entry); + self.visit_block_body(finalbody); + self.current_use_def_map_mut().exit_finally_suite(); + } + /// Records that all remaining statements in the current block are unreachable. fn mark_unreachable(&mut self) { self.current_use_def_map_mut().mark_unreachable(); @@ -6484,6 +6499,7 @@ impl<'db, 'ast> SemanticIndexBuilder<'db, 'ast> { let was_in_finally_statement = self.in_finally_statement; self.in_finally_statement |= !finalbody.is_empty(); self.record_ambiguous_reachability(); + let try_entry = self.current_reachability(); let exception_handlers = if handlers.is_empty() { ExceptionHandlers::None @@ -6627,7 +6643,7 @@ impl<'db, 'ast> SemanticIndexBuilder<'db, 'ast> { for snapshot in snapshots { self.flow_merge(snapshot); } - self.visit_block_body(finalbody); + self.visit_finally_suite(finalbody, try_entry); if !self.flow_snapshot().is_always_unreachable() { if !finalbody.is_empty() && has_escaping_exception { self.record_exception_checkpoint(); @@ -6685,7 +6701,7 @@ impl<'db, 'ast> SemanticIndexBuilder<'db, 'ast> { } // Mixed normal and terminal entry states are still handled by the normal path // only. See the corresponding TODO tests in `terminal_statements.md`. - self.visit_block_body(finalbody); + self.visit_finally_suite(finalbody, try_entry); if !finalbody.is_empty() && has_escaping_exception && self.current_use_def_map().reachability @@ -6767,6 +6783,8 @@ impl<'db, 'ast> SemanticIndexBuilder<'db, 'ast> { self.record_returned_place_members(value); } self.record_return_exit(stmt); + self.current_use_def_map_mut() + .record_return_reachability(stmt.range()); self.record_terminal_finally_entry(); // Everything in the current block after a terminal statement is unreachable. self.mark_unreachable(); diff --git a/crates/ty_python_core/src/use_def.rs b/crates/ty_python_core/src/use_def.rs index ef7d4ed90c..77efdb903a 100644 --- a/crates/ty_python_core/src/use_def.rs +++ b/crates/ty_python_core/src/use_def.rs @@ -674,6 +674,10 @@ struct UseDefMapExtra { /// basedpython: the state of a function's parameter places over every way out of its body, /// merged. normal_exit: Box<[PlaceBindings]>, + + /// basedpython: whether each `return` in this scope can be reached, sorted by range. a + /// `return` that is always reachable is left out + return_reachability: Box<[(TextRange, ScopedReachabilityConstraintId)]>, } /// basedpython: the state of a function's parameter places where a `return` leaves the body. @@ -842,6 +846,13 @@ pub struct UseDefMap<'db> { /// /// This is used by `can_implicitly_return_none` in the `ty_python_semantic` crate. end_of_scope_reachability: ScopedReachabilityConstraintId, + + /// Whether control can leave the scope normally: reach a `return`, or the end of the scope. + /// + /// This is the union of [`Self::end_of_scope_reachability`] and the reachability of every + /// `return` statement in the scope. A function body for which it is always false never hands + /// control back to its caller. + normal_exit_reachability: ScopedReachabilityConstraintId, } /// Shares equivalent scope-local binding and declaration tables within a file. @@ -961,6 +972,11 @@ impl<'db> UseDefMap<'db> { self.end_of_scope_reachability } + /// Whether control can reach a `return` in this scope, or the end of it. + pub fn normal_exit_reachability(&self) -> ScopedReachabilityConstraintId { + self.normal_exit_reachability + } + /// Definitions relevant to usage analysis, including standalone declarations. /// /// The early declaration part of a combined definition is omitted: its later binding entry @@ -1070,6 +1086,23 @@ impl<'db> UseDefMap<'db> { )) } + /// basedpython: whether the `return` statement at `range` can be reached + /// + /// a `return` in a `finally` suite is reached through an exception entering the suite as well + /// as through the flow it is visited in; see `UseDefMapBuilder::record_return_reachability` + pub fn return_reachability(&self, range: TextRange) -> ScopedReachabilityConstraintId { + let returns = self + .extra + .as_deref() + .map(|extra| extra.return_reachability.as_ref()) + .unwrap_or_default(); + returns + .binary_search_by_key(&range.start(), |(range, _)| range.start()) + .map_or(ScopedReachabilityConstraintId::ALWAYS_TRUE, |index| { + returns[index].1 + }) + } + fn return_exit_records(&self) -> &[ReturnExit] { self.extra .as_deref() @@ -1958,6 +1991,21 @@ pub(super) struct UseDefMapBuilder<'db> { /// start of the scope. pub(super) reachability: ScopedReachabilityConstraintId, + /// The union of the reachability of every `return` recorded so far. + return_reachability: ScopedReachabilityConstraintId, + + /// basedpython: the reachability of each `return` recorded so far, when it is not always + /// reachable + return_reachability_by_range: Vec<(TextRange, ScopedReachabilityConstraintId)>, + + /// basedpython: for each `finally` suite being visited, the reachability of its `try` statement + /// + /// the suite is visited once, in the state that finishing the `try` statement leaves (or, when + /// it cannot finish, the states its `return`s and `raise`s leave). an exception can enter the + /// suite from anywhere in the statement, and a `return` in the suite swallows that exception, + /// so it is a way out of the scope wherever the statement is reached + try_entries_of_finally_suites: Vec, + /// Tracks the reachability constraint for statements and certain sub-expressions, /// keyed by their text range. range_reachability: Vec<(TextRange, RangeInfo)>, @@ -2017,6 +2065,9 @@ impl<'db> UseDefMapBuilder<'db> { return_exits: Vec::new(), normal_exit: Vec::new(), reachability: ScopedReachabilityConstraintId::ALWAYS_TRUE, + return_reachability: ScopedReachabilityConstraintId::ALWAYS_FALSE, + return_reachability_by_range: Vec::new(), + try_entries_of_finally_suites: Vec::new(), range_reachability: Vec::new(), checkpoint_flow: ScopedReachabilityConstraintId::ALWAYS_TRUE, checkpoint_state: ExceptionCheckpointState::default(), @@ -2674,6 +2725,39 @@ impl<'db> UseDefMapBuilder<'db> { } } + /// Record that control can leave the scope through a `return` at the current point. + /// + /// basedpython: a `return` in a `finally` suite is also reached through an exception entering + /// the suite. what the suite does before the `return` is not tracked on that path, so the + /// `return` counts as reached wherever the `try` statement is + pub(super) fn record_return_reachability(&mut self, range: TextRange) { + let reachability = self.try_entries_of_finally_suites.iter().fold( + self.reachability, + |reachability, &entry| { + self.reachability_constraints + .add_or_constraint(reachability, entry) + }, + ); + self.return_reachability = self + .reachability_constraints + .add_or_constraint(self.return_reachability, reachability); + if reachability != ScopedReachabilityConstraintId::ALWAYS_TRUE { + self.return_reachability_by_range + .push((range, reachability)); + } + } + + /// basedpython: start visiting the `finally` suite of a `try` statement reached with + /// `try_entry`; see [`Self::try_entries_of_finally_suites`] + pub(super) fn enter_finally_suite(&mut self, try_entry: ScopedReachabilityConstraintId) { + self.try_entries_of_finally_suites.push(try_entry); + } + + /// basedpython: finish visiting the `finally` suite [`Self::enter_finally_suite`] started + pub(super) fn exit_finally_suite(&mut self) { + self.try_entries_of_finally_suites.pop(); + } + /// basedpython: record the state of `places` where the `return` at `range` leaves the body. /// /// Unlike a use, this is the state a caller sees once the call has returned, so pending @@ -3144,6 +3228,11 @@ impl<'db> UseDefMapBuilder<'db> { } } self.reachability_constraints.mark_used(self.reachability); + let normal_exit_reachability = self + .reachability_constraints + .add_or_constraint(self.return_reachability, self.reachability); + self.reachability_constraints + .mark_used(normal_exit_reachability); let symbol_states = Self::zip_place_states(end_of_scope_symbols, reachable_definitions_by_symbol); let member_states = @@ -3152,12 +3241,19 @@ impl<'db> UseDefMapBuilder<'db> { let loop_headers = self.loop_headers; let return_exits = self.return_exits.into_boxed_slice(); let normal_exit = self.normal_exit.into_boxed_slice(); + let mut return_reachability = self.return_reachability_by_range; + return_reachability.sort_unstable_by_key(|(range, _)| range.start()); + for &(_, reachability) in &return_reachability { + self.reachability_constraints.mark_used(reachability); + } + let return_reachability = return_reachability.into_boxed_slice(); let extra = (!bindings_by_use.is_empty() || !member_states.is_empty() || !enclosing_snapshots.is_empty() || !loop_headers.is_empty() || !return_exits.is_empty() - || !normal_exit.is_empty()) + || !normal_exit.is_empty() + || !return_reachability.is_empty()) .then(|| { Box::new(UseDefMapExtra { bindings_by_use: bindings_by_use.into(), @@ -3167,6 +3263,7 @@ impl<'db> UseDefMapBuilder<'db> { loop_headers: loop_headers.into(), return_exits, normal_exit, + return_reachability, }) }); let predicates = self.predicates.build(); @@ -3196,6 +3293,7 @@ impl<'db> UseDefMapBuilder<'db> { definitions_by_definition, extra, end_of_scope_reachability: self.reachability, + normal_exit_reachability, } } diff --git a/crates/ty_python_semantic/resources/mdtest/basedpython_terminal_calls.md b/crates/ty_python_semantic/resources/mdtest/basedpython_terminal_calls.md new file mode 100644 index 0000000000..4d94ed96ca --- /dev/null +++ b/crates/ty_python_semantic/resources/mdtest/basedpython_terminal_calls.md @@ -0,0 +1,1071 @@ +# basedpython: calls to unannotated functions that never return + +A statement-level call ends the flow when the function it calls cannot return, that is, when its +return type is `Never`. A function annotated `-> NoReturn` says so in its signature. With +`infer-unannotated-signatures` (which `sound-types` implies), a function that leaves its return type +out has it recovered from its body: when no path through the body reaches a `return` or the end of +the body without passing something that cannot complete — a `raise`, a loop that never exits, or +another call that cannot return — the function returns `Never`, and the code after a call to it is +unreachable. + +```toml +[environment] +python-version = "3.13" + +[analysis] +infer-unannotated-signatures = true +``` + +## a function that exits + +`die` never returns, because `sys.exit` is declared `NoReturn`. The code after a call to it is +unreachable, so `x` is only ever `"test"` once the branches meet: + +```py +import sys + +def die(): + sys.exit(1) + +def f(flag: bool): + if flag: + x = "terminal" + die() + else: + x = "test" + reveal_type(x) # revealed: Literal["test"] +``` + +A function that ends with such a call does not implicitly return `None`: + +```py +def g() -> int: + die() +``` + +## a chain of functions that exit + +`a` calls `b`, which calls `sys.exit`, so neither returns: + +```py +import sys + +def b(): + sys.exit(1) + +def a(): + b() + +def f(flag: bool): + if flag: + x = "terminal" + a() + else: + x = "test" + reveal_type(x) # revealed: Literal["test"] + +def g() -> int: + a() +``` + +## a function that sometimes returns + +A function with a path that reaches the end of its body, or a `return`, can return, so the code +after a call to it stays reachable: + +```py +import sys + +def exit_if(flag: bool): + if flag: + sys.exit(1) + +def exit_unless(flag: bool): + if flag: + return + sys.exit(1) + +def f(flag: int): + if flag == 1: + x = "first" + exit_if(True) + elif flag == 2: + x = "second" + exit_unless(False) + else: + x = "test" + reveal_type(x) # revealed: Literal["first", "second", "test"] + +# error: [invalid-return-type] +def g() -> int: + exit_if(True) +``` + +## `raise` in every branch + +```py +def fail(flag: bool): + if flag: + raise ValueError + else: + raise TypeError + +def f(flag: bool): + if flag: + x = "terminal" + fail(flag) + else: + x = "test" + reveal_type(x) # revealed: Literal["test"] +``` + +## a loop that never exits + +`while True:` without a `break` never falls through: + +```py +def spin(): + while True: + pass + +def f(flag: bool): + if flag: + x = "terminal" + spin() + else: + x = "test" + reveal_type(x) # revealed: Literal["test"] +``` + +A `break` is a way out of the loop, and out of the function: + +```py +def spin_until(flag: bool): + while True: + if flag: + break + +def g(flag: bool): + if flag: + x = "call" + spin_until(flag) + else: + x = "test" + reveal_type(x) # revealed: Literal["call", "test"] +``` + +## `try` and `finally` + +A `finally` suite runs on the way out, but the exception that `sys.exit` raises still propagates +after it. An exit in the `finally` suite itself ends every path through the `try`: + +```py +import sys + +def exits_in_try(): + try: + sys.exit(1) + finally: + print("cleanup") + +def exits_in_finally(): + try: + print("work") + finally: + sys.exit(1) + +def f(flag: int): + if flag == 1: + x = "try" + exits_in_try() + elif flag == 2: + x = "finally" + exits_in_finally() + else: + x = "test" + reveal_type(x) # revealed: Literal["test"] +``` + +An `except` clause that catches the exception is a way out, whether the exception is raised in the +`try` suite itself or in a function it calls: + +```py +def caught(): + try: + sys.exit(1) + except SystemExit: + pass + +def caught_from_a_call(): + try: + exits_in_try() + except SystemExit: + pass + +def g(flag: int): + if flag == 1: + x = "raised" + caught() + elif flag == 2: + x = "called" + caught_from_a_call() + else: + x = "test" + reveal_type(x) # revealed: Literal["raised", "called", "test"] +``` + +A `return` in the `finally` suite swallows the exception, so a function whose `try` suite exits +still returns when its `finally` suite does. That holds whether the `try` suite exits itself or +calls a function that does: + +```py +def returns_in_finally(): + try: + sys.exit(1) + finally: + return + +def returns_in_finally_after_a_call(): + try: + exits_in_try() + finally: + return + +def h(flag: int): + if flag == 1: + x = "exit" + returns_in_finally() + elif flag == 2: + x = "call" + returns_in_finally_after_a_call() + else: + x = "test" + reveal_type(x) # revealed: Literal["exit", "call", "test"] +``` + +A `break` in the `finally` suite swallows the exception too, and the function goes on after the +loop. The flow after a `finally` suite is only followed from the state the `try` suite finished in, +so that way out is not seen yet: + +```py +def breaks_in_finally(): + while True: + try: + sys.exit(1) + finally: + break + +def k(flag: bool): + if flag: + x = "call" + breaks_in_finally() + else: + x = "test" + # TODO: `breaks_in_finally()` returns, so this should be `Literal["call", "test"]` + reveal_type(x) # revealed: Literal["test"] +``` + +## recursion + +A function that calls itself unconditionally never returns: every call makes another, until +`RecursionError` is raised. + +```py +def forever(): + forever() + +def f(flag: bool): + if flag: + x = "terminal" + forever() + else: + x = "test" + reveal_type(x) # revealed: Literal["test"] +``` + +A recursive call behind a condition leaves the other path, which returns: + +```py +def countdown(n: int): + if n: + countdown(n - 1) + +def g(flag: bool): + if flag: + x = "call" + countdown(3) + else: + x = "test" + reveal_type(x) # revealed: Literal["call", "test"] +``` + +Mutual recursion is read the same way. `ping` and `pong` only ever call each other, so neither +returns: + +```py +def ping(): + pong() + +def pong(): + ping() + +def h(flag: bool): + if flag: + x = "terminal" + ping() + else: + x = "test" + reveal_type(x) # revealed: Literal["test"] +``` + +When one of them has a way out, both do: + +```py +def tick(n: int): + if n: + tock(n - 1) + +def tock(n: int): + tick(n) + +def k(flag: bool): + if flag: + x = "call" + tock(3) + else: + x = "test" + reveal_type(x) # revealed: Literal["call", "test"] +``` + +## a body that only types can read + +A body that exits only on a path that its types decide is read through those types, like any other +body. + +`exit` is a builtin object rather than a function, and calling it never returns: + +```py +import sys + +def exits_through_a_builtin(): + exit(1) + +def f(flag: bool): + if flag: + x = "terminal" + exits_through_a_builtin() + else: + x = "test" + reveal_type(x) # revealed: Literal["test"] + reveal_type(exits_through_a_builtin()) # revealed: Never +``` + +A condition that is always true because of its type, rather than because it is a literal, is read +through that type: + +```py +def exits_behind_isinstance(): + if isinstance(sys.argv, list): + sys.exit(1) + +def g(flag: bool): + if flag: + x = "terminal" + exits_behind_isinstance() + else: + x = "test" + reveal_type(x) # revealed: Literal["test"] + reveal_type(exits_behind_isinstance()) # revealed: Never +``` + +A function that ends with such a call does not implicitly return `None`: + +```py +def h() -> int: + exits_behind_isinstance() +``` + +A method called on a module global is read through the global's type. `ArgumentParser.error` never +returns: + +```py +import argparse + +parser = argparse.ArgumentParser() + +def usage(message: str): + parser.error(message) + +def k(flag: bool): + if flag: + x = "terminal" + usage("bad") + else: + x = "test" + reveal_type(x) # revealed: Literal["test"] +``` + +A method called through `self` in a class with bases is read through the type of `self`: + +```py +class Tool: + def describe(self) -> str: + return "tool" + +class Runner(Tool): + def die(self): + sys.exit(1) + + def run(self): + self.die() + +def m(flag: bool, runner: Runner): + if flag: + x = "terminal" + runner.run() + else: + x = "test" + reveal_type(x) # revealed: Literal["test"] +``` + +A function called through another name bound to it is read through that name's type: + +```py +def die(): + sys.exit(1) + +alias = die + +def through_an_alias(): + alias() + +def n(flag: bool): + if flag: + x = "terminal" + through_an_alias() + else: + x = "test" + reveal_type(x) # revealed: Literal["test"] +``` + +## what a `return` hands back + +A `return` that can never be reached hands nothing back, so it plays no part in the return type. A +function that exits before its only `return` returns `Never`, and a call to it ends the flow: + +```py +import sys + +def exits_before_returning(): + sys.exit(1) + return 1 + +def f(flag: bool): + if flag: + x = "terminal" + exits_before_returning() + else: + x = "test" + reveal_type(x) # revealed: Literal["test"] + reveal_type(exits_before_returning()) # revealed: Never +``` + +A `return` whose value never finishes evaluating returns `Never` too, but a call to it is still read +as one that reaches its `return`: + +```py +def returns_an_exit(): + return sys.exit(1) + +def g(flag: bool): + if flag: + x = "call" + returns_an_exit() + else: + x = "test" + # TODO: `returns_an_exit()` returns `Never`, so this should be `Literal["test"]` + reveal_type(x) # revealed: Literal["call", "test"] + reveal_type(returns_an_exit()) # revealed: Never +``` + +## a callee that cannot be resolved + +A call the checker cannot see through is treated as returning. A call to a parameter is one: which +function it calls is up to the caller. + +```py +import sys + +def call(callback): + callback() + +def exit_one(): + sys.exit(1) + +def f(flag: bool): + if flag: + x = "call" + call(exit_one) + else: + x = "test" + reveal_type(x) # revealed: Literal["call", "test"] +``` + +A name that is bound to one of two functions, depending on a condition, is another, even though +neither function returns: + +```py +def exit_two(): + sys.exit(2) + +def choose(flag: bool): + if flag: + chosen = exit_one + else: + chosen = exit_two + chosen() + +def g(flag: bool): + if flag: + x = "call" + choose(True) + else: + x = "test" + reveal_type(x) # revealed: Literal["call", "test"] +``` + +A call to something that does not resolve at all returns too: + +```py +from unknown_module import helper # error: [unresolved-import] + +def calls_unknown(): + helper() + +def h(flag: bool): + if flag: + x = "call" + calls_unknown() + else: + x = "test" + reveal_type(x) # revealed: Literal["call", "test"] +``` + +## methods + +A method and a `classmethod` are read the same way as a function, whether they are called directly +or through `self` and `cls`: + +```py +import sys + +class Exiter: + def die(self): + sys.exit(1) + + @classmethod + def class_die(cls): + sys.exit(1) + + def die_through_self(self): + self.die() + + @classmethod + def die_through_cls(cls): + cls.class_die() + +def f(flag: int, exiter: Exiter): + if flag == 1: + x = "method" + exiter.die() + elif flag == 2: + x = "classmethod" + Exiter.class_die() + elif flag == 3: + x = "self" + exiter.die_through_self() + elif flag == 4: + x = "cls" + Exiter.die_through_cls() + else: + x = "test" + reveal_type(x) # revealed: Literal["test"] +``` + +A `staticmethod` is read the same way, whether it is called through the class directly or from the +body of another function: + +```py +class Tools: + @staticmethod + def die(): + sys.exit(1) + +def dies_through_the_class(): + Tools.die() + +def h(flag: int): + if flag == 1: + x = "direct" + Tools.die() + elif flag == 2: + x = "through a function" + dies_through_the_class() + else: + x = "test" + reveal_type(x) # revealed: Literal["test"] +``` + +Every class inherits what `object` defines, and all of it states its return type. A method that +overrides one of those, such as `__init__`, takes its return type from `object` rather than from its +body, so a call to it returns: + +```py +class Resettable: + def __init__(self): + sys.exit(1) + + def reset(self): + self.__init__() + +def k(flag: bool, resettable: Resettable): + if flag: + x = "call" + resettable.reset() + else: + x = "test" + reveal_type(x) # revealed: Literal["call", "test"] +``` + +An override that leaves its return type out takes the one its base declares, so its body does not +decide whether a call to it returns: + +```py +class Base: + def run(self) -> int: + return 1 + +class Child(Base): + def run(self): + sys.exit(1) + +def g(flag: bool, child: Child): + if flag: + x = "call" + child.run() + else: + x = "test" + reveal_type(x) # revealed: Literal["call", "test"] +``` + +A base that leaves its own return type out supplies the one recovered from its body in turn, so an +override of a method that cannot return cannot return either: + +```py +class Stopper: + def stop(self): + sys.exit(1) + +class LoggingStopper(Stopper): + def stop(self): + print("stopping") + +def m(flag: bool, stopper: LoggingStopper): + if flag: + x = "call" + stopper.stop() + else: + x = "test" + # TODO: `LoggingStopper.stop` can return, which the `Never` it takes from its base hides + reveal_type(x) # revealed: Literal["test"] + reveal_type(stopper.stop()) # revealed: Never +``` + +A base whose return type cannot stand for an override's leaves the override's return type gradual, +and a call to it returns. Which type an overloaded base returns depends on the arguments, and a type +variable in the base's return type belongs to the base: + +```py +from typing import overload + +class Parser: + @overload + def parse(self, text: str) -> str: ... + @overload + def parse(self, text: bytes) -> bytes: ... + def parse(self, text: str | bytes) -> str | bytes: + return text + +class StrictParser(Parser): + def parse(self, text): + sys.exit(1) + +class Echo: + def echo[T](self, value: T) -> T: + return value + +class SilentEcho(Echo): + def echo(self, value): + sys.exit(1) + +def n(flag: int, parser: StrictParser, echo: SilentEcho): + if flag == 1: + x = "overloaded" + parser.parse("text") + elif flag == 2: + x = "generic" + echo.echo(1) + else: + x = "test" + reveal_type(x) # revealed: Literal["overloaded", "generic", "test"] +``` + +A narrowing return type is not handed to an override at all: it is a claim about what the base's +body tests, which the override's body need not test. The override's body decides instead: + +```py +from typing_extensions import TypeIs + +class Checker: + def is_int(self, value: object) -> TypeIs[int]: + return isinstance(value, int) + +class FailingChecker(Checker): + def is_int(self, value): + sys.exit(1) + +def p(flag: bool, checker: FailingChecker): + if flag: + x = "call" + checker.is_int(1) + else: + x = "test" + reveal_type(x) # revealed: Literal["test"] + reveal_type(checker.is_int(1)) # revealed: Never +``` + +## decorated functions + +A decorator can replace the function it decorates, so a call to a decorated function returns +whatever the decorator's result does. Here that result is a function that returns, so neither call +ends the flow, though the body the decorator was handed exits: + +```py +import sys +from typing import Callable + +def replace(function) -> Callable[[], None]: + return lambda: None + +@replace +def exits(): + sys.exit(1) + +def calls_decorated(): + exits() + +def f(flag: int): + if flag == 1: + x = "direct" + exits() + elif flag == 2: + x = "through a function" + calls_decorated() + else: + x = "test" + reveal_type(x) # revealed: Literal["direct", "through a function", "test"] +``` + +## callees in other modules + +A call to a function in another module is read the same way, whichever form of import names the +function or its module: + +`tools/exits.py`: + +```py +import sys + +def die(): + sys.exit(1) +``` + +`everything.py`: + +```py +from tools.exits import * + +def through_a_star_import(): + die() +``` + +`main.py`: + +```py +import tools.exits as exits +from tools.exits import die +from everything import through_a_star_import + +def through_an_alias(): + exits.die() + +def through_the_name(): + die() + +def f(flag: int): + if flag == 1: + x = "alias" + through_an_alias() + elif flag == 2: + x = "name" + through_the_name() + elif flag == 3: + x = "star import" + through_a_star_import() + else: + x = "test" + reveal_type(x) # revealed: Literal["test"] +``` + +A stub only exports what it re-exports, so a function it imports without re-exporting is not one of +its members, and a call that names it through the stub returns: + +`lib/_impl.pyi`: + +```pyi +from typing import NoReturn + +def die() -> NoReturn: ... +def stop() -> NoReturn: ... +``` + +`lib/__init__.pyi`: + +```pyi +from lib._impl import die as die +from lib._impl import stop +``` + +`uses_lib.py`: + +```py +import lib + +def dies(): + lib.die() + +def stops(): + lib.stop() # error: [unresolved-attribute] + +def g(flag: int): + if flag == 1: + x = "re-exported" + dies() + elif flag == 2: + x = "not re-exported" + stops() + else: + x = "test" + reveal_type(x) # revealed: Literal["not re-exported", "test"] +``` + +## generators and coroutines + +Calling a generator function builds a generator without running the body, so the call returns even +when the body never does: + +```py +import sys + +def exits_later(): + sys.exit(1) + yield 1 + +def f(flag: bool): + if flag: + x = "call" + exits_later() + else: + x = "test" + reveal_type(x) # revealed: Literal["call", "test"] +``` + +Calling an `async def` builds a coroutine, which is what returns; awaiting it runs the body: + +```py +async def exits_when_awaited(): + sys.exit(1) + +async def g(flag: bool): + if flag: + x = "created" + exits_when_awaited() # error: [unused-awaitable] + else: + x = "test" + reveal_type(x) # revealed: Literal["created", "test"] + + if flag: + y = "awaited" + await exits_when_awaited() + else: + y = "test" + reveal_type(y) # revealed: Literal["test"] +``` + +An `async def` that awaits one that never returns cannot return either once it is awaited: + +```py +async def awaits_the_exit(): + await exits_when_awaited() + +async def h(flag: bool): + if flag: + x = "awaited" + await awaits_the_exit() + else: + x = "test" + reveal_type(x) # revealed: Literal["test"] +``` + +## without the setting + +Without `infer-unannotated-signatures`, a function that leaves its return type out returns +`Unknown`, and a call to it is treated as returning, whatever its body does: + +```toml +[environment] +python-version = "3.13" + +[analysis] +infer-unannotated-signatures = false +``` + +```py +import sys + +def die(): + sys.exit(1) + +def f(flag: bool): + if flag: + x = "call" + die() + else: + x = "test" + reveal_type(x) # revealed: Literal["call", "test"] +``` + +## `sound-types` implies the setting + +`sound-types` turns `infer-unannotated-signatures` on, so a call to a function that cannot return +ends the flow even where the setting itself is off: + +```toml +[environment] +python-version = "3.13" + +[analysis] +infer-unannotated-signatures = false +sound-types = true +``` + +```py +import sys + +def die(): + sys.exit(1) + +def f(flag: bool): + if flag: + x = "terminal" + die() + else: + x = "test" + reveal_type(x) # revealed: Literal["test"] +``` + +## a module-level call into a module that reads the package back + +A package can call, at module level and behind a condition, a function in a submodule that reads the +package's own globals. Whether the rest of the package is reachable then depends on whether that +function returns, and reading its body must not need those globals to be known first: + +`pkg/__init__.py`: + +```py +import os + +VALUE = os.environ.get("VALUE") + +if "FLAG" in os.environ: + import pkg.sub as sub + + sub.enable() + AFTER = 2 +``` + +`pkg/sub.py`: + +```py +import sys + +import pkg + +def enable(): + check() + +def check(): + if pkg.VALUE: + sys.exit(1) + sys.exit(2) +``` + +`main.py`: + +```py +import pkg + +reveal_type(pkg.VALUE) # revealed: str | None +# error: [unresolved-attribute] +reveal_type(pkg.AFTER) # revealed: Unknown +``` + +## a callee whose annotation depends on the call + +A callee's return annotation can name a global of the module whose reachability the call decides. +Here `NoReturn = int` runs only once `enable()` has returned, and whether `enable()` returns depends +on what `pkg.NoReturn` is. Reading the call as one that does not return is consistent, and it is +what happens at run time: `stop` raises before `NoReturn = int` is reached, so `AFTER` is never +bound: + +`pkg/__init__.py`: + +```py +import os +from typing import NoReturn + +if "FLAG" in os.environ: + import pkg.sub as sub + + sub.enable() + NoReturn = int + AFTER = 1 +``` + +`pkg/sub.py`: + +```py +import pkg + +def enable(): + stop() + +def stop() -> pkg.NoReturn: + raise SystemExit +``` + +`main.py`: + +```py +from pkg import AFTER # error: [unresolved-import] +``` diff --git a/crates/ty_python_semantic/src/reachability.rs b/crates/ty_python_semantic/src/reachability.rs index 9001e6220b..c6a86ebf31 100644 --- a/crates/ty_python_semantic/src/reachability.rs +++ b/crates/ty_python_semantic/src/reachability.rs @@ -195,10 +195,12 @@ use crate::ProgramEnvironment; use std::cell::RefCell; +use std::ops::ControlFlow; use crate::types::context_sensitive::case_name_pattern_type; use crate::types::function::KnownFunction; use crate::types::narrow::pattern_subject_type; +use crate::types::normal_completion::body_recovered_call_returns; use crate::{ Db, dunder_all::dunder_all_names, @@ -785,30 +787,20 @@ fn evaluate_reachability_path<'db>( constraints: &ReachabilityConstraints, predicates: &IndexSlice>, call_predicates: Option<&[ScopedPredicateId]>, - mut id: ScopedReachabilityConstraintId, + id: ScopedReachabilityConstraintId, mut use_checkpoint: bool, ) -> Truthiness { let env = ProgramEnvironment::from_scope(scope); let mut visited = 0; - loop { - if let Some(reachability) = terminal_reachability(id) { - return reachability; + walk_decision_diagram(constraints, id, |id, atom| { + if use_checkpoint && is_reachability_checkpoint(call_predicates, atom, visited) { + return ControlFlow::Break(evaluate_reachability_checkpoint(db, scope, id)); } - - let node = constraints.get_interior_node(id); - if use_checkpoint && is_reachability_checkpoint(call_predicates, node.atom(), visited) { - return evaluate_reachability_checkpoint(db, scope, id); - } - - id = match analyze_single(db, &env, &predicates[node.atom()]) { - Truthiness::AlwaysTrue => node.if_true(), - Truthiness::Ambiguous => node.if_ambiguous(), - Truthiness::AlwaysFalse => node.if_false(), - }; use_checkpoint = true; visited += 1; - } + ControlFlow::Continue(analyze_single(db, &env, &predicates[atom])) + }) } /// Evaluates a canonical suffix of a reachability decision diagram. @@ -1837,6 +1829,23 @@ fn analyze_non_terminal_call<'db>( return Truthiness::AlwaysTrue; } + // basedpython: whether a function whose return type is recovered from its body returns is + // first read off the body's control flow. Recovering the type infers the whole body, and a + // body reached from a module-level call reads the module's globals, whose visibility depends + // on whether this very call returns. Only when the control flow leaves the answer open is the + // recovered type read, below. See `normal_completion`. + let function = match ty { + Type::FunctionLiteral(function) => Some(function), + Type::BoundMethod(method) => Some(method.function(db)), + _ => None, + }; + if let Some(function) = function + && let Some(returns @ (Truthiness::AlwaysTrue | Truthiness::AlwaysFalse)) = + body_recovered_call_returns(db, function, is_await) + { + return returns; + } + let overloads_iterator = if let Some(callable) = ty .try_upcast_to_callable(db, &env) .and_then(CallableTypes::exactly_one) @@ -1969,19 +1978,122 @@ fn analyze_condition<'db>(db: &'db dyn Db, expression: Expression<'db>) -> Truth .unwrap_or(Truthiness::Ambiguous) } +/// basedpython: how the predicates of a reachability decision diagram are decided +/// +/// the checker decides them by inferring the types they test ([`TypeReading`]). whether a call +/// to a function whose return type is recovered from its body can return is read off the same +/// diagram without inferring any expression (`normal_completion`), where a predicate only types +/// decide is ambiguous. both decide each predicate with [`analyze_predicate`] +pub(crate) trait PredicateReading<'db> { + /// the environment types are inferred in, or `None` for a reading that infers no expression + /// + /// a reading without types answers `Ambiguous` wherever only types could decide, and its + /// `Ambiguous` then says that it cannot tell, not that the predicate can go either way + fn types(&self) -> Option<&ProgramEnvironment<'db>>; + + /// whether the statement-level call `call` can return + fn statement_call(&mut self, call: CallableAndCallExpr<'db>) -> Truthiness; + + /// the reachability of `continuation`, a node of `scope`'s own diagram + fn continuation( + &mut self, + scope: ScopeId<'db>, + continuation: ScopedReachabilityConstraintId, + ) -> Truthiness; +} + +/// the checker's reading of a diagram: every predicate is decided by the types it tests +struct TypeReading<'a, 'db> { + db: &'db dyn Db, + env: &'a ProgramEnvironment<'db>, +} + +impl<'db> PredicateReading<'db> for TypeReading<'_, 'db> { + fn types(&self) -> Option<&ProgramEnvironment<'db>> { + Some(self.env) + } + + fn statement_call(&mut self, call: CallableAndCallExpr<'db>) -> Truthiness { + analyze_non_terminal_call(self.db, call.callable, call.call_expr, call.is_await) + } + + fn continuation( + &mut self, + scope: ScopeId<'db>, + continuation: ScopedReachabilityConstraintId, + ) -> Truthiness { + evaluate_finally_continuation(self.db, scope, continuation) + } +} + +/// walks `constraints` from `id` to one of its terminals, deciding each predicate on the way with +/// `reading` +pub(crate) fn evaluate_with_reading<'db>( + db: &'db dyn Db, + constraints: &ReachabilityConstraints, + predicates: &IndexSlice>, + id: ScopedReachabilityConstraintId, + reading: &mut impl PredicateReading<'db>, +) -> Truthiness { + walk_decision_diagram(constraints, id, |_, atom| { + ControlFlow::Continue(analyze_predicate(db, reading, &predicates[atom])) + }) +} + +/// walks `constraints` from `id` to one of its terminals, taking at each node the branch for the +/// truthiness `decide` gives the node's predicate. `decide` can instead break with the +/// reachability of the node it was given, which ends the walk there +fn walk_decision_diagram( + constraints: &ReachabilityConstraints, + mut id: ScopedReachabilityConstraintId, + mut decide: impl FnMut( + ScopedReachabilityConstraintId, + ScopedPredicateId, + ) -> ControlFlow, +) -> Truthiness { + loop { + if let Some(reachability) = terminal_reachability(id) { + return reachability; + } + let node = constraints.get_interior_node(id); + id = match decide(id, node.atom()) { + ControlFlow::Break(reachability) => return reachability, + ControlFlow::Continue(Truthiness::AlwaysTrue) => node.if_true(), + ControlFlow::Continue(Truthiness::Ambiguous) => node.if_ambiguous(), + ControlFlow::Continue(Truthiness::AlwaysFalse) => node.if_false(), + }; + } +} + fn analyze_single(db: &dyn Db, env: &ProgramEnvironment<'_>, predicate: &Predicate) -> Truthiness { - let _span = tracing::trace_span!("analyze_single", ?predicate).entered(); + analyze_predicate(db, &mut TypeReading { db, env }, predicate) +} + +/// the truthiness of `predicate`, as `reading` decides it +fn analyze_predicate<'db>( + db: &'db dyn Db, + reading: &mut impl PredicateReading<'db>, + predicate: &Predicate<'db>, +) -> Truthiness { + let _span = tracing::trace_span!("analyze_predicate", ?predicate).entered(); match predicate.node { - PredicateNode::Expression(test_expr) => { - infer_same_file_expression_type(db, test_expr, TypeContext::default()) - .bool(db, env) - .negate_if(!predicate.is_positive) + PredicateNode::Expression(test_expr) => match reading.types() { + Some(env) => { + infer_same_file_expression_type(db, test_expr, TypeContext::default()).bool(db, env) + } + None => analyze_literal_condition(db, test_expr), } - PredicateNode::Condition(test_expr) => { - analyze_condition(db, test_expr).negate_if(!predicate.is_positive) + .negate_if(!predicate.is_positive), + PredicateNode::Condition(test_expr) => match reading.types() { + Some(_) => analyze_condition(db, test_expr), + None => analyze_literal_condition(db, test_expr), } + .negate_if(!predicate.is_positive), PredicateNode::ChainedComparisonCondition(test_expr) => { + let Some(env) = reading.types() else { + return Truthiness::Ambiguous; + }; let inference = infer_expression_types(db, test_expr, TypeContext::default()); let expression = test_expr.node_ref(db); inference @@ -1992,42 +2104,69 @@ fn analyze_single(db: &dyn Db, env: &ProgramEnvironment<'_>, predicate: &Predica PredicateNode::ContextManagerSuppresses { expression, is_async, - } => Truthiness::from( - infer_same_file_expression_type(db, expression, TypeContext::default()) - .can_suppress_exceptions(db, env, EvaluationMode::from_is_async(is_async)), - ) - .negate_if(!predicate.is_positive), + } => { + let Some(env) = reading.types() else { + return Truthiness::Ambiguous; + }; + Truthiness::from( + infer_same_file_expression_type(db, expression, TypeContext::default()) + .can_suppress_exceptions(db, env, EvaluationMode::from_is_async(is_async)), + ) + .negate_if(!predicate.is_positive) + } PredicateNode::FinallyNormalPathImpossible { scope, continuation, - } => Truthiness::from( - evaluate_finally_continuation(db, scope, continuation).is_always_false(), - ) - .negate_if(!predicate.is_positive), - PredicateNode::IsNonTerminalCall(CallableAndCallExpr { - callable, - call_expr, - is_await, - }) => analyze_non_terminal_call(db, callable, call_expr, is_await) + } => { + let decides_every_predicate = reading.types().is_some(); + match reading.continuation(scope, continuation) { + Truthiness::AlwaysFalse => Truthiness::AlwaysTrue, + Truthiness::AlwaysTrue => Truthiness::AlwaysFalse, + Truthiness::Ambiguous if decides_every_predicate => Truthiness::AlwaysFalse, + // basedpython: a continuation a reading without types cannot decide may still be + // unreachable once types are read + Truthiness::Ambiguous => Truthiness::Ambiguous, + } + .negate_if(!predicate.is_positive) + } + PredicateNode::IsNonTerminalCall(call) => reading + .statement_call(call) .negate_if(!predicate.is_positive), // basedpython: an assertion guard is recorded as an unconditional narrowing // constraint, so its truth value is the fact that the call returned PredicateNode::AssertsCall(_) => Truthiness::AlwaysTrue.negate_if(!predicate.is_positive), - PredicateNode::Pattern(inner) => analyze_pattern_predicate(db, inner), + PredicateNode::Pattern(inner) => { + if reading.types().is_none() { + return Truthiness::Ambiguous; + } + analyze_pattern_predicate(db, inner) + } PredicateNode::OrPatternAlternative(_) => Truthiness::Ambiguous, // basedpython: the capture a bare `case A:` would bind exists only where // the name did not resolve to an enum member of the subject PredicateNode::CaseNameCapture(capture) => { + let Some(env) = reading.types() else { + return Truthiness::Ambiguous; + }; Truthiness::from(case_name_pattern_type(db, env, capture.kind(db)).is_none()) .negate_if(!predicate.is_positive) } PredicateNode::SubjectElementPattern(subject_element) => { + if reading.types().is_none() { + return Truthiness::Ambiguous; + } analyze_pattern_predicate(db, subject_element.pattern) } PredicateNode::IsNonEmptyIterable(iterable) => { + if reading.types().is_none() { + return Truthiness::Ambiguous; + } analyze_non_empty_iterable(db, iterable).negate_if(!predicate.is_positive) } PredicateNode::StarImportPlaceholder(star_import) => { + let Some(env) = reading.types() else { + return Truthiness::Ambiguous; + }; let place_table = place_table(db, star_import.scope(db)); let symbol = place_table.symbol(star_import.symbol_id(db)); let program_file = star_import.referenced_file(db); @@ -2070,6 +2209,27 @@ fn analyze_single(db: &dyn Db, env: &ProgramEnvironment<'_>, predicate: &Predica } } +/// basedpython: the truthiness of a condition read without types, where only a literal whose +/// type states its truthiness counts: `True`, `False`, `None`, an integer, a string or bytes. +/// types would decide those the same way +fn analyze_literal_condition(db: &dyn Db, condition: Expression<'_>) -> Truthiness { + let module = parsed_module(db, condition.python_file(db)).load(db); + analyze_condition_expression(condition.node_ref(db).node(&module), &|leaf| { + Some(match leaf { + ast::Expr::BooleanLiteral(literal) => Truthiness::from(literal.value), + ast::Expr::NoneLiteral(_) => Truthiness::AlwaysFalse, + ast::Expr::NumberLiteral(ast::ExprNumberLiteral { + value: ast::Number::Int(int), + .. + }) => Truthiness::from(*int != 0), + ast::Expr::StringLiteral(literal) => Truthiness::from(!literal.value.is_empty()), + ast::Expr::BytesLiteral(literal) => Truthiness::from(!literal.value.is_empty()), + _ => Truthiness::Ambiguous, + }) + }) + .unwrap_or(Truthiness::Ambiguous) +} + /// Check whether a diagnostic emitted at `range` is in reachable code, considering both /// scope reachability and statement-level reachability within the scope. pub(crate) fn is_range_reachable<'db>( diff --git a/crates/ty_python_semantic/src/types.rs b/crates/ty_python_semantic/src/types.rs index 684584e4f2..e08e009fc1 100644 --- a/crates/ty_python_semantic/src/types.rs +++ b/crates/ty_python_semantic/src/types.rs @@ -223,6 +223,7 @@ pub(crate) mod name_fallback; pub(crate) mod narrow; pub(crate) mod narrowing_guards; mod newtype; +pub(crate) mod normal_completion; mod overlapping; mod overrides; mod protocol_class; diff --git a/crates/ty_python_semantic/src/types/class.rs b/crates/ty_python_semantic/src/types/class.rs index 0de48e08cd..6f28c12346 100644 --- a/crates/ty_python_semantic/src/types/class.rs +++ b/crates/ty_python_semantic/src/types/class.rs @@ -1478,36 +1478,6 @@ impl<'db> ClassType<'db> { self, db: &'db dyn Db, ) -> FxIndexMap> { - fn type_as_abstract_method<'db>( - db: &'db dyn Db, - ty: Type<'db>, - defining_class: ClassType<'db>, - ) -> Option { - match ty { - Type::FunctionLiteral(function) => function.as_abstract_method(db, defining_class), - Type::BoundMethod(method) => { - method.function(db).as_abstract_method(db, defining_class) - } - Type::PropertyInstance(property) => { - // A property is abstract if any of its accessors is abstract. - property - .getter(db) - .and_then(|getter| type_as_abstract_method(db, getter, defining_class)) - .or_else(|| { - property.setter(db).and_then(|setter| { - type_as_abstract_method(db, setter, defining_class) - }) - }) - .or_else(|| { - property.deleter(db).and_then(|deleter| { - type_as_abstract_method(db, deleter, defining_class) - }) - }) - } - _ => None, - } - } - let mut abstract_methods: FxIndexMap = FxIndexMap::default(); let env = &ProgramEnvironment::from_file(self.class_literal(db).program_file(db)); @@ -3577,3 +3547,37 @@ pub(super) enum MetaclassErrorKind<'db> { /// A cycle was encountered attempting to determine the metaclass Cycle, } + +/// whether `ty`, the type a class body binds a member to in `defining_class`, is an abstract +/// method, and why: an `@abstractmethod`, or a protocol method with no implementation. a property +/// is abstract when any of its accessors is +/// +/// what `abstract-instantiation` counts as abstract, and what an editor asks when it says whether a +/// member implements what it overrides or overrides an implementation +pub(in crate::types) fn type_as_abstract_method<'db>( + db: &'db dyn Db, + ty: Type<'db>, + defining_class: ClassType<'db>, +) -> Option { + match ty { + Type::FunctionLiteral(function) => function.as_abstract_method(db, defining_class), + Type::BoundMethod(method) => method.function(db).as_abstract_method(db, defining_class), + Type::PropertyInstance(property) => { + // A property is abstract if any of its accessors is abstract. + property + .getter(db) + .and_then(|getter| type_as_abstract_method(db, getter, defining_class)) + .or_else(|| { + property + .setter(db) + .and_then(|setter| type_as_abstract_method(db, setter, defining_class)) + }) + .or_else(|| { + property + .deleter(db) + .and_then(|deleter| type_as_abstract_method(db, deleter, defining_class)) + }) + } + _ => None, + } +} diff --git a/crates/ty_python_semantic/src/types/definition_resolution.rs b/crates/ty_python_semantic/src/types/definition_resolution.rs index 48b7fac311..8c5998290c 100644 --- a/crates/ty_python_semantic/src/types/definition_resolution.rs +++ b/crates/ty_python_semantic/src/types/definition_resolution.rs @@ -189,6 +189,29 @@ pub(crate) fn scoped_definitions_for_name<'db>( alias_resolution: ImportAliasResolution, ) -> Vec> { let env = ProgramEnvironment::from_scope(scope); + let all_definitions = visible_definitions_for_name(db, scope, name_str); + + // Resolve import definitions to their targets + let mut resolved_definitions = Vec::new(); + + for definition in &all_definitions { + let resolved = resolve_definition(db, &env, *definition, Some(name_str), alias_resolution); + resolved_definitions.extend(resolved); + } + + resolved_definitions +} + +/// The definitions a use of `name_str` in `scope` can refer to, from the nearest visible scope +/// that has any, without following imports and without falling back to implicit builtins. +/// +/// This reads only the semantic index: every binding and declaration of the name in that scope, +/// whether or not it is reachable from the use. +pub(crate) fn visible_definitions_for_name<'db>( + db: &'db dyn Db, + scope: ScopeId<'db>, + name_str: &str, +) -> FxIndexSet> { let file = scope.program_file(db); let index = semantic_index(db, file); let file_scope = scope.file_scope_id(db); @@ -282,15 +305,7 @@ pub(crate) fn scoped_definitions_for_name<'db>( } } - // Resolve import definitions to their targets - let mut resolved_definitions = Vec::new(); - - for definition in &all_definitions { - let resolved = resolve_definition(db, &env, *definition, Some(name_str), alias_resolution); - resolved_definitions.extend(resolved); - } - - resolved_definitions + all_definitions } /// Resolves a symbol in an implicit builtins scope. diff --git a/crates/ty_python_semantic/src/types/function.rs b/crates/ty_python_semantic/src/types/function.rs index 5ae93afbc9..4391a16605 100644 --- a/crates/ty_python_semantic/src/types/function.rs +++ b/crates/ty_python_semantic/src/types/function.rs @@ -482,8 +482,53 @@ pub(crate) struct CallbackParameterModifiers { pub(crate) kwarg_once: Option, } +/// basedpython: what a function's `def` line says; see [`OverloadLiteral::header`] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, get_size2::GetSize, salsa::SalsaValue)] +pub(crate) struct FunctionHeader { + /// an `async def` + pub(crate) is_async: bool, + /// `-> asserts x` + pub(crate) is_asserts_return: bool, + pub(crate) form: FunctionForm, +} + +/// basedpython: what a `def` defines, where that is more than a function +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, get_size2::GetSize, salsa::SalsaValue)] +pub(crate) enum FunctionForm { + Function, + /// the getter of a property with an accessor block + PropertyGetter, + /// a `decorator def` + DecoratorKeyword, +} + +impl FunctionHeader { + pub(crate) fn of(function: &ast::StmtFunctionDef) -> Self { + let form = if function.property_construct_range().is_some() { + FunctionForm::PropertyGetter + } else if is_decorator_keyword_def(function) { + FunctionForm::DecoratorKeyword + } else { + FunctionForm::Function + }; + Self { + is_async: function.is_async, + is_asserts_return: function.is_asserts_return, + form, + } + } +} + #[salsa::tracked] impl<'db> OverloadLiteral<'db> { + /// basedpython: what this function's `def` line says, read once so that asking it of a + /// function in another module does not tie the asker to that module's syntax tree + #[salsa::tracked(returns(copy), heap_size=ruff_memory_usage::heap_size)] + pub(crate) fn header(self, db: &'db dyn Db) -> FunctionHeader { + let module = parsed_module(db, self.python_file(db)).load(db); + FunctionHeader::of(self.node(db, self.file(db), &module)) + } + /// basedpython: the type an untyped property declares through its initialiser, when this /// function is one of that property's accessors /// @@ -1079,11 +1124,7 @@ impl<'db> OverloadLiteral<'db> { let program_file = self.program_file(db); let module = parsed_module(db, program_file.python_file(db)).load(db); let function_node = scope.node(db).expect_function().node(&module); - let is_decorator_keyword = function_node.decorator_list.iter().any(|decorator| { - matches!(&decorator.expression, ast::Expr::Name(name) - if name.ctx.is_invalid() && name.id == "decorator_keyword") - }); - if !is_decorator_keyword { + if !is_decorator_keyword_def(function_node) { return None; } let index = semantic_index(db, program_file); @@ -1336,15 +1377,12 @@ impl<'db> OverloadLiteral<'db> { .declare_unannotated_parameter(&value.parameter.name.id, property_type); } } - let returns_written = - function_stmt_node.returns.is_some() || (is_property_getter && property_type.is_some()); // basedpython: if this is the implementation of an overloaded function // (i.e. preceded by `@overload` stubs), infer unannotated parameter - // types and an unannotated return type from the union of the sibling - // overloads. this lets `def foo(i)` after `def foo(i: str) -> int` - // and `def foo(i: int) -> str` see `i` as `str | int` and have its - // return statements checked against `int | str` + // types from the union of the sibling overloads. this lets `def foo(i)` + // after `def foo(i: str) -> int` and `def foo(i: int) -> str` see `i` as + // `str | int`. the return type is `return_type_source`'s to decide if !self.is_overload(db) && let Some(previous) = self.previous_overload(db) { @@ -1354,38 +1392,23 @@ impl<'db> OverloadLiteral<'db> { .iter() .map(|ol| ol.raw_signature(db, ReturnCallableTypeVarScope::Public)) .collect(); - raw_signature.inherit_unannotated_from_overloads( - db, - env, - &overload_sigs, - !returns_written, - ); + raw_signature.inherit_unannotated_from_overloads(db, env, &overload_sigs, false); } } - // basedpython: an unannotated method inherits its parameter and return types from the - // method it overrides, so `def m(self, a)` over `def m(self, a: int) -> bytes` sees - // `a: int` and returns `bytes`. this is the first place a missing annotation is recovered - // from, so it shares the gate with the body-derived recovery below — reading the body - // while skipping this would answer with what the placeholder body happens to do rather - // than with what the base already declared + // basedpython: an unannotated method inherits its parameter types from the method it + // overrides, so `def m(self, a)` over `def m(self, a: int)` sees `a: int`. it shares the + // gate with the recovery of a return type, which draws on the same method first; see + // `return_type_source` if infers_unannotated_signatures(db, self.file(db)) - && raw_signature.has_inherited_annotations_to_fill(!returns_written) + && raw_signature.has_inherited_annotations_to_fill(false) && let Some(base_signature) = self.overridden_signature(db, env) { - // a narrowing return type is the exception. it is a claim about what the body tests, - // so an override that tests something else — or nothing — would be handed a claim it - // does not make, and every call through it would narrow on the strength of it - let inherits_return_type = !returns_written - && !matches!( - base_signature.return_ty, - Type::TypeIs(_) | Type::TypeGuard(_) - ); raw_signature.inherit_unannotated_from_overloads( db, env, std::slice::from_ref(&base_signature), - inherits_return_type, + false, ); } @@ -1404,24 +1427,6 @@ impl<'db> OverloadLiteral<'db> { raw_signature.inherit_defaults_from(db, env, base_raw_signature(db, base)); } - // basedpython: an overridden base that could not supply a whole signature can still - // supply its return type - // - // a narrowing return type is the exception: it is a claim about what the body tests, and - // an override that tests something else — or nothing — would be handed a claim it does - // not make, and every call through it would narrow on the strength of it. so that one is - // left to the override's own body - if infers_unannotated_signatures(db, self.file(db)) - && !returns_written - && !function_stmt_node.is_asserts_return - && raw_signature.return_ty.is_unknown() - && let OverriddenReturnType::Declared(base_return) = - self.overridden_return_type(db, env) - && !matches!(base_return, Type::TypeIs(_) | Type::TypeGuard(_)) - { - raw_signature.return_ty = base_return; - } - // basedpython: `@d` on `def f(i)` hands `f` straight to `d`, so a parameter left // unannotated takes its type from the callable `d` declares it accepts — the same context // that types the parameters of `d(lambda i: ...)`. this runs after the sources that say @@ -1445,35 +1450,36 @@ impl<'db> OverloadLiteral<'db> { ); } - // basedpython: a return type nothing else supplied is recovered from the body — the - // union of what it returns, and `None` when it can also fall off the end. this runs - // last so that an annotation, a sibling overload and an overridden base all still win - // - // the three blocks above plus this one are the sources a `def` that leaves its return - // type out draws on; [`OverloadLiteral::return_type_without_annotation`] mirrors them in - // this order, so a change here belongs there too + // basedpython: a return type nobody wrote is drawn from the first source that supplies + // one; see `return_type_source`. the body is the last of them, and recovering from it + // takes the union of what it returns, and `None` when it can also fall off the end let mut reads_body_for_guards = false; - if infers_unannotated_signatures(db, self.file(db)) - && !returns_written - && !function_stmt_node.is_asserts_return - && raw_signature.return_ty.is_unknown() - && self.recovers_return_type_from_body(db, env) - { - raw_signature.return_ty = inferred_return_type(db, self); - - // basedpython: a returned expression says more than its own type does — - // `return a is int` tells every caller what a truthy result means about the argument. - // That reading is recovered beside the return type and only where it is: a base's - // return type stands for its overrides, so a body that answered nothing about the - // return type answers nothing about the narrowing either - let guards = inferred_predicate_guards(db, self); - if !guards.is_empty() { - if let Some(narrowed) = Self::symmetric_guard_type(db, env, guards) { - raw_signature.return_ty = TypeIsType::from_type_expression(db, narrowed); + match self.return_type_source(db, env, false, |base| { + Self::overridden_return_type(db, env, base) + }) { + ReturnTypeSource::Overridden(ty) | ReturnTypeSource::Overloads(ty) => { + raw_signature.return_ty = ty; + } + ReturnTypeSource::Body => { + raw_signature.return_ty = inferred_return_type(db, self); + + // basedpython: a returned expression says more than its own type does — + // `return a is int` tells every caller what a truthy result means about the + // argument. That reading is recovered beside the return type and only where it + // is: a base's return type stands for its overrides, so a body that answered + // nothing about the return type answers nothing about the narrowing either + let guards = inferred_predicate_guards(db, self); + if !guards.is_empty() { + if let Some(narrowed) = Self::symmetric_guard_type(db, env, guards) { + raw_signature.return_ty = TypeIsType::from_type_expression(db, narrowed); + } + raw_signature.narrowing_guards.clone_from(guards); } - raw_signature.narrowing_guards.clone_from(guards); + reads_body_for_guards = true; } - reads_body_for_guards = true; + ReturnTypeSource::Written + | ReturnTypeSource::PropertyInitialiser(_) + | ReturnTypeSource::Gradual => {} } // basedpython: a body that hands back nothing makes assertions instead of predicates, and @@ -1620,11 +1626,6 @@ impl<'db> OverloadLiteral<'db> { /// basedpython: the return type this function would have if its return annotation were /// deleted. /// - /// The sources are the ones [`OverloadLiteral::raw_signature`] consults for a `def` that - /// leaves its return type out, in the same order: a sibling overload group, then an - /// overridden base method under `sound-types`, and — only when neither of those supplied - /// anything — the body. - /// /// The body is the one source not computed here. It reads the function's own scope /// inference, and the `redundant-return-annotation` lint calls this from inside that very /// inference, so the caller supplies it. @@ -1634,22 +1635,78 @@ impl<'db> OverloadLiteral<'db> { env: &ProgramEnvironment<'db>, from_body: impl FnOnce() -> Type<'db>, ) -> Type<'db> { - // the getter of an untyped property with an initialiser — the setter always writes its - // `-> None` — is held to that initialiser's type ahead of everything below - if !self.has_explicit_return_annotation(db) + match self.return_type_source(db, env, true, |base| { + Self::overridden_return_type(db, env, base) + }) { + ReturnTypeSource::PropertyInitialiser(ty) + | ReturnTypeSource::Overridden(ty) + | ReturnTypeSource::Overloads(ty) => ty, + ReturnTypeSource::Body => from_body(), + ReturnTypeSource::Written | ReturnTypeSource::Gradual => Type::unknown(), + } + } + + /// basedpython: where this function's return type comes from, in the order every reader of + /// it consults the sources: [`OverloadLiteral::raw_signature`], which builds the signature; + /// [`OverloadLiteral::return_type_without_annotation`], which the + /// `redundant-return-annotation` lint compares an annotation against; and the reading of + /// whether a call to the function can return (`normal_completion`) + /// + /// `without_annotation` answers for the function as though its return annotation were + /// deleted. `overridden` says what the method this one overrides supplies. the signature and + /// the lint read the base's signature for that; whether a call can return is answered without + /// recovering any return type, so that reading says what it can of the base without one + /// + /// the body is not read when a base answers. that includes a base whose answer cannot be + /// copied, because it is overloaded or in terms of its own type variables: the body must not + /// answer differently from the base, so the return type stays gradual. a narrowing return + /// type is not copied at all. it is a claim about what the base's body tests, so an override + /// that tests something else — or nothing — would be handed a claim it does not make, and + /// every call through it would narrow on the strength of it. that one is left to the + /// override's own body + /// + /// `__new__`'s return type is not read off its body. construction reads it structurally — it + /// decides whether the result is an instance and so whether `__init__` runs at all — and the + /// usual body, `return super().__new__(cls)`, does not currently describe one: + /// `object.__new__(cls)` solves to `Never` + pub(crate) fn return_type_source( + self, + db: &'db dyn Db, + env: &ProgramEnvironment<'db>, + without_annotation: bool, + overridden: impl FnOnce(FunctionType<'db>) -> OverriddenReturnType<'db>, + ) -> ReturnTypeSource<'db> { + let header = self.header(db); + if header.is_asserts_return + || (!without_annotation && self.has_explicit_return_annotation(db)) + { + return ReturnTypeSource::Written; + } + if header.form == FunctionForm::PropertyGetter && let Some(property_type) = self.property_initialiser_type(db) { - return property_type; + return ReturnTypeSource::PropertyInitialiser(property_type); } - let mut return_ty = Type::unknown(); + let infers = infers_unannotated_signatures(db, self.file(db)); + let base = if infers { + self.overridden_method(db, env) + .map_or(OverriddenReturnType::NotOverridden, overridden) + } else { + OverriddenReturnType::NotOverridden + }; + if let OverriddenReturnType::Declared(base_return) = base + && !matches!(base_return, Type::TypeIs(_) | Type::TypeGuard(_)) + { + return ReturnTypeSource::Overridden(base_return); + } if !self.is_overload(db) && let Some(previous) = self.previous_overload(db) { let (overload_list, _) = previous.overloads_and_implementation(db); if !overload_list.is_empty() { - return_ty = UnionType::from_elements( + let returned = UnionType::from_elements( db, env, overload_list.iter().map(|overload| { @@ -1658,67 +1715,35 @@ impl<'db> OverloadLiteral<'db> { .return_ty }), ); + if !returned.is_unknown() { + return ReturnTypeSource::Overloads(returned); + } } } - if db.analysis_settings(self.file(db)).sound_types - && let Some(base_signature) = self.overridden_signature(db, env) - { - return_ty = base_signature.return_ty; + if !infers || self.name(db) == "__new__" || base == OverriddenReturnType::Inexpressible { + return ReturnTypeSource::Gradual; } - - if return_ty.is_unknown() - && infers_unannotated_signatures(db, self.file(db)) - && let OverriddenReturnType::Declared(base_return) = - self.overridden_return_type(db, env) - { - return_ty = base_return; - } - - if infers_unannotated_signatures(db, self.file(db)) - && return_ty.is_unknown() - && self.recovers_return_type_from_body(db, env) - { - return_ty = from_body(); - } - - return_ty + ReturnTypeSource::Body } - /// basedpython: what the method this one overrides says its return type is, for the cases - /// where [`OverloadLiteral::overridden_signature`] could not supply the whole signature. + /// basedpython: what the method `base` says its return type is, as it would stand for the + /// return type of a method that overrides it and writes none /// - /// Inheriting a whole signature has to refuse a base whose *parameters* mention a type - /// variable — the implicit `Self` alone is enough — and a base that is overloaded, since - /// neither can be copied wholesale. Neither of those says anything about the return type, and - /// a base that declares one is still where a missing annotation should draw from: /// `def __len__(self): ...` under `Collection.__len__` returns `int`, not the `None` that - /// running its placeholder body would give. + /// running its placeholder body would give. this reads `base`'s whole signature, so a base + /// that writes no return type either supplies the one it recovers in turn fn overridden_return_type( - self, db: &'db dyn Db, env: &ProgramEnvironment<'db>, + base: FunctionType<'db>, ) -> OverriddenReturnType<'db> { - let Some(base) = self.overridden_method(db, env) else { - return OverriddenReturnType::NotOverridden; - }; if base.signature(db).overloads.len() != 1 { // which type an overloaded base returns depends on the arguments, so there is no one // type to copy return OverriddenReturnType::Inexpressible; } - let return_ty = base_raw_signature(db, base).return_ty; - if return_ty.is_unknown() { - return OverriddenReturnType::NotOverridden; - } - // a type variable in the base's return type is bound to the base method's scope, so - // copying it here would silently rebind it - if any_over_type(db, env, return_ty, false, |ty| { - matches!(ty, Type::TypeVar(_)) - }) { - return OverriddenReturnType::Inexpressible; - } - OverriddenReturnType::Declared(return_ty) + OverriddenReturnType::of(db, env, base_raw_signature(db, base).return_ty) } /// basedpython: how to name the method this one overrides, for a diagnostic about it. @@ -1820,35 +1845,6 @@ impl<'db> OverloadLiteral<'db> { self.node(db, self.file(db), &module).is_asserts_return } - /// basedpython: whether a return type nobody wrote is read off the body. - /// - /// It always is, except for `__new__`. Construction *reads* `__new__`'s return type - /// structurally — it decides whether the result is an instance and so whether `__init__` runs - /// at all — and the usual body, `return super().__new__(cls)`, does not currently describe - /// one: `object.__new__(cls)` solves to `Never`. That is a pre-existing gap, reproducible with - /// `infer-unannotated-signatures` turned off, so recovering a return type here would only - /// hand that gap the power to change what a call means. - /// - /// A base method that already answers the question is the other exception, even when what it - /// answered cannot be written down here — an overloaded base, or one whose return type is in - /// terms of its own type variables. The body must not answer differently from the base, so - /// where the base's answer cannot be copied the return type stays gradual. - /// - /// [`OverloadLiteral::raw_signature`] and [`OverloadLiteral::return_type_without_annotation`] - /// both consult this, so the signature and the `redundant-return-annotation` lint cannot - /// disagree about whether the body was the source. - fn recovers_return_type_from_body( - self, - db: &'db dyn Db, - env: &ProgramEnvironment<'db>, - ) -> bool { - self.name(db) != "__new__" - && !matches!( - self.overridden_return_type(db, env), - OverriddenReturnType::Inexpressible - ) - } - /// basedpython: the signature of the method that this method overrides, resolved by looking /// `self`'s name up in the enclosing class's MRO starting *after* the class itself — the same /// lookup `super()` performs. @@ -2310,6 +2306,15 @@ pub(super) fn same_module_uncached_raw_signature<'db>( .last_definition_raw_signature(db, return_callable_typevar_scope) } +/// basedpython: whether `function` is a `decorator def`, which the parser marks with a decorator +/// no source can spell. +fn is_decorator_keyword_def(function: &ast::StmtFunctionDef) -> bool { + function.decorator_list.iter().any(|decorator| { + matches!(&decorator.expression, ast::Expr::Name(name) + if name.ctx.is_invalid() && name.id == "decorator_keyword") + }) +} + /// basedpython: whether a function in `file` with no annotations is given the signature its body /// determines. /// @@ -2329,9 +2334,10 @@ pub(crate) fn infers_unannotated_signatures(db: &dyn Db, file: File) -> bool { } /// basedpython: what the method an unannotated method overrides says about its return type. -#[derive(Debug, Copy, Clone, PartialEq, Eq)] -enum OverriddenReturnType<'db> { - /// nothing in the MRO after this class defines the name, so the base has nothing to say +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash, get_size2::GetSize, salsa::SalsaValue)] +pub(crate) enum OverriddenReturnType<'db> { + /// the base has nothing to say: nothing in the MRO after this class defines the name, or what + /// does leaves its return type gradual NotOverridden, /// the base declares this, and it can stand as this method's own return type Declared(Type<'db>), @@ -2342,6 +2348,41 @@ enum OverriddenReturnType<'db> { Inexpressible, } +impl<'db> OverriddenReturnType<'db> { + /// what a base whose return type is `return_ty` says about the return type of an override + pub(crate) fn of(db: &'db dyn Db, env: &ProgramEnvironment<'db>, return_ty: Type<'db>) -> Self { + if return_ty.is_unknown() { + return Self::NotOverridden; + } + // a type variable in the base's return type is bound to the base method's scope, so + // copying it here would silently rebind it + if any_over_type(db, env, return_ty, false, |ty| { + matches!(ty, Type::TypeVar(_)) + }) { + return Self::Inexpressible; + } + Self::Declared(return_ty) + } +} + +/// basedpython: where a function's return type comes from; see +/// [`OverloadLiteral::return_type_source`] +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) enum ReturnTypeSource<'db> { + /// the function writes it: an annotation, or an `asserts` clause, which returns `None` + Written, + /// the getter of an untyped property is held to the type its initialiser declares + PropertyInitialiser(Type<'db>), + /// the method this one overrides supplies it + Overridden(Type<'db>), + /// the union of what the sibling `@overload`s return + Overloads(Type<'db>), + /// the function's own body + Body, + /// nothing supplies one, so it stays gradual + Gradual, +} + /// Indicates whether a method is explicitly or implicitly abstract. #[derive(Debug, Copy, Clone, PartialEq, Eq, Hash, get_size2::GetSize)] pub(super) enum AbstractMethodKind { @@ -2513,6 +2554,12 @@ impl<'db> FunctionType<'db> { .map_or(&[], |updated| &updated.specializations) } + /// Whether a decorator replaced this function's signature or its implementation's, as opposed + /// to only specializing them. + pub(super) fn has_replaced_signature(self, db: &'db dyn Db) -> bool { + self.updated_signature(db).is_some() || self.updated_implementation_callables(db).is_some() + } + pub(super) fn updated_signature(self, db: &'db dyn Db) -> Option<&'db CallableSignature<'db>> { self.updated_signatures(db) .as_deref() diff --git a/crates/ty_python_semantic/src/types/ide_support.rs b/crates/ty_python_semantic/src/types/ide_support.rs index 6c5afb02f4..936e09032b 100644 --- a/crates/ty_python_semantic/src/types/ide_support.rs +++ b/crates/ty_python_semantic/src/types/ide_support.rs @@ -8,7 +8,9 @@ use crate::reachability::is_range_reachable; use crate::types::EnumLiteralType; use crate::types::call::bind::CheckTypesMode; use crate::types::call::{CallArguments, CallError, MatchedArgument}; -use crate::types::class::{DynamicClassAnchor, DynamicEnumAnchor, DynamicNamedTupleAnchor}; +use crate::types::class::{ + DynamicClassAnchor, DynamicEnumAnchor, DynamicNamedTupleAnchor, type_as_abstract_method, +}; use crate::types::constraints::ConstraintSetBuilder; use crate::types::context_sensitive::for_each_candidate; use crate::types::dedicated::django; @@ -19,19 +21,21 @@ use crate::types::extensions::{ use crate::types::function::FunctionDecorators; use crate::types::generics::GenericContext; use crate::types::implicit_names::{ImplicitNamePosition, implicit_name}; -use crate::types::infer::{infer_definition_types, nearest_enclosing_function}; +use crate::types::infer::{ + infer_definition_types, nearest_enclosing_function, original_class_type, +}; use crate::types::list_members::all_end_of_scope_members; use crate::types::literal::LiteralValueTypeKind; -use crate::types::overrides::is_constructor_like_method; +use crate::types::overrides::{OverriddenDeclaration, OverrideStep, overridden_declarations}; use crate::types::receivers; use crate::types::repeated_underscore::{UnderscoreLowering, UnderscoreRefusal}; use crate::types::signatures::{ Parameter, ParameterKind, ParametersKind, ReturnCallableTypeVarScope, Signature, }; use crate::types::{ - CallDunderError, CallableTypes, ClassBase, ClassLiteral, KnownClass, KnownFunction, KnownUnion, - PropertyAccessorRole, SpecialFormType, Type, TypeContext, TypeVarBoundOrConstraints, - TypeVarVariance, binding_type, + CallDunderError, CallableTypes, ClassBase, ClassLiteral, ClassType, KnownClass, KnownFunction, + KnownUnion, PropertyAccessorRole, SpecialFormType, Type, TypeContext, + TypeVarBoundOrConstraints, TypeVarVariance, binding_type, }; use crate::{Db, HasDefinition, HasType, ProgramEnvironment, SemanticModel}; use ruff_db::files::{File, FileRange}; @@ -45,7 +49,8 @@ use ty_module_resolver::{ ImportingFile, Module, ModuleName, ResolverFile, resolve_module_confident, }; use ty_python_core::definition::{Definition, DefinitionKind}; -use ty_python_core::scope::FileScopeId; +use ty_python_core::scope::{FileScopeId, ScopeId}; +use ty_python_core::symbol::ScopedSymbolId; use ty_python_core::{ProgramFile, attribute_scopes, semantic_index, use_def_map}; mod data_flow; @@ -57,6 +62,7 @@ use crate::types::definition_resolution::{ self, find_symbol_in_scope, resolve_definition, user_visible_definitions, }; pub use crate::types::definition_resolution::{ImportAliasResolution, ResolvedDefinition}; +pub use crate::types::overrides::is_constructor_like_method; pub use data_flow::{ConditionVerdict, DataFlow, ValueVerdict, data_flow}; pub use stub_mapping::map_stub_definition; pub use unreachable_code::{UnreachableKind, UnreachableRange, unreachable_ranges}; @@ -2909,45 +2915,265 @@ pub fn implicit_enum_member_value<'db>( metadata.value_type(db, env, &name) } -/// basedpython: the superclass whose member `name` the class member `member` -/// overrides, when `member` is a method that is not already marked `override`. +/// basedpython: the superclass whose member the method `function` overrides, +/// when it is a method that is not already marked `override` /// -/// Mirrors the `missing-override-decorator` lint's notion of an override, so the -/// hint appears exactly where writing `override` would silence that lint. +/// the first superclass the override checks' walk up the MRO, +/// `overridden_declarations`, finds, which is the one `missing-override-decorator` +/// names, so the hint appears exactly where writing `override` would silence +/// that lint pub fn inferred_override<'db>( - db: &'db dyn Db, - env: &ProgramEnvironment<'db>, - class: Type<'db>, - member: Type<'db>, - name: &str, + model: &SemanticModel<'db>, + function: &ast::StmtFunctionDef, ) -> Option> { - let Type::FunctionLiteral(function) = member else { + let db = model.db(); + let env = model.program_environment(); + let definition = function.definition(model); + let Type::FunctionLiteral(function_type) = binding_type(db, definition) else { return None; }; - if function.has_known_decorator(db, FunctionDecorators::OVERRIDE) - || is_constructor_like_method(name) - || is_mangled_private(name) + if function_type.has_known_decorator(db, FunctionDecorators::OVERRIDE) + || is_constructor_like_method(&function.name) { return None; } - let Type::ClassLiteral(class) = class else { - return None; + let class = declaring_class(db, definition)?; + overridden_declarations(db, &env, class, &function.name.id) + .into_iter() + .find_map(|step| match step { + OverrideStep::Overrides(OverriddenDeclaration { superclass, .. }) => { + Some(Type::from(superclass)) + } + OverrideStep::Dynamic | OverrideStep::TypedDict => None, + }) +} + +/// basedpython: a binding or declaration a class body makes of one of the +/// class's members: a `def`, a nested class, an import, or any other name the +/// body binds or declares +#[derive(Debug, Clone)] +pub struct ClassMemberDefinition<'db> { + definition: Definition<'db>, + /// the member's name + pub name: Name, + /// the name of the class whose body makes the definition + pub class_name: Name, + /// where the definition names the member + pub name_range: TextRange, +} + +/// basedpython: every binding and declaration the class bodies of `file` make of +/// their members, as the semantic index records them, in source order +/// +/// these are the members the override checks walk: a binding or declaration in +/// code that is never reached makes no member +pub fn class_member_definitions<'db>( + db: &'db dyn Db, + file: ProgramFile<'db>, +) -> Vec> { + let index = semantic_index(db, file); + let module = parsed_module(db, file.python_file(db)).load(db); + let mut members = Vec::new(); + for scope in index.scope_ids() { + let Some(class_node) = scope.node(db).as_class() else { + continue; + }; + let class_name = &class_node.node(&module).name.id; + let file_scope = scope.file_scope_id(db); + let table = index.place_table(file_scope); + let mut seen = FxHashSet::default(); + for (symbol, declarations, bindings) in + index.use_def_map(file_scope).all_reachable_symbols() + { + let name = table.symbol(symbol).name(); + let definitions = declarations + .filter_map(|declaration| declaration.declaration.definition()) + .chain(bindings.filter_map(|binding| binding.binding.definition())); + for definition in definitions { + if !definition.kind(db).is_user_visible() || !seen.insert(definition) { + continue; + } + members.push(ClassMemberDefinition { + definition, + name: name.clone(), + class_name: class_name.clone(), + name_range: definition.focus_range(db, &module).range(), + }); + } + } + } + members.sort_by_key(|member| member.name_range.start()); + members +} + +/// basedpython: the superclass members a class member definition overrides +/// directly, and whether the member is itself abstract +#[derive(Debug, Clone, Default)] +pub struct MemberOverrides<'db> { + /// whether the member is itself abstract, in the sense of + /// [`OverriddenMember::is_abstract`]: an abstract member that overrides an + /// abstract one still leaves it to be implemented + pub is_abstract: bool, + /// what it overrides directly, in MRO order + pub overridden: Vec>, +} + +/// basedpython: a superclass member that a class member overrides +#[derive(Debug, Clone)] +pub struct OverriddenMember<'db> { + /// the superclass that declares the member + pub superclass: ClassLiteral<'db>, + /// the superclass's name + pub superclass_name: Name, + /// where the superclass's body writes it. for a property, the accessors that + /// play the overriding `def`'s part when any does: a setter's is the setter. + /// empty when the superclass synthesizes the member + pub definitions: Vec>, + /// whether the superclass synthesizes the member rather than writing it in + /// its body, as a dataclass does its `__init__` + pub synthesized: bool, + /// whether the member is abstract where the superclass declares it, an + /// `@abstractmethod` or a protocol method with no implementation, so that a + /// member overriding it implements it rather than replacing an + /// implementation: what `abstract-instantiation` counts as abstract + pub is_abstract: bool, +} + +/// basedpython: the superclass members that the class member `member` overrides +/// directly, in MRO order +/// +/// what counts as an override is the override checks' own walk up the MRO, +/// `overridden_declarations`, from the class as the checks see it: +/// undecorated, and specialized to its own type parameters. "directly" is what +/// that walk finds less what is reached through what it found first: a member +/// overriding `B.f`, where `B.f` overrides `A.f`, overrides `B.f`, as `A.f` is +/// `B.f`'s to override. a class with several bases that each declare the member +/// overrides each of them +/// +/// empty for a member that overrides nothing, including a `private` one, which +/// python mangles into a name of its own +pub fn overridden_members<'db>( + db: &'db dyn Db, + member: &ClassMemberDefinition<'db>, +) -> MemberOverrides<'db> { + let ClassMemberDefinition { + definition, name, .. + } = member; + let env = ProgramEnvironment::from_scope(definition.scope(db)); + let Some(class) = declaring_class(db, *definition) else { + return MemberOverrides::default(); + }; + + // a getter overrides a getter and a setter a setter + let accessor_role = if matches!(definition.kind(db), DefinitionKind::Function(_)) + && let Some(property) = binding_type(db, *definition).as_property_instance() + { + property.accessor_role(db, *definition) + } else { + None }; + let mut overridden: Vec> = Vec::new(); + let mut found: Vec> = Vec::new(); + for step in overridden_declarations(db, &env, class, name) { + let OverrideStep::Overrides(OverriddenDeclaration { + superclass, + scope, + symbol, + .. + }) = step + else { + continue; + }; + // reached through a member already found, which overrides it in turn + if found + .iter() + .any(|nearer| nearer.is_subclass_of(db, &env, superclass)) + { + continue; + } + found.push(superclass); + let is_abstract = is_abstract_in(db, &env, superclass, name); + let definitions = symbol + .map(|symbol| { + let written = written_member_definitions(db, scope, symbol, accessor_role); + if written.is_empty() { + // a setter overriding a property whose superclass has only a + // getter overrides that property all the same + written_member_definitions(db, scope, symbol, None) + } else { + written + } + }) + .unwrap_or_default(); + let superclass = superclass.class_literal(db); + overridden.push(OverriddenMember { + superclass, + superclass_name: superclass.name(db).clone(), + definitions, + synthesized: symbol.is_none(), + is_abstract, + }); + } + MemberOverrides { + is_abstract: is_abstract_in(db, &env, class, name), + overridden, + } +} + +/// where a superclass's body, `scope`, writes its member `symbol`: each `def`, +/// nested class, import or name the body binds or declares it with, a `def` +/// with overloads as its implementation. for a property, only the accessors +/// that play `accessor_role`'s part +fn written_member_definitions<'db>( + db: &'db dyn Db, + scope: ScopeId<'db>, + symbol: ScopedSymbolId, + accessor_role: Option, +) -> Vec> { + let use_def = use_def_map(db, scope); + reachable_implementation_definitions( + db, + use_def + .reachable_symbol_declarations(symbol) + .filter_map(|declaration| declaration.declaration.definition()) + .chain( + use_def + .reachable_symbol_bindings(symbol) + .filter_map(|binding| binding.binding.definition()), + ), + ) + .into_iter() + .filter(|definition| property_accessor_role_matches(db, *definition, accessor_role)) + .filter_map(|definition| match definition.kind(db) { + DefinitionKind::Function(_) => member_implementation_definition(db, definition), + _ => Some(ResolvedDefinition::Definition(definition)), + }) + .collect() +} + +/// the class whose body makes `definition`, as the override checks see it: +/// undecorated, and specialized to its own type parameters. `None` when +/// `definition` is not made in a class body +fn declaring_class<'db>(db: &'db dyn Db, definition: Definition<'db>) -> Option> { + let scope = definition.scope(db); + let class_node = scope.node(db).as_class()?; + let class = semantic_index(db, scope.program_file(db)).try_definition(class_node)?; + Some(original_class_type(db, class)?.identity_specialization(db)) +} + +/// whether `class`'s own member `name`, as its body leaves it bound, is abstract +fn is_abstract_in<'db>( + db: &'db dyn Db, + env: &ProgramEnvironment<'db>, + class: ClassType<'db>, + name: &Name, +) -> bool { class - .default_specialization(db) - .iter_mro(db) - .skip(1) - .filter_map(ClassBase::into_class) - .find(|superclass| { - !superclass - .own_class_member(db, env, None, name) - .inner - .place - .is_undefined() - }) - .map(Type::from) + .own_class_member(db, env, None, name) + .ignore_possibly_undefined() + .is_some_and(|ty| type_as_abstract_method(db, ty, class).is_some()) } /// The enum members `target` admits under their bare name, each as its name diff --git a/crates/ty_python_semantic/src/types/infer/builder/function.rs b/crates/ty_python_semantic/src/types/infer/builder/function.rs index 91770ec965..64e9c46a1e 100644 --- a/crates/ty_python_semantic/src/types/infer/builder/function.rs +++ b/crates/ty_python_semantic/src/types/infer/builder/function.rs @@ -859,7 +859,7 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { scope_id.to_scope_id(db, self.program_file()), function, scope_id.is_generator_function(self.index), - can_implicitly_return_none(db, self.index.use_def_map(scope_id)), + self.index.use_def_map(scope_id), |expr| self.expression_type(expr), ) }); diff --git a/crates/ty_python_semantic/src/types/inferred_signature.rs b/crates/ty_python_semantic/src/types/inferred_signature.rs index c62025c49e..8c5f483864 100644 --- a/crates/ty_python_semantic/src/types/inferred_signature.rs +++ b/crates/ty_python_semantic/src/types/inferred_signature.rs @@ -89,7 +89,7 @@ pub(crate) fn inferred_return_type<'db>( body_scope, node, file_scope_id.is_generator_function(index), - can_implicitly_return_none(db, index.use_def_map(file_scope_id)), + index.use_def_map(file_scope_id), |expr| inference.expression_type(expr), ) } @@ -136,19 +136,25 @@ fn divergence_bounded<'db>( /// runs from inside the very scope that would be re-entered. The two share this so they cannot /// drift: the lint advises deleting an annotation, and a body type it disagreed with would make /// that advice silently change the function's type. +/// +/// A `return` that can never be reached hands nothing back, so its value is left out, as the end +/// of the body is when control cannot fall off it. A body whose every `return` and whose end are +/// unreachable returns `Never`: a call to it cannot return, and the checker ends the flow there. pub(crate) fn return_type_from_body<'db>( db: &'db dyn Db, env: &ProgramEnvironment<'db>, body_scope: ScopeId<'db>, node: &ast::StmtFunctionDef, is_generator: bool, - can_implicitly_return_none: bool, + use_def: &UseDefMap<'db>, expression_type: impl Fn(&Expr) -> Type<'db>, ) -> Type<'db> { + let can_implicitly_return_none = can_implicitly_return_none(db, use_def); let mut collector = BodyValueCollector { db, env: env.clone(), body_scope, + use_def, // a `type def` hands back a type rather than a value, so there is no value whose members // a caller could read and nothing for a structural claim about them to describe carries_member_narrowing: !ast::helpers::is_type_def(node), @@ -2396,17 +2402,18 @@ pub(crate) fn can_implicitly_return_none<'db>(db: &'db dyn Db, use_def: &UseDefM /// /// A nested function or lambda has its own body scope and its own returns, so /// this never descends into one. A class body is skipped for the same reason. -struct BodyValueCollector<'db, F> { +struct BodyValueCollector<'a, 'db, F> { db: &'db dyn Db, env: ProgramEnvironment<'db>, body_scope: ScopeId<'db>, + use_def: &'a UseDefMap<'db>, carries_member_narrowing: bool, expression_type: F, returns: Vec>, yields: Vec>, } -impl<'db, F> Visitor<'_> for BodyValueCollector<'db, F> +impl<'db, F> Visitor<'_> for BodyValueCollector<'_, 'db, F> where F: Fn(&Expr) -> Type<'db>, { @@ -2419,6 +2426,15 @@ where Stmt::FunctionDef(_) | Stmt::ClassDef(_) => {} Stmt::Return(ret) => { + let reachability = self.use_def.return_reachability(ret.range()); + if self + .use_def + .reachability_constraints() + .evaluate(self.db, self.use_def.predicates(), reachability) + .is_always_false() + { + return; + } let returned = match ret.value.as_deref() { Some(value) => { self.visit_expr(value); diff --git a/crates/ty_python_semantic/src/types/normal_completion.rs b/crates/ty_python_semantic/src/types/normal_completion.rs new file mode 100644 index 0000000000..b5862bac78 --- /dev/null +++ b/crates/ty_python_semantic/src/types/normal_completion.rs @@ -0,0 +1,912 @@ +//! basedpython: whether a call to a function whose return type nobody wrote can return, read off +//! the function's control flow before its recovered return type +//! +//! a statement-level call ends the flow when the function it calls cannot return: when its return +//! type is `Never`. a function annotated `-> NoReturn` says so in its signature. a function whose +//! return type is recovered from its body (`infer-unannotated-signatures`, which `sound-types` +//! implies) returns `Never` when no `return` in the body and not the end of the body can be +//! reached — but recovering that type means inferring the whole body, and everything the body +//! reads. for a call made at module level, what the body reads includes the module's own globals, +//! whose visibility at the end of the module depends on whether that very call returns. every body +//! involved then joins one salsa cycle, which is iterated as a whole until it settles +//! +//! so the question is first answered here from the body's control flow: the reachability decision +//! diagram the semantic index already builds for the body +//! ([`UseDefMap::normal_exit_reachability`]), decided without inferring any expression: +//! +//! - `raise`, `return`, `break` and `continue` are already part of the diagram's shape +//! - a condition is known only when it is a literal the type of which would be known too +//! (`while True`), and ambiguous otherwise +//! - a statement-level call in the body is decided only when its callee resolves through the +//! semantic index alone to a single function, by that function's written return type, or by its +//! body under this same reading. resolution follows a name to the one definition its scope has +//! for it, an import to the module it names, and `self`, `cls` or a class's own name to a method +//! of a class with no bases. every other call is ambiguous +//! +//! the only types read are written return annotations: a callee's, and those of the methods it +//! overrides, which [`OverloadLiteral::return_type_source`] consults before the body. no module +//! global is looked up by type, and no reachability is decided with types, so none of this depends +//! on the visibility of a module's globals at its end: the cycle has nothing to close through +//! +//! the reading has three answers. a body no `return` and no end of which can be reached cannot +//! complete, and a call to it does not return. a body that always reaches one completes, and a +//! call to it returns. a body whose answer depends on something only types decide leaves the +//! question open, and the checker reads the recovered return type instead, as it reads any other +//! callee's. only then is the body inferred, and a cycle through the module's globals is one the +//! answer really depends on +//! +//! wherever types would be needed to say more, the reading is ambiguous, and ternary logic is +//! monotone in it, so its two definite answers are ones the types agree with. a body it finds +//! cannot complete has no reachable `return` and no reachable end by types either, and the +//! recovered return type leaves out what cannot be reached, so it is `Never`. a body it finds +//! always completes reaches a `return` or its end by types too. the one place the two can still +//! differ is the value a reached `return` hands back, which would need the value's type: +//! `return sys.exit()` is a `return` that is reached, and the call is read as returning although +//! its recovered return type is `Never` +//! +//! recursion between bodies is resolved as a least fixed point: a body's answer starts as "cannot +//! complete" and can only grow towards "completes". that is the reading execution agrees with — a +//! call that returns is a finite one, so `def f(): f()` never returns, it ends in `RecursionError` +//! — and it never calls a body terminal that has a completing path, because such a path is found +//! without assuming anything about the body it is in +//! +//! a cycle can also pass through the checker's own inference, when a callee's written annotation +//! names a global whose reachability depends on the call. there too, every call the least fixed +//! point calls terminal is one whose body cannot complete, or one whose annotation declares +//! `NoReturn` and whose body the checker holds to that. what the fixed point leaves out is only +//! code that runs after such a call returns, so if one of them did return, the first to do so +//! would have to have returned through a path that is blocked by an earlier one + +use ruff_db::files::File; +use ruff_db::parsed::parsed_module; +use ruff_python_ast as ast; +use ruff_python_ast::name::Name; +use std::cell::Cell; + +use rustc_hash::{FxHashMap, FxHashSet}; +use ty_module_resolver::{ + ImportingFile, Module, ModuleName, resolve_module, resolve_module_for_import_from, +}; +use ty_python_core::definition::{Definition, DefinitionKind, ParameterDefinitionNodeKind}; +use ty_python_core::predicate::CallableAndCallExpr; +use ty_python_core::reachability_constraints::ScopedReachabilityConstraintId; +use ty_python_core::scope::{NodeWithScopeRef, ScopeId}; +use ty_python_core::{ + ProgramFile, Truthiness, UseDefMap, global_scope, semantic_index, use_def_map, +}; + +use crate::place::implicit_builtins_symbol_scope; +use crate::reachability::{PredicateReading, evaluate_with_reading}; +use crate::types::Type; +use crate::types::definition_resolution::{find_symbol_in_scope, visible_definitions_for_name}; +use crate::types::function::{ + FunctionForm, FunctionHeader, FunctionType, OverloadLiteral, OverriddenReturnType, + ReturnTypeSource, infers_unannotated_signatures, +}; +use crate::types::signatures::function_signature_expression_type; +use crate::{Db, ProgramEnvironment, attribute_assignments, attribute_declarations}; + +/// whether a statement-level call to `function` can return, when the function's return type is +/// recovered from a body: `AlwaysFalse` when it cannot, `AlwaysTrue` when it does, and `Ambiguous` +/// when only the recovered return type can tell +/// +/// `None` when the return type is not recovered from a body: it is written down, or supplied by an +/// overload group, a replaced signature or a `decorator def`. the signature answers those without +/// reading a body, and the caller reads it +pub(crate) fn body_recovered_call_returns<'db>( + db: &'db dyn Db, + function: FunctionType<'db>, + is_await: bool, +) -> Option { + if function.has_replaced_signature(db) { + return None; + } + let (overloads, implementation) = function.overloads_and_implementation(db); + let implementation = implementation.filter(|_| overloads.is_empty())?; + unannotated_call_returns(db, implementation, is_await) +} + +/// [`body_recovered_call_returns`] for the one definition of a function +/// +/// a cycle through it starts from the answer that the call returns, which is what the checker +/// answers for any call it cannot see through +#[salsa::tracked( + returns(copy), + cycle_initial = |_, _, _, _| Some(Truthiness::AlwaysTrue), + heap_size = ruff_memory_usage::heap_size, +)] +fn unannotated_call_returns<'db>( + db: &'db dyn Db, + overload: OverloadLiteral<'db>, + is_await: bool, +) -> Option { + let header = overload.header(db); + if header.form == FunctionForm::DecoratorKeyword { + return None; + } + // an `async def` called without `await` hands back a coroutine, whatever its body does + if header.is_async && !is_await { + return Some(Truthiness::AlwaysTrue); + } + let env = ProgramEnvironment::from_scope(overload.body_scope(db)); + let base_is_undecided = Cell::new(false); + let source = overload.return_type_source(db, &env, false, |base| { + overridden_return_type(db, base).unwrap_or_else(|| { + base_is_undecided.set(true); + OverriddenReturnType::NotOverridden + }) + }); + // what the base supplies is not known without its recovered return type, so neither is where + // this function's return type comes from + if base_is_undecided.get() { + return Some(Truthiness::Ambiguous); + } + match source { + ReturnTypeSource::Body => Some(body_call_returns(db, overload.body_scope(db))), + ReturnTypeSource::Overridden(return_ty) => Some(Truthiness::from( + !return_ty.is_equivalent_to(db, &env, Type::Never), + )), + ReturnTypeSource::Written + | ReturnTypeSource::PropertyInitialiser(_) + | ReturnTypeSource::Overloads(_) + | ReturnTypeSource::Gradual => None, + } +} + +/// what the method `base` supplies as the return type of a method that overrides it, told without +/// recovering any return type; `None` when that cannot be told +fn overridden_return_type<'db>( + db: &'db dyn Db, + base: FunctionType<'db>, +) -> Option> { + let (overloads, implementation) = base.overloads_and_implementation(db); + // which type an overloaded base returns depends on the arguments, so the override's return + // type stays gradual + match implementation.filter(|_| overloads.is_empty()) { + Some(base) => return_type_seen_by_overrides(db, base), + None => Some(OverriddenReturnType::Inexpressible), + } +} + +/// [`overridden_return_type`] for the one definition of a base method +/// +/// this follows [`OverloadLiteral::overridden_return_type`], which reads the base's signature and +/// so recovers the base's own return type from its body when it writes none. here that body is +/// read the same way as any other body. one that cannot complete returns `Never`. one that +/// reaches a `return` or its end returns a type this reading does not recover, which stands as +/// the gradual type: it is only used to tell whether a call returns, and a call returning the +/// gradual type is taken to. one that only types can read leaves the answer open +/// +/// a cycle through it starts from the open answer, which sends the caller to the types +#[salsa::tracked( + returns(copy), + cycle_initial = |_, _, _| None, + heap_size = ruff_memory_usage::heap_size, +)] +fn return_type_seen_by_overrides<'db>( + db: &'db dyn Db, + base: OverloadLiteral<'db>, +) -> Option> { + let env = ProgramEnvironment::from_scope(base.body_scope(db)); + let base_is_undecided = Cell::new(false); + let source = base.return_type_source(db, &env, false, |base| { + overridden_return_type(db, base).unwrap_or_else(|| { + base_is_undecided.set(true); + OverriddenReturnType::NotOverridden + }) + }); + if base_is_undecided.get() { + return None; + } + Some(match source { + ReturnTypeSource::Written => match written_return_type(db, base.definition(db)) { + Some(return_ty) => OverriddenReturnType::of(db, &env, return_ty), + None => OverriddenReturnType::NotOverridden, + }, + ReturnTypeSource::PropertyInitialiser(return_ty) + | ReturnTypeSource::Overridden(return_ty) + | ReturnTypeSource::Overloads(return_ty) => OverriddenReturnType::of(db, &env, return_ty), + ReturnTypeSource::Body => match body_call_returns(db, base.body_scope(db)) { + Truthiness::AlwaysTrue => OverriddenReturnType::Declared(Type::unknown()), + Truthiness::AlwaysFalse => OverriddenReturnType::Declared(Type::Never), + Truthiness::Ambiguous => return None, + }, + ReturnTypeSource::Gradual => OverriddenReturnType::NotOverridden, + }) +} + +/// the type the return annotation of the function `definition` declares, `None` when it has none +/// +/// an `asserts` clause names a place rather than a type, and such a function returns `None` +#[salsa::tracked( + returns(copy), + cycle_initial = |_, _, _| Some(Type::unknown()), + heap_size = ruff_memory_usage::heap_size, +)] +fn written_return_type<'db>(db: &'db dyn Db, definition: Definition<'db>) -> Option> { + let module = parsed_module(db, definition.python_file(db)).load(db); + let function = definition.kind(db).as_function()?.node(&module); + let returns = function.returns.as_deref()?; + Some(if function.is_asserts_return { + Type::none(db, &ProgramEnvironment::from_scope(definition.scope(db))) + } else { + function_signature_expression_type(db, definition, returns) + }) +} + +/// whether a call to the function whose body is `body_scope` returns, given that the call is not +/// an `async def` left unawaited: a generator function's call builds the generator without running +/// the body, and any other call returns when the body completes +fn body_call_returns<'db>(db: &'db dyn Db, body_scope: ScopeId<'db>) -> Truthiness { + let index = semantic_index(db, body_scope.program_file(db)); + if body_scope.file_scope_id(db).is_generator_function(index) { + return Truthiness::AlwaysTrue; + } + function_body_completion(db, body_scope) +} + +/// whether control leaves the function body `body_scope` normally — reaches a `return` or the end +/// of the body — as read off its control flow, without inferring any expression in it +/// +/// `AlwaysFalse` when it cannot and `AlwaysTrue` when it always can. `Ambiguous` when that depends +/// on something only types can decide, such as a condition that is not a literal +/// +/// recursion is solved as a least fixed point: the provisional answer is `AlwaysFalse`, and an +/// iteration can only move it towards `AlwaysTrue`. see the module documentation for why that is +/// sound +#[salsa::tracked( + returns(copy), + cycle_initial = |_, _, _| Truthiness::AlwaysFalse, + cycle_fn = |_, _, previous: &Truthiness, result: Truthiness, _| previous.or(result), + heap_size = ruff_memory_usage::heap_size, +)] +pub(crate) fn function_body_completion<'db>( + db: &'db dyn Db, + body_scope: ScopeId<'db>, +) -> Truthiness { + let use_def = use_def_map(db, body_scope); + let mut reading = ControlFlowReading { + db, + env: ProgramEnvironment::from_scope(body_scope), + scope: body_scope, + use_def, + continuations: FxHashMap::default(), + }; + evaluate_with_reading( + db, + use_def.reachability_constraints(), + use_def.predicates(), + use_def.normal_exit_reachability(), + &mut reading, + ) +} + +/// decides a scope's reachability predicates from control flow alone +struct ControlFlowReading<'a, 'db> { + db: &'db dyn Db, + env: ProgramEnvironment<'db>, + scope: ScopeId<'db>, + use_def: &'a UseDefMap<'db>, + /// a sequence of `finally` suites nests each one's continuation in the next, so each is + /// evaluated once + continuations: FxHashMap, +} + +impl<'db> PredicateReading<'db> for ControlFlowReading<'_, 'db> { + fn types(&self) -> Option<&ProgramEnvironment<'db>> { + None + } + + fn statement_call(&mut self, call: CallableAndCallExpr<'db>) -> Truthiness { + let db = self.db; + let module = parsed_module(db, self.scope.python_file(db)).load(db); + let callee = call.callable.node_ref(db).node(&module); + let mut resolver = CalleeResolver { + db, + env: &self.env, + visited: FxHashSet::default(), + }; + match resolver.resolve(call.callable.scope(db), callee) { + Some(Resolved::Function(definition)) => { + call_truthiness(db, &self.env, definition, false, call.is_await) + } + Some(Resolved::Method(definition)) => { + call_truthiness(db, &self.env, definition, true, call.is_await) + } + Some(Resolved::Module(_) | Resolved::Class(_) | Resolved::Receiver(_)) | None => { + Truthiness::Ambiguous + } + } + } + + fn continuation( + &mut self, + scope: ScopeId<'db>, + continuation: ScopedReachabilityConstraintId, + ) -> Truthiness { + if scope != self.scope { + return Truthiness::Ambiguous; + } + if let Some(reachability) = self.continuations.get(&continuation) { + return *reachability; + } + let use_def = self.use_def; + let reachability = evaluate_with_reading( + self.db, + use_def.reachability_constraints(), + use_def.predicates(), + continuation, + self, + ); + self.continuations.insert(continuation, reachability); + reachability + } +} + +/// what the `def` line of a function says, and which body it runs +#[derive(Debug, Clone, PartialEq, Eq, Hash, get_size2::GetSize, salsa::SalsaValue)] +struct CalleeFacts<'db> { + header: FunctionHeader, + body_scope: ScopeId<'db>, + /// its return annotation, if it writes one + writes_return_type: bool, + name: Name, + /// it has decorators, and each is the builtin `staticmethod` or `classmethod` + only_method_kind_decorators: bool, + has_decorators: bool, +} + +/// [`CalleeFacts`] of the function `definition`, `None` when it is not a function +#[salsa::tracked(returns(ref), heap_size = ruff_memory_usage::heap_size)] +fn callee_facts<'db>(db: &'db dyn Db, definition: Definition<'db>) -> Option> { + let file = definition.program_file(db); + let module = parsed_module(db, file.python_file(db)).load(db); + let node = definition.kind(db).as_function()?.node(&module); + let env = ProgramEnvironment::from_scope(definition.scope(db)); + let only_method_kind_decorators = node.decorator_list.iter().all(|decorator| { + ["staticmethod", "classmethod"].into_iter().any(|name| { + is_builtin_name(db, &env, definition.scope(db), &decorator.expression, name) + }) + }); + Some(CalleeFacts { + header: FunctionHeader::of(node), + body_scope: semantic_index(db, file) + .node_scope(NodeWithScopeRef::Function(node)) + .to_scope_id(db, file), + writes_return_type: node.returns.is_some(), + name: node.name.id.clone(), + only_method_kind_decorators, + has_decorators: !node.decorator_list.is_empty(), + }) +} + +/// whether a call to the function `definition` returns: `AlwaysFalse` when it cannot, `AlwaysTrue` +/// when it does, and `Ambiguous` when only types can tell +/// +/// `in_class` is whether the function was found in a class body, where `staticmethod` and +/// `classmethod` leave it the same function to call; any other decorator could replace it +fn call_truthiness<'db>( + db: &'db dyn Db, + env: &ProgramEnvironment<'db>, + definition: Definition<'db>, + in_class: bool, + is_await: bool, +) -> Truthiness { + let Some(facts) = callee_facts(db, definition).as_ref() else { + return Truthiness::Ambiguous; + }; + let decorators_keep_the_function = + !facts.has_decorators || (in_class && facts.only_method_kind_decorators); + if !decorators_keep_the_function { + return Truthiness::Ambiguous; + } + // an `async def` called without `await` hands back a coroutine, whatever its body does + if facts.header.is_async && !is_await { + return Truthiness::AlwaysTrue; + } + + if facts.writes_return_type { + // what awaiting something other than a coroutine hands back is a question for its type + if is_await && !facts.header.is_async { + return Truthiness::Ambiguous; + } + let Some(return_ty) = written_return_type(db, definition) else { + return Truthiness::Ambiguous; + }; + // a type variable in the return type is solved by the call's arguments, which can solve + // it to `Never` + return if return_ty.is_equivalent_to(db, env, Type::Never) { + Truthiness::AlwaysFalse + } else if return_ty.has_typevar(db, env) { + Truthiness::Ambiguous + } else { + Truthiness::AlwaysTrue + }; + } + + // without the setting, what an unannotated function returns is the gradual type + if !infers_unannotated_signatures(db, definition.file(db)) { + return Truthiness::AlwaysTrue; + } + // `__new__`'s return type is never read off its body, and what a method of a class with no + // bases returns when it overrides something `object` defines comes from `object` + if facts.name == "__new__" + || facts.header.form == FunctionForm::DecoratorKeyword + || (in_class && defined_on_object(db, env, &facts.name)) + { + return Truthiness::Ambiguous; + } + body_call_returns(db, facts.body_scope) +} + +/// what a callee expression resolves to through the semantic index +#[derive(Debug, Clone, Copy)] +enum Resolved<'db> { + Module(ProgramFile<'db>), + Class(Definition<'db>), + Function(Definition<'db>), + /// a function found in the body of a class + Method(Definition<'db>), + /// the first parameter of a method of the class `Definition`: its instance or, in a + /// `classmethod`, the class itself + Receiver(Definition<'db>), +} + +/// resolves callee expressions through the semantic index alone: no expression's type is inferred +/// +/// a name, or a member of a module or class, resolves only when every definition the scope has for +/// it is one definition. the type of a name the scope binds more than once may be a union, which a +/// call is never terminal through +struct CalleeResolver<'a, 'db> { + db: &'db dyn Db, + env: &'a ProgramEnvironment<'db>, + /// imports already followed, which ends a cycle of re-exports + visited: FxHashSet>, +} + +impl<'db> CalleeResolver<'_, 'db> { + fn resolve(&mut self, scope: ScopeId<'db>, expr: &ast::Expr) -> Option> { + match expr { + ast::Expr::Name(name) => self.resolve_name(scope, name.id.as_str()), + ast::Expr::Attribute(attribute) => match self.resolve(scope, &attribute.value)? { + Resolved::Module(file) => self.module_member(file, attribute.attr.as_str()), + Resolved::Class(class) | Resolved::Receiver(class) => { + self.class_member(class, attribute.attr.as_str()) + } + Resolved::Function(_) | Resolved::Method(_) => None, + }, + _ => None, + } + } + + fn resolve_name(&mut self, scope: ScopeId<'db>, name: &str) -> Option> { + let db = self.db; + let mut definitions = visible_definitions_for_name(db, scope, name); + if definitions.is_empty() { + let builtins = implicit_builtins_symbol_scope(db, self.env, name)?; + definitions = find_symbol_in_scope(db, builtins, name) + .into_iter() + .filter(|definition| definition.is_reexported(db)) + .collect(); + } + let definition = single(definitions)?; + self.resolve_definition(definition, name) + } + + fn resolve_definition( + &mut self, + definition: Definition<'db>, + name: &str, + ) -> Option> { + let db = self.db; + if !self.visited.insert(definition) { + return None; + } + match definition.kind(db) { + DefinitionKind::Function(_) => Some(Resolved::Function(definition)), + DefinitionKind::Class(_) => Some(Resolved::Class(definition)), + DefinitionKind::Parameter(parameter) => self.receiver(definition, parameter), + DefinitionKind::Import(_) + | DefinitionKind::ImportFrom(_) + | DefinitionKind::StarImport(_) => match import_target(db, definition).as_ref()? { + ImportTarget::Module(file) => Some(self.module(*file)), + ImportTarget::Member { + module, + name, + importing_file, + } => self.imported_member(*module, *importing_file, name), + // a star import binds every name it finds, so the one looked up is its own + ImportTarget::Members { + module, + importing_file, + } => self.imported_member(*module, *importing_file, name), + }, + _ => None, + } + } + + fn module(&self, file: File) -> Resolved<'db> { + Resolved::Module(ProgramFile::new(self.db, file, self.env.program(self.db))) + } + + /// `from module import name`: the module's own `name`, or its submodule of that name + fn imported_member( + &mut self, + module: Module<'db>, + importing_file: File, + name: &str, + ) -> Option> { + let db = self.db; + if let Some(file) = module.file(db) { + let file = ProgramFile::new(db, file, self.env.program(db)); + if !self.module_members(file, name).is_empty() { + return self.module_member(file, name); + } + } + let mut submodule = module.name(db).clone(); + submodule.extend(&ModuleName::new(name)?); + let importing = ImportingFile::File(importing_file, self.env.resolver_environment(db)); + Some(self.module(resolve_module(db, importing, &submodule)?.file(db)?)) + } + + /// the definitions of `name` in `file`'s global scope that an import can see: in a stub, only + /// the ones it re-exports + fn module_members(&self, file: ProgramFile<'db>, name: &str) -> Vec> { + let db = self.db; + let is_stub = file.file(db).is_stub(db); + find_symbol_in_scope(db, global_scope(db, file), name) + .into_iter() + .filter(|definition| !is_stub || definition.is_reexported(db)) + .collect() + } + + fn module_member(&mut self, file: ProgramFile<'db>, name: &str) -> Option> { + let definition = single(self.module_members(file, name))?; + self.resolve_definition(definition, name) + } + + /// a function defined in the body of `class`, when nothing else can supply the member: the + /// class has no bases, no decorators and no metaclass, and no method assigns the name on an + /// instance + fn class_member(&mut self, class: Definition<'db>, name: &str) -> Option> { + let db = self.db; + let body = class_body(db, class)?; + if !body.is_closed { + return None; + } + let assigned_on_instance = + attribute_assignments(db, body.scope, name).any(|(mut bindings, _)| { + bindings.any(|binding| binding.binding.definition().is_some()) + }) || attribute_declarations(db, body.scope, name).any(|(mut declarations, _)| { + declarations.any(|declaration| declaration.declaration.definition().is_some()) + }); + if assigned_on_instance { + return None; + } + let definition = single(find_symbol_in_scope(db, body.scope, name))?; + definition + .kind(db) + .as_function() + .map(|_| Resolved::Method(definition)) + } + + /// the first parameter of a method: `self`, or `cls` in a `classmethod` + /// + /// a name resolves within its own file, so the method is in the file being read + fn receiver( + &self, + definition: Definition<'db>, + parameter: &ParameterDefinitionNodeKind, + ) -> Option> { + let db = self.db; + let ParameterDefinitionNodeKind::Parameter(parameter) = parameter else { + return None; + }; + let file = definition.program_file(db); + let module = parsed_module(db, file.python_file(db)).load(db); + let index = semantic_index(db, file); + let function_scope = definition.scope(db).file_scope_id(db); + let function = index + .scope(function_scope) + .node() + .as_function()? + .node(&module); + let first = function + .parameters + .posonlyargs + .iter() + .chain(&function.parameters.args) + .next()?; + if first.parameter.name.as_str() != parameter.node(&module).parameter.name.as_str() { + return None; + } + let class_scope = index.parent_scope_id(function_scope)?; + let class = index.scope(class_scope).node().as_class()?; + // a `staticmethod` has no receiver, and any other decorator could have changed what the + // first parameter is bound to + let class_body = class_scope.to_scope_id(db, file); + let receives = match function.decorator_list.as_slice() { + [] => true, + [decorator] => is_builtin_name( + db, + self.env, + class_body, + &decorator.expression, + "classmethod", + ), + _ => false, + }; + if !receives { + return None; + } + index.try_definition(class).map(Resolved::Receiver) + } +} + +/// what an import definition binds, read once so that following an import into another module +/// does not tie the reader to that module's syntax tree +#[derive(Debug, Clone, PartialEq, Eq, Hash, salsa::SalsaValue)] +enum ImportTarget<'db> { + /// `import a.b` binds `a`, and `import a.b as c` binds `a.b` + Module(File), + /// `from module import name`, in `importing_file` + Member { + module: Module<'db>, + name: Name, + importing_file: File, + }, + /// `from module import *`, in `importing_file` + Members { + module: Module<'db>, + importing_file: File, + }, +} + +#[salsa::tracked(returns(ref))] +fn import_target<'db>(db: &'db dyn Db, definition: Definition<'db>) -> Option> { + let file = definition.program_file(db); + let env = ProgramEnvironment::from_scope(definition.scope(db)); + let importing_file = file.file(db); + let importing = ImportingFile::File(importing_file, env.resolver_environment(db)); + let module = parsed_module(db, file.python_file(db)).load(db); + match definition.kind(db) { + DefinitionKind::Import(import) => { + let alias = import.alias(&module); + let module_name = if alias.asname.is_some() { + ModuleName::new(&alias.name)? + } else { + ModuleName::new(alias.name.split('.').next()?)? + }; + Some(ImportTarget::Module( + resolve_module(db, importing, &module_name)?.file(db)?, + )) + } + DefinitionKind::ImportFrom(import) => Some(ImportTarget::Member { + module: resolve_module_for_import_from(db, importing, import.import(&module))?, + name: import.alias(&module).name.id.clone(), + importing_file, + }), + DefinitionKind::StarImport(import) => Some(ImportTarget::Members { + module: resolve_module_for_import_from(db, importing, import.import(&module))?, + importing_file, + }), + _ => None, + } +} + +/// the body of a class, and whether nothing but it can supply a member: the class has no +/// decorators and no bases but `object`, and passes no keyword such as `metaclass` +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, get_size2::GetSize, salsa::SalsaValue)] +struct ClassBody<'db> { + scope: ScopeId<'db>, + is_closed: bool, +} + +#[salsa::tracked(returns(copy), heap_size = ruff_memory_usage::heap_size)] +fn class_body<'db>(db: &'db dyn Db, class: Definition<'db>) -> Option> { + let file = class.program_file(db); + let module = parsed_module(db, file.python_file(db)).load(db); + let node = class.kind(db).as_class()?.node(&module); + let env = ProgramEnvironment::from_scope(class.scope(db)); + let is_closed = node.decorator_list.is_empty() + && node.arguments.as_deref().is_none_or(|arguments| { + arguments.keywords.is_empty() + && arguments + .args + .iter() + .all(|base| is_builtin_name(db, &env, class.scope(db), base, "object")) + }); + Some(ClassBody { + scope: semantic_index(db, file) + .node_scope(NodeWithScopeRef::Class(node)) + .to_scope_id(db, file), + is_closed, + }) +} + +/// the one element of `definitions`, if it has exactly one +fn single<'db>(definitions: impl IntoIterator>) -> Option> { + let mut definitions = definitions.into_iter(); + let definition = definitions.next()?; + definitions.next().is_none().then_some(definition) +} + +/// whether `expr`, written in `scope`, names the builtin class `name`: a name no enclosing scope +/// defines, which the builtins module defines as a class +fn is_builtin_name<'db>( + db: &'db dyn Db, + env: &ProgramEnvironment<'db>, + scope: ScopeId<'db>, + expr: &ast::Expr, + name: &str, +) -> bool { + let ast::Expr::Name(expr) = expr else { + return false; + }; + expr.id.as_str() == name + && visible_definitions_for_name(db, scope, name).is_empty() + && builtin_class(db, env, name).is_some() +} + +/// the builtins module's definition of the class `name` +fn builtin_class<'db>( + db: &'db dyn Db, + env: &ProgramEnvironment<'db>, + name: &str, +) -> Option> { + let builtins = implicit_builtins_symbol_scope(db, env, name)?; + let definition = single(find_symbol_in_scope(db, builtins, name))?; + definition + .kind(db) + .as_class() + .is_some() + .then_some(definition) +} + +/// whether `object` defines a member `name`, which every class inherits +fn defined_on_object<'db>(db: &'db dyn Db, env: &ProgramEnvironment<'db>, name: &str) -> bool { + let Some(object) = builtin_class(db, env, "object").and_then(|object| class_body(db, object)) + else { + return true; + }; + !find_symbol_in_scope(db, object.scope, name).is_empty() +} + +#[cfg(test)] +mod tests { + use std::fmt::Write as _; + + use ruff_db::files::system_path_to_file; + use ruff_db::system::DbWithWritableSystem as _; + use ruff_db::testing::assert_function_query_was_not_run_by_name; + use ty_python_core::ProgramFile; + + use crate::db::tests::setup_db; + use crate::place::{DefinedPlace, Place, global_symbol}; + + /// A package that, at module level and behind a condition, calls the first of `length` + /// unannotated functions in a submodule, each of which calls the next both as a statement and + /// in its `return`. The last one ends with `last`. Returns whether `AFTER`, bound after the + /// call, is reachable. + /// + /// Recovering the chain's return types infers every body in it, and the last one reads the + /// package back, so all of them joined one cycle with the package's globals: at 500 links + /// checking the package took 45 CPU seconds in a release build, and far longer in a debug one. + fn chain(length: usize, last: &str) -> anyhow::Result<(crate::db::tests::TestDb, bool)> { + let mut db = setup_db(); + db.write_dedented( + "/src/pkg/__init__.py", + r#" + import os + + VALUE = os.environ.get("VALUE") + + if "FLAG" in os.environ: + import pkg.sub as sub + + sub.f0() + AFTER = 1 + "#, + )?; + let mut sub = String::from("import sys\n\nimport pkg\n\n"); + for link in 0..length { + if link + 1 < length { + let next = link + 1; + writeln!( + sub, + "def f{link}():\n f{next}()\n return f{next}()\n\n" + )?; + } else { + writeln!(sub, "def f{link}():\n {last}\n")?; + } + } + db.write_file("/src/pkg/sub.py", sub)?; + + let file = system_path_to_file(&db, "/src/pkg/__init__.py")?; + let file = ProgramFile::new(&db, file, db.program_environment().program(&db)); + // a binding the flow never reaches has the type `Never` + let Place::Defined(DefinedPlace { ty: after, .. }) = + global_symbol(&db, file, "AFTER").place + else { + anyhow::bail!("`AFTER` is always bound in the source"); + }; + let after = !after.is_never(); + Ok((db, after)) + } + + /// runs `test` on a thread with the stack ty's checking threads have + /// + /// each link of a chain is read by a query the reading of the link before it waits on, so how + /// deep the reading goes grows with the chain, and a chain of 500 links needs more than the + /// stack a test thread starts with + fn on_a_checking_stack( + test: impl FnOnce() -> anyhow::Result<()> + Send + 'static, + ) -> anyhow::Result<()> { + std::thread::Builder::new() + .stack_size(ruff_db::STACK_SIZE) + .spawn(test)? + .join() + .unwrap_or_else(|panic| std::panic::resume_unwind(panic)) + } + + #[test] + fn a_long_chain_is_read_without_recovering_its_return_types() -> anyhow::Result<()> { + on_a_checking_stack(|| { + let (mut db, after) = chain(500, "return pkg.VALUE")?; + assert!(after, "the chain returns, so `AFTER` is reachable"); + let events = db.take_salsa_events(); + assert_function_query_was_not_run_by_name(&db, "inferred_return_type", None, &events); + assert_reads_each_link_once(&db, &events, 500); + Ok(()) + }) + } + + #[test] + fn a_long_chain_that_exits_never_returns() -> anyhow::Result<()> { + on_a_checking_stack(|| { + let (mut db, after) = chain(500, "sys.exit(1)")?; + assert!(!after, "the chain never returns, so `AFTER` is unreachable"); + let events = db.take_salsa_events(); + assert_function_query_was_not_run_by_name(&db, "inferred_return_type", None, &events); + assert_reads_each_link_once(&db, &events, 500); + Ok(()) + }) + } + + /// Each link's body is read once, and neither the reading nor the module-level call it + /// answers is part of a cycle that has to be iterated: the cost is linear in the chain. + fn assert_reads_each_link_once( + db: &crate::db::tests::TestDb, + events: &[salsa::Event], + length: usize, + ) { + salsa::Database::attach(db, |_| { + let executions = |query: &str| { + events + .iter() + .filter(|event| match event.kind { + salsa::EventKind::WillExecute { database_key } => { + format!("{database_key:?}").starts_with(query) + } + _ => false, + }) + .count() + }; + assert_eq!(executions("function_body_completion("), length); + let iterated = events + .iter() + .filter_map(|event| match event.kind { + salsa::EventKind::WillIterateCycle { database_key, .. } => { + Some(format!("{database_key:?}")) + } + _ => None, + }) + .filter(|head| { + head.starts_with("function_body_completion(") + || head.starts_with("analyze_non_terminal_call(") + }) + .collect::>(); + assert_eq!(iterated, Vec::::new()); + }); + } +} diff --git a/crates/ty_python_semantic/src/types/overrides.rs b/crates/ty_python_semantic/src/types/overrides.rs index 1cb093a5de..48c764a407 100644 --- a/crates/ty_python_semantic/src/types/overrides.rs +++ b/crates/ty_python_semantic/src/types/overrides.rs @@ -437,6 +437,134 @@ fn conflicting_named_tuple_field_in_mro<'db>( None } +/// a superclass's declaration of a member that a class's own member of the same +/// name overrides +pub(crate) struct OverriddenDeclaration<'db> { + /// the superclass that declares it + pub(crate) superclass: ClassType<'db>, + /// the superclass's body scope + pub(crate) scope: ScopeId<'db>, + /// the member's symbol in [`Self::scope`], or `None` when the superclass does + /// not write the member in its body but synthesizes it, as a dataclass does + /// its `__init__` + pub(crate) symbol: Option, + /// whether the superclass synthesizes the member, and as what kind of class + pub(crate) method_kind: MethodKind<'db>, + /// the member's type on an instance of the superclass + pub(crate) ty: Type<'db>, +} + +/// one step of the walk [`overridden_declarations`] makes up a class's MRO +pub(crate) enum OverrideStep<'db> { + /// a base whose members cannot be known, `Any`, an unknown or a divergent + /// base, so anything may be declared there + Dynamic, + /// a `TypedDict` base, whose keys are not members a class body overrides + TypedDict, + /// a superclass that declares the member + Overrides(OverriddenDeclaration<'db>), +} + +/// every superclass declaration the member `name` of `class` overrides, in MRO +/// order, with the dynamic and `TypedDict` bases met along the way +/// +/// this is what the override checks mean by one member overriding another, and +/// the one place that says it: the checks walk it, and so do the IDE's answers +/// about overrides, where the members a member overrides are and the `override` +/// inlay hint +/// +/// a superclass counts when its body binds or declares `name`, or when it +/// synthesizes it. the walk ends at the first superclass that does so without an +/// instance of it having the member, and `Protocol` and `Generic` are passed +/// over +pub(crate) fn overridden_declarations<'db>( + db: &'db dyn Db, + env: &ProgramEnvironment<'db>, + class: ClassType<'db>, + name: &Name, +) -> Vec> { + let mut steps = Vec::new(); + + // basedpython: a `private` member is emitted under a name python mangles per + // class, so it is a member of its own rather than an override of anything. a + // `protected` one keeps one name across the hierarchy, so it overrides like + // any other member + if is_private_to(db, class, name) { + return steps; + } + + for class_base in class.iter_mro(db).skip(1) { + let superclass = match class_base { + ClassBase::Protocol | ClassBase::Generic => continue, + ClassBase::Any | ClassBase::Dynamic(_) | ClassBase::Divergent(_) => { + steps.push(OverrideStep::Dynamic); + continue; + } + ClassBase::TypedDict(_) => { + steps.push(OverrideStep::TypedDict); + continue; + } + ClassBase::Class(class) => class, + }; + + // basedpython: a `private` member of a superclass is emitted under a + // name mangled with that superclass's own, so nothing a subclass + // declares can override it. it is not a contract for this member to + // meet, and the subclass's member is not missing an `@override` + if is_private_to(db, superclass, name) { + continue; + } + + let Some((superclass_literal, superclass_specialization)) = + superclass.static_class_literal(db) + else { + continue; + }; + let superclass_scope = superclass_literal.body_scope(db); + let superclass_symbol_table = place_table(db, superclass_scope); + let superclass_symbol_id = superclass_symbol_table.symbol_id(name); + + let mut method_kind = MethodKind::default(); + + // If the member is not defined on the class itself, skip it + if let Some(id) = superclass_symbol_id { + let superclass_symbol = superclass_symbol_table.symbol(id); + if !(superclass_symbol.is_bound() || superclass_symbol.is_declared()) { + continue; + } + } else { + if superclass_literal + .own_synthesized_member(db, env, superclass_specialization, None, name) + .is_none() + { + continue; + } + method_kind = CodeGeneratorKind::from_class(db, superclass_literal.into()) + .map(MethodKind::Synthesized) + .unwrap_or_default(); + } + + let superclass_instance_member = Type::instance(db, env, superclass).member(db, env, name); + let Place::Defined(DefinedPlace { + ty: superclass_type, + .. + }) = superclass_instance_member.place + else { + // If not defined on any superclass, no point in continuing to walk up the MRO + break; + }; + + steps.push(OverrideStep::Overrides(OverriddenDeclaration { + superclass, + scope: superclass_scope, + symbol: superclass_symbol_id, + method_kind, + ty: superclass_type, + })); + } + steps +} + fn check_class_declaration<'db>( context: &InferContext<'db, '_>, configuration: OverrideRulesConfig, @@ -645,12 +773,6 @@ fn check_class_declaration<'db>( let mut missing_override_target: Option> = None; let mut overridden_final_method = None; let mut overridden_final_variable: Option<(ClassType<'db>, Option>)> = None; - // basedpython: a `private` member is emitted under a name python mangles per - // class, so it is a member of its own rather than an override of anything. a - // `protected` one keeps one name across the hierarchy, so it overrides like - // any other member - let is_private_member = is_private_to(db, class, &member.name); - // basedpython: a member emitted under a different name from the member it // inherits under the same written name does not override it — it sits // beside it, and the inherited one still answers. that is a keyword declaring @@ -685,353 +807,298 @@ fn check_class_declaration<'db>( let mut immediate_parent_method: Option<(ClassType<'db>, Type<'db>)> = None; let mut immediate_parent_variable_kind: Option<(ClassType<'db>, VariableKind)> = None; - if !is_private_member { - for class_base in class.iter_mro(db).skip(1) { - let superclass = match class_base { - ClassBase::Protocol | ClassBase::Generic => continue, - ClassBase::Any | ClassBase::Dynamic(_) => { - has_dynamic_superclass = true; - continue; - } - ClassBase::Divergent(_) => { - has_dynamic_superclass = true; - continue; - } - ClassBase::TypedDict(_) => { - has_typeddict_in_mro = true; - continue; - } - ClassBase::Class(class) => class, - }; - - // basedpython: a `private` member of a superclass is emitted under a - // name mangled with that superclass's own, so nothing a subclass - // declares can override it. it is not a contract for this member to - // meet, and the subclass's member is not missing an `@override` - if is_private_to(db, superclass, &member.name) { + for step in overridden_declarations(db, env, class, &member.name) { + let OverriddenDeclaration { + superclass, + scope: superclass_scope, + symbol: superclass_symbol_id, + method_kind, + ty: superclass_type, + } = match step { + OverrideStep::Dynamic => { + has_dynamic_superclass = true; continue; } - - let Some((superclass_literal, superclass_specialization)) = - superclass.static_class_literal(db) - else { + OverrideStep::TypedDict => { + has_typeddict_in_mro = true; continue; - }; - let superclass_scope = superclass_literal.body_scope(db); - let superclass_symbol_table = place_table(db, superclass_scope); - let superclass_symbol_id = superclass_symbol_table.symbol_id(&member.name); - - let mut method_kind = MethodKind::default(); - - // If the member is not defined on the class itself, skip it - if let Some(id) = superclass_symbol_id { - let superclass_symbol = superclass_symbol_table.symbol(id); - if !(superclass_symbol.is_bound() || superclass_symbol.is_declared()) { - continue; - } - } else { - if superclass_literal - .own_synthesized_member(db, env, superclass_specialization, None, &member.name) - .is_none() - { - continue; - } - method_kind = CodeGeneratorKind::from_class(db, superclass_literal.into()) - .map(MethodKind::Synthesized) - .unwrap_or_default(); } + OverrideStep::Overrides(declaration) => declaration, + }; - let superclass_instance_member = - Type::instance(db, env, superclass).member(db, env, &member.name); - let Place::Defined(DefinedPlace { - ty: superclass_type, - .. - }) = superclass_instance_member.place - else { - // If not defined on any superclass, no point in continuing to walk up the MRO - break; - }; + subclass_overrides_superclass_declaration = true; - subclass_overrides_superclass_declaration = true; + // Record the first overridden superclass member that is subject to the missing override + // decorator check so that we can later confirm that the overriding definition is indeed + // marked with the decorator. + if configuration.check_missing_overrides() + && missing_override_target.is_none() + && !is_constructor_like_method(&member.name) + { + missing_override_target = Some(MissingOverrideTarget::for_superclass( + db, + superclass, + superclass_scope, + superclass_symbol_id, + )); + } - // Record the first overridden superclass member that is subject to the missing override - // decorator check so that we can later confirm that the overriding definition is indeed - // marked with the decorator. - if configuration.check_missing_overrides() - && missing_override_target.is_none() - && !is_constructor_like_method(&member.name) - { - missing_override_target = Some(MissingOverrideTarget::for_superclass( - db, - superclass, - superclass_scope, - superclass_symbol_id, - )); - } + // Record the first superclass that defines this method as the "immediate parent method" + if immediate_parent_method.is_none() { + immediate_parent_method = Some((superclass, superclass_type)); + } - // Record the first superclass that defines this method as the "immediate parent method" - if immediate_parent_method.is_none() { - immediate_parent_method = Some((superclass, superclass_type)); - } + if (configuration.check_final_method_overridden() && overridden_final_method.is_none()) + || (configuration.check_final_variable_overridden() + && overridden_final_variable.is_none()) + { + let own_class_member = superclass.own_class_member(db, env, None, &member.name); - if (configuration.check_final_method_overridden() && overridden_final_method.is_none()) - || (configuration.check_final_variable_overridden() - && overridden_final_variable.is_none()) - { - let own_class_member = superclass.own_class_member(db, env, None, &member.name); - - if configuration.check_final_method_overridden() { - overridden_final_method = overridden_final_method.or_else(|| { - let superclass_symbol_id = superclass_symbol_id?; - - // TODO: `@final` should be more like a type qualifier: - // we should also recognise `@final`-decorated methods that don't end up - // as being function- or property-types (because they're wrapped by other - // decorators that transform the type into something else). - let underlying_functions = extract_underlying_functions( - db, - own_class_member.ignore_possibly_undefined()?, - ); + if configuration.check_final_method_overridden() { + overridden_final_method = overridden_final_method.or_else(|| { + let superclass_symbol_id = superclass_symbol_id?; - if underlying_functions.iter().any(|function| { - function.has_known_decorator(db, FunctionDecorators::FINAL) - }) && is_function_definition(db, superclass_scope, superclass_symbol_id) - { - Some((superclass, underlying_functions)) - } else { - None - } - }); - } + // TODO: `@final` should be more like a type qualifier: + // we should also recognise `@final`-decorated methods that don't end up + // as being function- or property-types (because they're wrapped by other + // decorators that transform the type into something else). + let underlying_functions = extract_underlying_functions( + db, + own_class_member.ignore_possibly_undefined()?, + ); - if configuration.check_final_variable_overridden() { - overridden_final_variable = overridden_final_variable.or_else(|| { - if !own_class_member - .qualifiers() - .contains(TypeQualifiers::FINAL) - { - return None; - } + if underlying_functions + .iter() + .any(|function| function.has_known_decorator(db, FunctionDecorators::FINAL)) + && is_function_definition(db, superclass_scope, superclass_symbol_id) + { + Some((superclass, underlying_functions)) + } else { + None + } + }); + } - // `let` is read-only, not `final`. but models a - // read-only property, so overriding it is allowed. - if superclass_symbol_id - .is_some_and(|id| *is_let_declaration(db, superclass_scope, id)) - { - return None; - } + if configuration.check_final_variable_overridden() { + overridden_final_variable = overridden_final_variable.or_else(|| { + if !own_class_member + .qualifiers() + .contains(TypeQualifiers::FINAL) + { + return None; + } - // Find the declaration definition in the superclass for the secondary - // annotation. - let superclass_definition = superclass_symbol_id.and_then(|id| { - use_def_map(db, superclass_scope) - .end_of_scope_symbol_declarations(id) - .find_map(|decl| decl.declaration.definition()) - }); + // `let` is read-only, not `final`. but models a + // read-only property, so overriding it is allowed. + if superclass_symbol_id + .is_some_and(|id| *is_let_declaration(db, superclass_scope, id)) + { + return None; + } - Some((superclass, superclass_definition)) + // Find the declaration definition in the superclass for the secondary + // annotation. + let superclass_definition = superclass_symbol_id.and_then(|id| { + use_def_map(db, superclass_scope) + .end_of_scope_symbol_declarations(id) + .find_map(|decl| decl.declaration.definition()) }); - } - } - // ********************************************************** - // Everything below this point in the loop - // is about Liskov Substitution Principle checks - // ********************************************************** - - // Only one Liskov diagnostic should be emitted per each invalid override, - // even if it overrides multiple superclasses incorrectly! - if liskov_diagnostic_emitted { - continue; + Some((superclass, superclass_definition)) + }); } + } - if !configuration.check_liskov_violations() { - continue; - } + // ********************************************************** + // Everything below this point in the loop + // is about Liskov Substitution Principle checks + // ********************************************************** - if configuration.check_attribute_liskov_violations() { - if let Some(superclass_variable_kind) = - effective_superclass_variable_kind(db, superclass, member.name.clone()) - { - if immediate_parent_variable_kind.is_none() { - immediate_parent_variable_kind = - Some((superclass, superclass_variable_kind)); - } + // Only one Liskov diagnostic should be emitted per each invalid override, + // even if it overrides multiple superclasses incorrectly! + if liskov_diagnostic_emitted { + continue; + } - let subclass_kind = *subclass_variable_kind.get_or_insert_with(|| { - variable_kind( - db, - env, - class.own_class_member(db, env, None, &member.name).inner, - subclass_instance_member, - ) - }); + if !configuration.check_liskov_violations() { + continue; + } - if let Some(subclass_kind) = subclass_kind - && subclass_kind != superclass_variable_kind - { - // An unannotated class-body assignment can inherit an overridden `ClassVar` - // declaration instead of introducing a conflicting instance variable. This - // also applies to augmented assignments after the initial class-body - // assignment, e.g. `epilog = "..."; epilog += "..."`. - if subclass_kind == VariableKind::Instance - && superclass_variable_kind == VariableKind::Class - && matches!( - first_reachable_definition.kind(db), - DefinitionKind::Assignment(_) - | DefinitionKind::AugmentedAssignment(_) - ) - { - continue; - } + if configuration.check_attribute_liskov_violations() { + if let Some(superclass_variable_kind) = + effective_superclass_variable_kind(db, superclass, member.name.clone()) + { + if immediate_parent_variable_kind.is_none() { + immediate_parent_variable_kind = Some((superclass, superclass_variable_kind)); + } - if let Some((immediate_parent, immediate_parent_kind)) = - immediate_parent_variable_kind - && immediate_parent != superclass - && immediate_parent.is_subclass_of(db, env, superclass) - && immediate_parent_kind != superclass_variable_kind - { - continue; - } + let subclass_kind = *subclass_variable_kind.get_or_insert_with(|| { + variable_kind( + db, + env, + class.own_class_member(db, env, None, &member.name).inner, + subclass_instance_member, + ) + }); - let superclass_definition = superclass_symbol_id - .and_then(|id| symbol_definition(db, superclass_scope, id)); - report_invalid_attribute_override( - context, - &member.name, - *first_reachable_definition, - superclass, - superclass_definition, - subclass_kind, - superclass_variable_kind, - ); - liskov_diagnostic_emitted = true; + if let Some(subclass_kind) = subclass_kind + && subclass_kind != superclass_variable_kind + { + // An unannotated class-body assignment can inherit an overridden `ClassVar` + // declaration instead of introducing a conflicting instance variable. This + // also applies to augmented assignments after the initial class-body + // assignment, e.g. `epilog = "..."; epilog += "..."`. + if subclass_kind == VariableKind::Instance + && superclass_variable_kind == VariableKind::Class + && matches!( + first_reachable_definition.kind(db), + DefinitionKind::Assignment(_) | DefinitionKind::AugmentedAssignment(_) + ) + { continue; } - } - } - - if !configuration.check_method_liskov_violations() { - continue; - } - let Type::FunctionLiteral(subclass_function) = member.ty else { - continue; - }; - - // Constructor methods are not checked for Liskov compliance - if is_constructor_like_method(&member.name) { - continue; - } - - // Synthesized `__replace__` methods on dataclasses are not checked - if &member.name == "__replace__" - && class_kind.is_some_and(CodeGeneratorKind::is_dataclass_like) - { - continue; - } + if let Some((immediate_parent, immediate_parent_kind)) = + immediate_parent_variable_kind + && immediate_parent != superclass + && immediate_parent.is_subclass_of(db, env, superclass) + && immediate_parent_kind != superclass_variable_kind + { + continue; + } - // basedpython: a reified type-parameter list is part of the - // method's runtime interface — `a.f[int]()` through the base - // dispatches to the override's wrapper. the value-level callable - // check below cannot see it (type parameters are phantom there), - // so check list compatibility first - let superclass_function = match superclass_type { - Type::FunctionLiteral(function) => Some(function), - Type::BoundMethod(bound_method) => Some(bound_method.function(db)), - _ => None, - }; - if let Some(superclass_function) = superclass_function - && let Some(error) = crate::types::reified_infer::reified_override_error( - db, - env, - superclass_function, - subclass_function, - ) - { - report_invalid_reified_override( - context, - &member.name, - *first_reachable_definition, - subclass_function, - superclass, - error, - ); - liskov_diagnostic_emitted = true; - continue; + let superclass_definition = superclass_symbol_id + .and_then(|id| symbol_definition(db, superclass_scope, id)); + report_invalid_attribute_override( + context, + &member.name, + *first_reachable_definition, + superclass, + superclass_definition, + subclass_kind, + superclass_variable_kind, + ); + liskov_diagnostic_emitted = true; + continue; + } } + } - // basedpython: an override may not raise more than what it - // overrides. orthogonal to Liskov compatibility below, so it runs - // whether or not the signatures line up - // only against the nearest superclass defining the method: a more - // distant ancestor's bound is already enforced on that superclass, - // and reporting it again here blames the wrong class - if let Some(superclass_function) = superclass_function - && immediate_parent_method.is_some_and(|(parent, _)| parent == superclass) - { - crate::types::exceptions::check_override_raises( - context, - &member.name, - subclass_function, - superclass_function, - superclass, - ); - } + if !configuration.check_method_liskov_violations() { + continue; + } - let Some((subclass_override_type, superclass_override_type)) = - method_override_types(db, env, type_on_subclass_instance, superclass_type) - else { - continue; - }; + let Type::FunctionLiteral(subclass_function) = member.ty else { + continue; + }; - if subclass_override_type.is_assignable_to(db, env, superclass_override_type) { - continue; - } + // Constructor methods are not checked for Liskov compliance + if is_constructor_like_method(&member.name) { + continue; + } - // If this superclass is not the immediate parent for this method, - // check if the immediate parent itself already has an LSP violation with this ancestor. - // If so, don't report the same violation for the child class -- it would be a false positive - // since the child cannot fix the violation without contradicting its immediate parent's contract. - // See: https://github.com/astral-sh/ty/issues/2000 - if let Some((immediate_parent, immediate_parent_type)) = immediate_parent_method { - if immediate_parent != superclass { - // The immediate parent already defines this method and is different from the - // current ancestor we're checking. Check if the immediate parent's method - // is also incompatible with this ancestor. - if !is_assignable_method_override( - db, - env, - immediate_parent_type, - superclass_type, - ) { - // The immediate parent already has an LSP violation with this ancestor. - // Don't report the same violation for the child. - continue; - } - } - } + // Synthesized `__replace__` methods on dataclasses are not checked + if &member.name == "__replace__" + && class_kind.is_some_and(CodeGeneratorKind::is_dataclass_like) + { + continue; + } - report_invalid_method_override( + // basedpython: a reified type-parameter list is part of the + // method's runtime interface — `a.f[int]()` through the base + // dispatches to the override's wrapper. the value-level callable + // check below cannot see it (type parameters are phantom there), + // so check list compatibility first + let superclass_function = match superclass_type { + Type::FunctionLiteral(function) => Some(function), + Type::BoundMethod(bound_method) => Some(bound_method.function(db)), + _ => None, + }; + if let Some(superclass_function) = superclass_function + && let Some(error) = crate::types::reified_infer::reified_override_error( + db, + env, + superclass_function, + subclass_function, + ) + { + report_invalid_reified_override( context, &member.name, - class, *first_reachable_definition, subclass_function, superclass, - superclass_type, - method_kind, - || { - subclass_override_type.assignability_error_context( - db, - env, - superclass_override_type, - ) - }, + error, ); - liskov_diagnostic_emitted = true; + continue; + } + + // basedpython: an override may not raise more than what it + // overrides. orthogonal to Liskov compatibility below, so it runs + // whether or not the signatures line up + // only against the nearest superclass defining the method: a more + // distant ancestor's bound is already enforced on that superclass, + // and reporting it again here blames the wrong class + if let Some(superclass_function) = superclass_function + && immediate_parent_method.is_some_and(|(parent, _)| parent == superclass) + { + crate::types::exceptions::check_override_raises( + context, + &member.name, + subclass_function, + superclass_function, + superclass, + ); } + + let Some((subclass_override_type, superclass_override_type)) = + method_override_types(db, env, type_on_subclass_instance, superclass_type) + else { + continue; + }; + + if subclass_override_type.is_assignable_to(db, env, superclass_override_type) { + continue; + } + + // If this superclass is not the immediate parent for this method, + // check if the immediate parent itself already has an LSP violation with this ancestor. + // If so, don't report the same violation for the child class -- it would be a false positive + // since the child cannot fix the violation without contradicting its immediate parent's contract. + // See: https://github.com/astral-sh/ty/issues/2000 + if let Some((immediate_parent, immediate_parent_type)) = immediate_parent_method { + if immediate_parent != superclass { + // The immediate parent already defines this method and is different from the + // current ancestor we're checking. Check if the immediate parent's method + // is also incompatible with this ancestor. + if !is_assignable_method_override(db, env, immediate_parent_type, superclass_type) { + // The immediate parent already has an LSP violation with this ancestor. + // Don't report the same violation for the child. + continue; + } + } + } + + report_invalid_method_override( + context, + &member.name, + class, + *first_reachable_definition, + subclass_function, + superclass, + superclass_type, + method_kind, + || { + subclass_override_type.assignability_error_context( + db, + env, + superclass_override_type, + ) + }, + ); + + liskov_diagnostic_emitted = true; } if !subclass_overrides_superclass_declaration && !has_dynamic_superclass { @@ -1508,7 +1575,7 @@ fn report_invalid_attribute_override<'db>( } #[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] -pub(super) enum MethodKind<'db> { +pub(crate) enum MethodKind<'db> { Synthesized(CodeGeneratorKind<'db>), #[default] NotSynthesized, @@ -1865,7 +1932,13 @@ fn stored_member_name(db: &dyn Db, class: ClassType<'_>, name: &str) -> String { } } -pub(crate) fn is_constructor_like_method(name: &str) -> bool { +/// whether a member named `name` is a constructor, or a method the constructors +/// call: `__init__`, `__new__`, `__post_init__`, `__init_subclass__` +/// +/// the override checks do not hold one to the signature of the member it +/// overrides, nor ask it to be marked `@override`: a subclass's constructor +/// takes what that subclass needs +pub fn is_constructor_like_method(name: &str) -> bool { matches!( name, "__init__" | "__new__" | "__post_init__" | "__init_subclass__" diff --git a/crates/ty_server/src/server.rs b/crates/ty_server/src/server.rs index 1544eb82db..7ce88a4a6f 100644 --- a/crates/ty_server/src/server.rs +++ b/crates/ty_server/src/server.rs @@ -24,6 +24,7 @@ mod script_progress; use crate::project_server::Listener as ProjectServerListener; use crate::session::client::Client; pub(crate) use api::Error; +pub(crate) use api::SuspendedWorkspaceRequestKind; pub(crate) use api::{ publish_all_document_diagnostics, publish_diagnostics_if_needed, publish_settings_diagnostics, }; diff --git a/crates/ty_server/src/server/api.rs b/crates/ty_server/src/server/api.rs index af51453fae..239fcea404 100644 --- a/crates/ty_server/src/server/api.rs +++ b/crates/ty_server/src/server/api.rs @@ -27,6 +27,7 @@ use crate::session::client::Client; pub(crate) use diagnostics::{ publish_all_document_diagnostics, publish_diagnostics_if_needed, publish_settings_diagnostics, }; +pub(crate) use requests::SuspendedWorkspaceRequestKind; use ruff_db::panic::PanicError; /// Processes a request from the client to the server. @@ -58,6 +59,9 @@ pub(super) fn request(req: server::Request) -> Task { >( req, BackgroundSchedule::Worker ), + requests::CheckWorkspaceRequestHandler::METHOD => background_request_task::< + requests::CheckWorkspaceRequestHandler, + >(req, BackgroundSchedule::Worker), requests::GotoTypeDefinitionRequestHandler::METHOD => background_document_request_task::< requests::GotoTypeDefinitionRequestHandler, >( @@ -204,6 +208,14 @@ pub(super) fn request(req: server::Request) -> Task { BackgroundSchedule::LatencySensitive, ) } + requests::SuperMembersRequestHandler::METHOD => background_document_request_task::< + requests::SuperMembersRequestHandler, + >(req, BackgroundSchedule::Worker), + requests::DocumentSuperMembersRequestHandler::METHOD => background_document_request_task::< + requests::DocumentSuperMembersRequestHandler, + >( + req, BackgroundSchedule::Worker + ), requests::PrepareTypeHierarchyRequestHandler::METHOD => background_document_request_task::< requests::PrepareTypeHierarchyRequestHandler, >( @@ -560,7 +572,7 @@ fn sync_notification_task( // If there's a pending workspace diagnostic long-polling request, // resume it, but only if the session revision changed (e.g. because some document changed). - session.resume_suspended_workspace_diagnostic_request(client); + session.resume_suspended_workspace_diagnostic_requests(client); })) } @@ -650,7 +662,12 @@ fn respond( Req: RequestHandler, { if let Err(err) = &result { - tracing::error!("An error occurred with request ID {id}: {err}"); + // a deliberate refusal is an answer, not a failure of the server + if matches!(err.code, ErrorCode::RequestFailed) { + tracing::debug!("Request ID {id} was refused: {err}"); + } else { + tracing::error!("An error occurred with request ID {id}: {err}"); + } report_unexpected_failure(client, err, log_guidance); } client.respond(id, result); diff --git a/crates/ty_server/src/server/api/changes.rs b/crates/ty_server/src/server/api/changes.rs index 7171454f05..de46a700c0 100644 --- a/crates/ty_server/src/server/api/changes.rs +++ b/crates/ty_server/src/server/api/changes.rs @@ -15,7 +15,13 @@ use crate::session::Session; use crate::session::client::Client; use crate::system::AnySystemPath; -/// Applies `changes` to every project and refreshes what the client is showing. +/// applies `changes` to every project and, if that changed anything, refreshes what the client is +/// showing +/// +/// only then: a batch of changes is often a write to something no answer depends on (a build's +/// output, an ignored directory, a file nothing imports), and a refresh makes the client ask again +/// for the diagnostics and inlay hints of everything it shows, and poll the workspace's +/// diagnostics again pub(crate) fn apply(session: &mut Session, client: &Client, changes: &[ChangeEvent]) { if changes.is_empty() { return; @@ -28,17 +34,30 @@ pub(crate) fn apply(session: &mut Session, client: &Client, changes: &[ChangeEve .map(|(root, _)| root.clone()) .collect(); + let mut changed = false; for root in roots { tracing::debug!("Applying changes to `{root}`"); - session.apply_changes(client, &AnySystemPath::System(root.clone()), changes); - publish_settings_diagnostics(session, client, root); + let result = session.apply_changes(client, &AnySystemPath::System(root.clone()), changes); + if result.database_changed() { + changed = true; + publish_settings_diagnostics(session, client, root); + } } // a change can move what there is to watch: a new virtual environment, a configuration // naming other search paths session.update_file_watcher(client); + if !changed { + tracing::debug!("The changes changed nothing the session answers from"); + return; + } + + // a workspace diagnostic request waiting for the session to change is answered here rather + // than by a notification's handler: most changes come from the session's own watcher + session.resume_suspended_workspace_diagnostic_requests(client); + if client_capabilities.supports_workspace_diagnostic_refresh() { client.send_request::(session, (), |_, ()| {}); } else { diff --git a/crates/ty_server/src/server/api/requests.rs b/crates/ty_server/src/server/api/requests.rs index 3d8f17a186..4c9d730f2a 100644 --- a/crates/ty_server/src/server/api/requests.rs +++ b/crates/ty_server/src/server/api/requests.rs @@ -16,6 +16,7 @@ mod alignment_groups; mod build_output; mod call_hierarchy_incoming_calls; mod call_hierarchy_outgoing_calls; +mod check_workspace; mod code_action; mod code_action_resolve; mod code_lens; @@ -24,6 +25,7 @@ mod data_flow; mod detached_transpile; mod diagnostic; mod doc_highlights; +mod document_super_members; mod document_symbols; mod entry_point; mod execute_command; @@ -48,6 +50,7 @@ mod semantic_tokens; mod semantic_tokens_range; mod shutdown; mod signature_help; +mod super_members; mod syntax_outline; mod test_items; mod transpile; @@ -62,6 +65,8 @@ pub(super) use alignment_groups::AlignmentGroupsRequestHandler; pub(super) use build_output::BuildOutputRequestHandler; pub(super) use call_hierarchy_incoming_calls::CallHierarchyIncomingCallsRequestHandler; pub(super) use call_hierarchy_outgoing_calls::CallHierarchyOutgoingCallsRequestHandler; +pub(super) use check_workspace::CheckWorkspaceRequestHandler; +pub(crate) use check_workspace::SuspendedWorkspaceRequestKind; pub(super) use code_action::CodeActionRequestHandler; pub(super) use code_action_resolve::CodeActionResolveRequestHandler; pub(super) use code_lens::CodeLensRequestHandler; @@ -69,6 +74,7 @@ pub(super) use completion::CompletionRequestHandler; pub(super) use data_flow::DataFlowRequestHandler; pub(super) use diagnostic::DocumentDiagnosticRequestHandler; pub(super) use doc_highlights::DocumentHighlightRequestHandler; +pub(super) use document_super_members::DocumentSuperMembersRequestHandler; pub(super) use document_symbols::DocumentSymbolRequestHandler; pub(super) use entry_point::EntryPointHandler; pub(super) use execute_command::ExecuteCommand; @@ -93,6 +99,7 @@ pub(super) use semantic_tokens::SemanticTokensRequestHandler; pub(super) use semantic_tokens_range::SemanticTokensRangeRequestHandler; pub(super) use shutdown::ShutdownHandler; pub(super) use signature_help::SignatureHelpRequestHandler; +pub(super) use super_members::SuperMembersRequestHandler; pub(super) use syntax_outline::SyntaxOutlineRequestHandler; pub(super) use test_items::TestItemsHandler; pub(super) use transpile::TranspileRequestHandler; diff --git a/crates/ty_server/src/server/api/requests/check_workspace.rs b/crates/ty_server/src/server/api/requests/check_workspace.rs new file mode 100644 index 0000000000..72279f4476 --- /dev/null +++ b/crates/ty_server/src/server/api/requests/check_workspace.rs @@ -0,0 +1,161 @@ +//! `by/checkWorkspace` — the workspace's diagnostics, answered once they are checked +//! +//! the same report as `workspace/diagnostic`, from the same check, for a client that needs the +//! answer rather than the next change. `workspace/diagnostic` is long-polled: when nothing differs +//! from the result ids the client sent, the request is held open until something does. that is +//! what an editor keeping a problems list current wants, and it leaves a client that asks once — +//! an inspection run over the whole project, a batch run with no editor at all — waiting forever +//! on a project with no diagnostics, because "nothing" is never different from nothing. this one +//! answers as soon as the check is done, unchanged or not +//! +//! the one thing it still waits for is the workspace being checkable: while a script's initial +//! environment is being set up it is held open the way a long poll is, and answered once the +//! environment is there + +use anyhow::anyhow; +use lsp_server::{ErrorCode, RequestId}; +use lsp_types::{ + LspRequestMethod, MessageDirection, Request, WorkspaceDiagnosticParams, + WorkspaceDiagnosticReport, WorkspaceDiagnosticRequest, +}; +use serde_json::json; + +use crate::server::Action; +use crate::server::api::Error; +use crate::server::api::requests::workspace_diagnostic::{WorkspaceCheck, check_workspace}; +use crate::server::api::traits::{ + BackgroundRequestHandler, RequestHandler, RetriableRequestHandler, +}; +use crate::session::client::Client; +use crate::session::{SessionSnapshot, SuspendedWorkspaceDiagnosticRequest}; + +/// the request a client sends to have the workspace checked and be told the result +pub(crate) enum CheckWorkspaceRequest {} + +impl Request for CheckWorkspaceRequest { + type Params = WorkspaceDiagnosticParams; + type Result = WorkspaceDiagnosticReport; + // not a method LSP defines, so it goes across as a custom one, under the same `by/` prefix as + // the rest of them + const METHOD: LspRequestMethod<'static> = LspRequestMethod::Custom("by/checkWorkspace"); + const MESSAGE_DIRECTION: MessageDirection = MessageDirection::ClientToServer; +} + +pub(crate) struct CheckWorkspaceRequestHandler; + +impl RequestHandler for CheckWorkspaceRequestHandler { + type RequestType = CheckWorkspaceRequest; +} + +impl BackgroundRequestHandler for CheckWorkspaceRequestHandler { + // the request is answered by `handle_request`, which holds the request open while the + // workspace is not ready rather than refuse it. this is the answer on its own, without the + // holding + fn run( + snapshot: &SessionSnapshot, + client: &Client, + params: WorkspaceDiagnosticParams, + ) -> crate::server::Result { + match check_workspace(snapshot, client, answered_whole(params)) { + WorkspaceCheck::Checked(report) => Ok(report), + // an answer that would be different a moment later, which is what `ContentModified` + // says + WorkspaceCheck::NotReady => Err(Error::new( + anyhow!("the workspace is not ready to be checked"), + ErrorCode::ContentModified, + )), + WorkspaceCheck::Disabled => Err(not_checked()), + } + } + + fn handle_request( + id: &RequestId, + snapshot: SessionSnapshot, + client: &Client, + params: WorkspaceDiagnosticParams, + ) { + let params = answered_whole(params); + match check_workspace(&snapshot, client, params.clone()) { + WorkspaceCheck::Checked(report) => client.respond(id, Ok(report)), + WorkspaceCheck::NotReady => { + tracing::debug!("Holding `by/checkWorkspace` open until the workspace is ready"); + client.queue_action(Action::SuspendWorkspaceDiagnostics(Box::new( + SuspendedWorkspaceDiagnosticRequest { + id: id.clone(), + kind: SuspendedWorkspaceRequestKind::CheckWorkspace, + params: json!(¶ms), + revision: snapshot.revision(), + }, + ))); + } + WorkspaceCheck::Disabled => { + client.respond::(id, Err(not_checked())); + } + } + } +} + +impl RetriableRequestHandler for CheckWorkspaceRequestHandler { + // an edit that lands mid-check cancels it; the client asked for the answer, not for the + // chance of one, so the check starts again on the edited workspace rather than telling the + // client to ask again + const RETRY_ON_CANCELLATION: bool = true; +} + +/// `params` without the tokens that would have the check report anything but its answer +/// +/// a check that an edit cancels starts again, so anything sent under the client's tokens before +/// the cancellation would be sent a second time: a second first batch of partial results, with +/// reports for the same files from before the edit, and a second start of progress on a token +/// whose progress has ended. so the report comes whole, in the response, and progress, where +/// the client supports it, under a token the server creates for each attempt +fn answered_whole(mut params: WorkspaceDiagnosticParams) -> WorkspaceDiagnosticParams { + params.partial_result_params.partial_result_token = None; + params.work_done_progress_params.work_done_token = None; + params +} + +/// an empty report would say the workspace has no diagnostics, which nobody checked +fn not_checked() -> Error { + Error::new( + anyhow!("the workspace is not checked: the diagnostic mode is not `workspace`"), + ErrorCode::RequestFailed, + ) +} + +/// which of the two requests that wait on the workspace a held one is +#[derive(Debug, Clone, Copy)] +pub(crate) enum SuspendedWorkspaceRequestKind { + /// `workspace/diagnostic`, held until something differs from what the client has + LongPoll, + + /// `by/checkWorkspace`, held until the workspace can be checked + CheckWorkspace, +} + +impl SuspendedWorkspaceRequestKind { + /// the method the request is run again as + pub(crate) fn method(self) -> String { + match self { + Self::LongPoll => WorkspaceDiagnosticRequest::METHOD.to_string(), + Self::CheckWorkspace => CheckWorkspaceRequest::METHOD.to_string(), + } + } + + /// answers the held request `id` when the server shuts down before it could be run again + pub(crate) fn answer_at_shutdown(self, id: &RequestId, client: &Client) { + match self { + // an empty report tells the client nothing changed, which is so + Self::LongPoll => client.respond(id, Ok(WorkspaceDiagnosticReport::default())), + // `by/checkWorkspace` promises the workspace's diagnostics, and an empty report would + // say it has none. it was never checked, so it is not answered as if it was + Self::CheckWorkspace => client.respond::( + id, + Err(Error::new( + anyhow!("the server shut down before the workspace could be checked"), + ErrorCode::RequestFailed, + )), + ), + } + } +} diff --git a/crates/ty_server/src/server/api/requests/document_super_members.rs b/crates/ty_server/src/server/api/requests/document_super_members.rs new file mode 100644 index 0000000000..c373461486 --- /dev/null +++ b/crates/ty_server/src/server/api/requests/document_super_members.rs @@ -0,0 +1,137 @@ +//! `by/documentSuperMembers`: every class member of a document that overrides something, and the +//! superclass members each one overrides +//! +//! `by/superMembers` for the whole document at once, answered from the same code. it is what an +//! editor asks to draw an "overrides" marker beside each member that has one: asking +//! `by/superMembers` of each member instead would be a request per member on every pass over the +//! document, most of them answered with an empty list +//! +//! one document at a time, whole. `textHash` says which revision of this document the answer is +//! about, but the answer also reads every file the document's classes inherit from, and no change +//! to those is announced for this request. a client asks again after it opens, edits or closes +//! any document of the workspace, not only this one, and when the server asks it to refresh its +//! inlay hints, which the server does, for a client that declares inlay hint refresh support, +//! after a change on disk or to the environment that changes what it answers from + +use std::borrow::Cow; + +use lsp_types::{LspRequestMethod, MessageDirection, Range, Request, TextDocumentIdentifier, Uri}; +use ty_ide::document_super_members; +use ty_project::{ProjectDatabase, SemanticDb as _}; + +use crate::document::ToRangeExt; +use crate::server::api::requests::super_members::SuperMember; +use crate::server::api::traits::{ + BackgroundDocumentRequestHandler, RequestHandler, RetriableRequestHandler, +}; +use crate::session::DocumentSnapshot; +use crate::session::client::Client; + +pub(crate) enum DocumentSuperMembersRequest {} + +impl Request for DocumentSuperMembersRequest { + type Params = DocumentSuperMembersParams; + type Result = Option>; + const METHOD: LspRequestMethod<'static> = LspRequestMethod::Custom("by/documentSuperMembers"); + const MESSAGE_DIRECTION: MessageDirection = MessageDirection::ClientToServer; +} + +/// which document to answer for +#[derive(Debug, serde::Deserialize, serde::Serialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub(crate) struct DocumentSuperMembersParams { + text_document: TextDocumentIdentifier, +} + +/// a class member of the document that overrides at least one superclass member +#[derive(Debug, serde::Deserialize, serde::Serialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct OverridingMember { + /// the member's name + name: String, + /// the class whose body declares it + container_name: String, + /// its name where it is declared: the name of a `def` or a nested class, a name a class body + /// assigns, annotates or captures, or an import's alias. the position `by/superMembers` answers + /// the same members at + selection_range: Range, + /// whether the member is itself abstract, as [`SuperMember`]'s `abstract` says of what it + /// overrides + #[serde(rename = "abstract")] + is_abstract: bool, + /// what it overrides directly, in MRO order, as `by/superMembers` answers it. never empty + super_members: Vec, +} + +pub(crate) struct DocumentSuperMembersRequestHandler; + +impl RequestHandler for DocumentSuperMembersRequestHandler { + type RequestType = DocumentSuperMembersRequest; +} + +impl BackgroundDocumentRequestHandler for DocumentSuperMembersRequestHandler { + // what the project's files say, which is the same whether this one is a buffer or the file on + // disk; the answer reads no version + const ANSWERS_CLOSED_DOCUMENTS: bool = true; + + fn document_uri(params: &DocumentSuperMembersParams) -> Cow<'_, Uri> { + Cow::Borrowed(¶ms.text_document.uri) + } + + /// `null` when language services are off, or for a document whose members are not answered + /// here: a notebook, whose ranges are per cell, or a template. otherwise the members in source + /// order, and an empty list when none overrides anything. a member declared more than once is + /// listed at each declaration that names it in the source. constructors and the methods they + /// call, `__init__`, `__new__`, `__post_init__` and `__init_subclass__`, are left out: neither + /// `invalid-method-override` nor `missing-override-decorator` holds one to what it overrides, + /// and `by/superMembers` still answers one when asked + fn run_with_snapshot( + db: &ProjectDatabase, + snapshot: &DocumentSnapshot, + _client: &Client, + _params: DocumentSuperMembersParams, + ) -> crate::server::Result>> { + if snapshot + .workspace_settings() + .is_language_services_disabled() + { + return Ok(None); + } + if snapshot.document().is_cell_or_notebook() || snapshot.is_django_template() { + return Ok(None); + } + let Some(file) = snapshot.to_notebook_or_file(db) else { + return Ok(None); + }; + let encoding = snapshot.encoding(); + + Ok(Some( + document_super_members(db, db.program_file(file)) + .into_iter() + .filter_map(|member| { + let selection_range = member + .name_range + .to_lsp_range(db, file, encoding)? + .local_range(); + let super_members: Vec<_> = member + .super_members + .iter() + .filter_map(|member| SuperMember::from_ide(db, member, encoding)) + .collect(); + if super_members.is_empty() { + return None; + } + Some(OverridingMember { + name: member.name.to_string(), + container_name: member.class.to_string(), + selection_range, + is_abstract: member.is_abstract, + super_members, + }) + }) + .collect(), + )) + } +} + +impl RetriableRequestHandler for DocumentSuperMembersRequestHandler {} diff --git a/crates/ty_server/src/server/api/requests/inlay_hints.rs b/crates/ty_server/src/server/api/requests/inlay_hints.rs index 53101670aa..569ca91c92 100644 --- a/crates/ty_server/src/server/api/requests/inlay_hints.rs +++ b/crates/ty_server/src/server/api/requests/inlay_hints.rs @@ -86,11 +86,11 @@ impl BackgroundDocumentRequestHandler for InlayHintRequestHandler { .to_lsp_position(db, file, snapshot.encoding())? .local_position(), label: inlay_hint_label(&hint.label, db, snapshot.encoding()), - kind: Some(inlay_hint_kind(&hint.kind)), + kind: Some(inlay_hint_kind(hint.kind)), tooltip: None, padding_left: hint.padding_left.then_some(true), padding_right: hint.padding_right.then_some(true), - data: None, + data: Some(kind_tag(hint.kind.setting())), text_edits: Some( hint.text_edits .into_iter() @@ -139,34 +139,28 @@ fn template_inlay_hint( tooltip: None, padding_left: hint.padding_left.then_some(true), padding_right: None, - data: None, + data: Some(kind_tag(hint.kind.setting())), text_edits: None, }) } -fn inlay_hint_kind(inlay_hint_kind: &InlayHintKind) -> lsp_types::InlayHintKind { - match inlay_hint_kind { - InlayHintKind::Type - // basedpython: an inferred exception set is a type, like a return type - | InlayHintKind::Raises - | InlayHintKind::Variance - | InlayHintKind::Reification - | InlayHintKind::TypeArgument - | InlayHintKind::Override - // basedpython-ui: a read set, a dependency set and an invalidation set - // are typing facts about the function, and `unstable` is a modifier - // like `override` - | InlayHintKind::Reads - | InlayHintKind::Stability - | InlayHintKind::DerivedDeps - | InlayHintKind::Invalidates - | InlayHintKind::NumericPromotion - | InlayHintKind::RevealedType - | InlayHintKind::EnumValue => lsp_types::InlayHintKind::Type, - InlayHintKind::CallArgumentName - | InlayHintKind::ImplicitParameter - | InlayHintKind::ImplicitArgument - | InlayHintKind::InheritedDefault => lsp_types::InlayHintKind::Parameter, +/// which kind of hint this is, named by the `inlayHints` option that switches +/// it, as `data.kind` +/// +/// LSP's own `kind` has two values for the two dozen things these hints are, +/// and a client that lets its user treat each kind differently has nothing else +/// to go on but the label, which says what a hint shows rather than what it is. +/// `data` is otherwise only carried back on `inlayHint/resolve`, which this +/// server does not offer +fn kind_tag(setting: &str) -> serde_json::Value { + serde_json::json!({ "kind": setting }) +} + +fn inlay_hint_kind(inlay_hint_kind: InlayHintKind) -> lsp_types::InlayHintKind { + if inlay_hint_kind.is_parameter() { + lsp_types::InlayHintKind::Parameter + } else { + lsp_types::InlayHintKind::Type } } diff --git a/crates/ty_server/src/server/api/requests/prepare_rename.rs b/crates/ty_server/src/server/api/requests/prepare_rename.rs index 47867e10bd..f2e89d2e01 100644 --- a/crates/ty_server/src/server/api/requests/prepare_rename.rs +++ b/crates/ty_server/src/server/api/requests/prepare_rename.rs @@ -5,7 +5,7 @@ use lsp_server::ErrorCode; use lsp_types::{ PrepareRenameParams, PrepareRenamePlaceholder, PrepareRenameRequest, PrepareRenameResult, Uri, }; -use ty_ide::{PreparedTemplateRename, can_rename, django_prepare_rename}; +use ty_ide::{PreparedRename, PreparedTemplateRename, django_prepare_rename, prepare_rename}; use ty_project::{ProjectDatabase, SemanticDb as _}; use crate::document::{PositionExt, ToRangeExt}; @@ -55,16 +55,27 @@ impl BackgroundDocumentRequestHandler for PrepareRenameRequestHandler { let template = snapshot.is_django_template(); - // a python symbol is answered exactly as it was; the django names a - // module writes as plain strings are what is left over once it declines - if !template && let Some(range) = can_rename(db, db.program_file(file), offset) { + // a template holds no python symbol, so only the django names are asked + // about there. in a module, the django names it writes as plain strings + // are what is left over once the python symbol is not one + let python = (!template).then(|| prepare_rename(db, db.program_file(file), offset)); + if let Some(PreparedRename::Ready { range, placeholder }) = python { return Ok(range .to_lsp_range(db, file, snapshot.encoding()) - .map(|lsp_range| lsp_range.local_range().into())); + .map(|lsp_range| { + PrepareRenamePlaceholder::new(lsp_range.local_range(), placeholder).into() + })); } match django_prepare_rename(db, file, offset, template) { - None => Ok(None), + // the editor shows why rather than doing nothing, which from where + // the user sits cannot be told apart from the key not working + None => match python { + Some(PreparedRename::Refused(why)) => { + Err(Error::new(anyhow!(why), ErrorCode::RequestFailed)) + } + Some(PreparedRename::Ready { .. } | PreparedRename::NoSymbol) | None => Ok(None), + }, // the editor shows this rather than offering a rename it could not // finish, which is the whole point of asking first Some(PreparedTemplateRename::Refused(why)) => { diff --git a/crates/ty_server/src/server/api/requests/rename.rs b/crates/ty_server/src/server/api/requests/rename.rs index c90e6ddcae..6e2760f53f 100644 --- a/crates/ty_server/src/server/api/requests/rename.rs +++ b/crates/ty_server/src/server/api/requests/rename.rs @@ -5,7 +5,10 @@ use anyhow::anyhow; use lsp_server::ErrorCode; use lsp_types::RenameRequest; use lsp_types::{RenameParams, TextEdit, Uri, WorkspaceEdit}; -use ty_ide::{TemplateRename, TemplateRenameOutcome, django_rename, rename}; +use ty_ide::{ + PreparedRename, TemplateRename, TemplateRenameOutcome, django_rename, invalid_new_name, + prepare_rename, rename, +}; use ty_project::{ProjectDatabase, SemanticDb as _}; use crate::document::{FileRangeExt, LspRange, PositionExt, ToLink}; @@ -55,9 +58,18 @@ impl BackgroundDocumentRequestHandler for RenameRequestHandler { let template = snapshot.is_django_template(); - // a python symbol is renamed exactly as it was; the django names a module - // writes as plain strings are what is left over once it declines - if !template + // a rename is checked as `textDocument/prepareRename` checks it, since a + // client need not ask that first. the django names a module writes as + // plain strings are what is left over once the python symbol is not one + let python = (!template).then(|| prepare_rename(db, db.program_file(file), offset)); + // only a python symbol's new name must be an identifier: a django rename + // may be naming a template file + if let Some(PreparedRename::Ready { .. }) = python + && let Some(why) = invalid_new_name(¶ms.new_name) + { + return Err(Error::new(anyhow!(why), ErrorCode::RequestFailed)); + } + if let Some(PreparedRename::Ready { .. }) = python && let Some(rename_results) = rename(db, db.program_file(file), offset, ¶ms.new_name) { @@ -87,7 +99,12 @@ impl BackgroundDocumentRequestHandler for RenameRequestHandler { } match django_rename(db, file, offset, ¶ms.new_name, template) { - None => Ok(None), + None => match python { + Some(PreparedRename::Refused(why)) => { + Err(Error::new(anyhow!(why), ErrorCode::RequestFailed)) + } + Some(PreparedRename::Ready { .. } | PreparedRename::NoSymbol) | None => Ok(None), + }, Some(TemplateRenameOutcome::Refused(why)) => { Err(Error::new(anyhow!(why), ErrorCode::RequestFailed)) } diff --git a/crates/ty_server/src/server/api/requests/shutdown.rs b/crates/ty_server/src/server/api/requests/shutdown.rs index 95addad3a1..62d50f5651 100644 --- a/crates/ty_server/src/server/api/requests/shutdown.rs +++ b/crates/ty_server/src/server/api/requests/shutdown.rs @@ -2,7 +2,6 @@ use crate::Session; use crate::server::api::traits::{RequestHandler, SyncRequestHandler}; use crate::session::client::Client; -use lsp_types::WorkspaceDiagnosticReport; use salsa::Database; pub(crate) struct ShutdownHandler; @@ -16,13 +15,8 @@ impl SyncRequestHandler for ShutdownHandler { tracing::debug!("Received shutdown request, waiting for exit notification"); // Respond to any pending workspace diagnostic requests - if let Some(suspended_workspace_request) = - session.take_suspended_workspace_diagnostic_request() - { - client.respond( - &suspended_workspace_request.id, - Ok(WorkspaceDiagnosticReport::default()), - ); + for suspended_workspace_request in session.take_suspended_workspace_diagnostic_requests() { + suspended_workspace_request.answer_at_shutdown(client); } session.set_shutdown_requested(true); diff --git a/crates/ty_server/src/server/api/requests/super_members.rs b/crates/ty_server/src/server/api/requests/super_members.rs new file mode 100644 index 0000000000..31157fab72 --- /dev/null +++ b/crates/ty_server/src/server/api/requests/super_members.rs @@ -0,0 +1,151 @@ +//! `by/superMembers`: the superclass members a class member overrides +//! +//! what an editor's *Go to Super* asks of a method or an attribute, and what an "overrides" gutter +//! marker points at. LSP has no request for it: `textDocument/implementation` goes the other way, +//! from a member to the members that override it, and `typeHierarchy/supertypes` answers about +//! classes, not their members +//! +//! what counts as an override is the override checks' own walk up the MRO. of the declarations it +//! finds, the answer is the nearest along each branch of the MRO, in MRO order, so a member of a +//! class with several bases can override several + +use std::borrow::Cow; + +use lsp_types::{ + LspRequestMethod, MessageDirection, Position, Range, Request, TextDocumentIdentifier, Uri, +}; +use ty_ide::super_members; +use ty_project::{ProjectDatabase, SemanticDb as _}; + +use crate::PositionEncoding; +use crate::document::{PositionExt, ToLink}; +use crate::server::api::traits::{ + BackgroundDocumentRequestHandler, RequestHandler, RetriableRequestHandler, +}; +use crate::session::DocumentSnapshot; +use crate::session::client::Client; + +pub(crate) enum SuperMembersRequest {} + +impl Request for SuperMembersRequest { + type Params = SuperMembersParams; + type Result = Option>; + const METHOD: LspRequestMethod<'static> = LspRequestMethod::Custom("by/superMembers"); + const MESSAGE_DIRECTION: MessageDirection = MessageDirection::ClientToServer; +} + +/// which member to ask about: the document, and a position on the member's name where it is +/// declared: the name of a `def` or a nested class, a name a class body assigns, annotates or +/// captures, or an import's alias +#[derive(Debug, serde::Deserialize, serde::Serialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub(crate) struct SuperMembersParams { + text_document: TextDocumentIdentifier, + position: Position, +} + +/// one superclass member the member overrides +#[derive(Debug, serde::Deserialize, serde::Serialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct SuperMember { + /// the member's name + name: String, + /// the superclass that declares it + container_name: String, + /// the file the superclass declares it in + uri: Uri, + /// the whole declaration + range: Range, + /// the declared name, which is where to go + selection_range: Range, + /// whether the superclass synthesizes the member rather than writing it, as a dataclass does + /// its `__init__`. the ranges are then the superclass's own + synthesized: bool, + /// whether the member is abstract where the superclass declares it, an `@abstractmethod` or a + /// protocol method with no implementation, so that overriding it implements it: what + /// `abstract-instantiation` counts as abstract + #[serde(rename = "abstract")] + is_abstract: bool, +} + +impl SuperMember { + /// the member as a client reads it, or `None` when where it is cannot be said in the client's + /// terms + pub(super) fn from_ide( + db: &ProjectDatabase, + member: &ty_ide::SuperMember, + encoding: PositionEncoding, + ) -> Option { + let link = member.target.to_link(db, None, encoding)?; + Some(SuperMember { + name: member.name.to_string(), + container_name: member.superclass.to_string(), + uri: link.target_uri, + range: link.target_range, + selection_range: link.target_selection_range, + synthesized: member.synthesized, + is_abstract: member.is_abstract, + }) + } +} + +pub(crate) struct SuperMembersRequestHandler; + +impl RequestHandler for SuperMembersRequestHandler { + type RequestType = SuperMembersRequest; +} + +impl BackgroundDocumentRequestHandler for SuperMembersRequestHandler { + // what the project's files say about a position, the same whether this one is a buffer or the + // file on disk; the answer reads no version + const ANSWERS_CLOSED_DOCUMENTS: bool = true; + + fn document_uri(params: &SuperMembersParams) -> Cow<'_, Uri> { + Cow::Borrowed(¶ms.text_document.uri) + } + + /// `null` when language services are off, for a template, which has no python members, and + /// when there is no class member declared at the position; an empty list when there is one and + /// it overrides nothing + fn run_with_snapshot( + db: &ProjectDatabase, + snapshot: &DocumentSnapshot, + _client: &Client, + params: SuperMembersParams, + ) -> crate::server::Result>> { + if snapshot + .workspace_settings() + .is_language_services_disabled() + { + return Ok(None); + } + if snapshot.is_django_template() { + return Ok(None); + } + + let Some(file) = snapshot.to_notebook_or_file(db) else { + return Ok(None); + }; + + let Some(offset) = + params + .position + .to_text_size(db, file, snapshot.uri(), snapshot.encoding()) + else { + return Ok(None); + }; + + let Some(members) = super_members(db, db.program_file(file), offset) else { + return Ok(None); + }; + + Ok(Some( + members + .iter() + .filter_map(|member| SuperMember::from_ide(db, member, snapshot.encoding())) + .collect(), + )) + } +} + +impl RetriableRequestHandler for SuperMembersRequestHandler {} diff --git a/crates/ty_server/src/server/api/requests/workspace_diagnostic.rs b/crates/ty_server/src/server/api/requests/workspace_diagnostic.rs index 1553e33069..eb89c59221 100644 --- a/crates/ty_server/src/server/api/requests/workspace_diagnostic.rs +++ b/crates/ty_server/src/server/api/requests/workspace_diagnostic.rs @@ -29,7 +29,7 @@ use crate::server::api::traits::{ BackgroundRequestHandler, RequestHandler, RetriableRequestHandler, }; use crate::server::lazy_work_done_progress::LazyWorkDoneProgress; -use crate::server::{Action, Result}; +use crate::server::{Action, Result, SuspendedWorkspaceRequestKind}; use crate::session::client::Client; use crate::session::index::Index; use crate::session::{GlobalSettings, SessionSnapshot, SuspendedWorkspaceDiagnosticRequest}; @@ -115,49 +115,13 @@ impl BackgroundRequestHandler for WorkspaceDiagnosticRequestHandler { client: &Client, params: WorkspaceDiagnosticParams, ) -> Result { - if !snapshot.global_settings().diagnostic_mode().is_workspace() { - tracing::debug!("Workspace diagnostics is disabled; returning empty report"); - return Ok(WorkspaceDiagnosticReport { items: vec![] }); - } - - if snapshot - .projects() - .iter() - .any(|db| db.uv_environments().has_pending_initializations()) - { - tracing::debug!( - "Deferring workspace diagnostics until script initialization completes" - ); - // Returning an empty workspace report makes `handle_request` suspend the request. - // Skip the response writer: it would clear diagnostics for previous result IDs - // that we have not checked yet. Suspension retains the request, not this snapshot. - return Ok(WorkspaceDiagnosticReport { items: vec![] }); - } - - let writer = ResponseWriter::new( - params.partial_result_params.partial_result_token, - params.previous_result_ids, - snapshot, - client, - ); - - // Use the work done progress token from the client request, if provided - // Note: neither VS Code nor Zed currently support this, - // see https://github.com/microsoft/vscode-languageserver-node/issues/528 - // That's why we fall back to server-initiated progress if no token is provided. - let work_done_progress = LazyWorkDoneProgress::new( - client, - params.work_done_progress_params.work_done_token, - "Checking", - snapshot.resolved_client_capabilities(), - ); - let mut reporter = WorkspaceDiagnosticsProgressReporter::new(work_done_progress, writer); - - for db in snapshot.projects() { - db.check_with_reporter(&mut reporter); - } - - Ok(reporter.into_final_report()) + Ok(match check_workspace(snapshot, client, params) { + WorkspaceCheck::Checked(report) => report, + // an empty report, which `handle_request` holds open + WorkspaceCheck::NotReady | WorkspaceCheck::Disabled => { + WorkspaceDiagnosticReport::default() + } + }) } fn handle_request( @@ -195,6 +159,7 @@ impl BackgroundRequestHandler for WorkspaceDiagnosticRequestHandler { client.queue_action(Action::SuspendWorkspaceDiagnostics(Box::new( SuspendedWorkspaceDiagnosticRequest { id: id.clone(), + kind: SuspendedWorkspaceRequestKind::LongPoll, params: json!(¶ms), revision: snapshot.revision(), }, @@ -209,6 +174,71 @@ impl BackgroundRequestHandler for WorkspaceDiagnosticRequestHandler { } } +/// what checking the workspace for a workspace diagnostics request came to +pub(crate) enum WorkspaceCheck { + /// the workspace was checked, and this is the report + Checked(WorkspaceDiagnosticReport), + + /// the workspace was not checked because a script's initial environment is still being set + /// up, and a report now would describe the script against an environment it does not have + /// yet. the request waits for it, held open like a long poll + NotReady, + + /// the workspace was not checked because the diagnostic mode does not report it + Disabled, +} + +/// checks the workspace for a request with `params`, streaming the reports that changed when the +/// params carry a partial result token +/// +/// shared by `workspace/diagnostic` and `by/checkWorkspace`, which differ in when they answer +pub(crate) fn check_workspace( + snapshot: &SessionSnapshot, + client: &Client, + params: WorkspaceDiagnosticParams, +) -> WorkspaceCheck { + if !snapshot.global_settings().diagnostic_mode().is_workspace() { + tracing::debug!("Workspace diagnostics is disabled; not checking the workspace"); + return WorkspaceCheck::Disabled; + } + + if snapshot + .projects() + .iter() + .any(|db| db.uv_environments().has_pending_initializations()) + { + tracing::debug!("Deferring workspace diagnostics until script initialization completes"); + // Skip the response writer: it would clear diagnostics for previous result IDs + // that we have not checked yet. Suspension retains the request, not this snapshot. + return WorkspaceCheck::NotReady; + } + + let writer = ResponseWriter::new( + params.partial_result_params.partial_result_token, + params.previous_result_ids, + snapshot, + client, + ); + + // Use the work done progress token from the client request, if provided + // Note: neither VS Code nor Zed currently support this, + // see https://github.com/microsoft/vscode-languageserver-node/issues/528 + // That's why we fall back to server-initiated progress if no token is provided. + let work_done_progress = LazyWorkDoneProgress::new( + client, + params.work_done_progress_params.work_done_token, + "Checking", + snapshot.resolved_client_capabilities(), + ); + let mut reporter = WorkspaceDiagnosticsProgressReporter::new(work_done_progress, writer); + + for db in snapshot.projects() { + db.check_with_reporter(&mut reporter); + } + + WorkspaceCheck::Checked(reporter.into_final_report()) +} + impl RetriableRequestHandler for WorkspaceDiagnosticRequestHandler { fn salsa_cancellation_error() -> lsp_server::ResponseError { lsp_server::ResponseError { diff --git a/crates/ty_server/src/server/main_loop.rs b/crates/ty_server/src/server/main_loop.rs index 3154f0293a..bb94c2a6f8 100644 --- a/crates/ty_server/src/server/main_loop.rs +++ b/crates/ty_server/src/server/main_loop.rs @@ -192,7 +192,7 @@ impl Server { } Action::SuspendWorkspaceDiagnostics(suspended_request) => { - self.session.set_suspended_workspace_diagnostics_request( + self.session.add_suspended_workspace_diagnostics_request( *suspended_request, &client, ); diff --git a/crates/ty_server/src/session.rs b/crates/ty_server/src/session.rs index 8dc309d207..ebe8f64830 100644 --- a/crates/ty_server/src/session.rs +++ b/crates/ty_server/src/session.rs @@ -15,8 +15,7 @@ use lsp_types::{ }; use lsp_types::{DidChangeWatchedFilesNotification, ExitNotification, Notification}; use lsp_types::{ - DocumentDiagnosticRequest, RegistrationRequest, Request, ShutdownRequest, - UnregistrationRequest, WorkspaceDiagnosticRequest, + DocumentDiagnosticRequest, RegistrationRequest, Request, ShutdownRequest, UnregistrationRequest, }; use ruff_db::Db; use ruff_db::files::{File, system_path_to_file}; @@ -40,8 +39,8 @@ use crate::capabilities::{ResolvedClientCapabilities, server_diagnostic_options} use crate::db::Db as _; use crate::document::{DocumentKey, DocumentVersion, LanguageId, NotebookDocument}; use crate::server::{ - Action, LazyWorkDoneProgress, ScriptProgress, publish_all_document_diagnostics, - publish_diagnostics_if_needed, publish_settings_diagnostics, + Action, LazyWorkDoneProgress, ScriptProgress, SuspendedWorkspaceRequestKind, + publish_all_document_diagnostics, publish_diagnostics_if_needed, publish_settings_diagnostics, }; use crate::session::client::Client; use crate::session::index::Document; @@ -110,11 +109,21 @@ pub(crate) struct Session { /// could result in different workspace diagnostics. revision: u64, - /// A pending workspace diagnostics request because there were no diagnostics - /// or no changes when when the request ran last time. - /// We'll re-run the request after every change to `Session` (see `revision`) - /// to see if there are now changes and, if so, respond to the client. - suspended_workspace_diagnostics_request: Option, + /// the workspace diagnostics requests held open, see [`SuspendedWorkspaceDiagnosticRequest`]. + /// each is run again after every change to `Session` (see `revision`), and answered once + /// it has something to say + /// + /// more than one, because a client can have a `workspace/diagnostic` request held open and + /// send a `by/checkWorkspace` beside it, and neither may be dropped: a request that is never + /// answered is one the client waits on until it gives up + /// + /// each is run again on its own, so a change costs one workspace check per request held. the + /// checks share the per-file results salsa caches, so the second and later ones mostly redo + /// the walk over the project's files and the building of each report. they are not merged + /// into one check, because each request compares against its own previous result ids and + /// reports its own progress, and is re-dispatched like any other request so that a + /// cancellation or a retry reaches it alone + suspended_workspace_diagnostics_requests: Vec, /// Document requests about a text the session does not hold yet, each waiting for the session /// to change. See [`HeldRequest`]. @@ -207,7 +216,7 @@ impl Session { request_queue: RequestQueue::new(), shutdown_requested: false, in_test, - suspended_workspace_diagnostics_request: None, + suspended_workspace_diagnostics_requests: Vec::new(), held_requests: Vec::new(), revision: 0, registrations: HashSet::new(), @@ -246,35 +255,42 @@ impl Session { self.shutdown_requested = requested; } - pub(crate) fn set_suspended_workspace_diagnostics_request( + /// holds `request` open beside any others already held + /// + /// once the server is shutting down nothing resumes a held request, so one whose check was + /// still running when the `shutdown` request came is answered now, as `shutdown` answered + /// the ones held before it + pub(crate) fn add_suspended_workspace_diagnostics_request( &mut self, request: SuspendedWorkspaceDiagnosticRequest, client: &Client, ) { - self.suspended_workspace_diagnostics_request = Some(request); + if self.shutdown_requested { + request.answer_at_shutdown(client); + return; + } + self.suspended_workspace_diagnostics_requests.push(request); // Run the suspended workspace diagnostic request immediately in case there // were changes since the workspace diagnostics background thread queued // the action to suspend the workspace diagnostic request. - self.resume_suspended_workspace_diagnostic_request(client); + self.resume_suspended_workspace_diagnostic_requests(client); } - pub(crate) fn take_suspended_workspace_diagnostic_request( + pub(crate) fn take_suspended_workspace_diagnostic_requests( &mut self, - ) -> Option { - self.suspended_workspace_diagnostics_request.take() + ) -> Vec { + std::mem::take(&mut self.suspended_workspace_diagnostics_requests) } - /// Resumes (retries) the workspace diagnostic request if there - /// were any changes to the [`Session`] (the revision got bumped) - /// since the workspace diagnostic request ran last time. + /// runs each held workspace diagnostics request again if the [`Session`] changed (the + /// revision got bumped) since it last ran /// - /// The workspace diagnostic requests is ignored if the request - /// was cancelled in the meantime. - pub(crate) fn resume_suspended_workspace_diagnostic_request(&mut self, client: &Client) { - self.suspended_workspace_diagnostics_request = self - .suspended_workspace_diagnostics_request - .take() - .and_then(|request| { + /// a request the client cancelled in the meantime is dropped + pub(crate) fn resume_suspended_workspace_diagnostic_requests(&mut self, client: &Client) { + let suspended = std::mem::take(&mut self.suspended_workspace_diagnostics_requests); + self.suspended_workspace_diagnostics_requests = suspended + .into_iter() + .filter_map(|request| { if !self.request_queue.incoming().is_pending(&request.id) { // Clear out the suspended request if the request has been cancelled. tracing::debug!( @@ -286,7 +302,8 @@ impl Session { } request.resume_if_revision_changed(self.revision, client) - }); + }) + .collect(); } /// Keeps `request` until the session changes, because what it asks about is a text the session @@ -420,7 +437,7 @@ impl Session { self.bump_revision(); - self.resume_suspended_workspace_diagnostic_request(client); + self.resume_suspended_workspace_diagnostic_requests(client); let capabilities = self.client_capabilities(); if capabilities.supports_workspace_diagnostic_refresh() { @@ -660,14 +677,17 @@ impl Session { self.projects.values_mut().next().unwrap() } + /// applies `changes` to the project holding `path` + /// + /// the session's revision moves only if the project's database did: a change that changed + /// nothing leaves every answer as it was, and must not wake what waits for the session to + /// change pub(crate) fn apply_changes( &mut self, client: &Client, path: &AnySystemPath, changes: &[ChangeEvent], ) -> ChangeResult { - self.bump_revision(); - let capabilities = self.resolved_client_capabilities; let script_progress = self.script_progress.clone(); let db = self.project_db_mut(path); @@ -689,6 +709,9 @@ impl Session { } let scripts = result.scripts_to_synchronize(db); Self::synchronize_closed_scripts(db, &scripts, client, capabilities, &script_progress); + if result.database_changed() { + self.bump_revision(); + } result } @@ -2141,14 +2164,18 @@ impl Workspace { } } -/// A workspace diagnostic request that didn't yield any changes or diagnostic -/// when it ran the last time. +/// a workspace diagnostics request held open until the session changes: a `workspace/diagnostic` +/// request whose last run found nothing that differs from what the client has, or either kind of +/// request when the workspace was not ready to be checked #[derive(Debug)] pub(crate) struct SuspendedWorkspaceDiagnosticRequest { /// The LSP request id pub(crate) id: RequestId, - /// The params passed to the `workspace/diagnostic` request. + /// which request this is + pub(crate) kind: SuspendedWorkspaceRequestKind, + + /// the params the request was sent with pub(crate) params: serde_json::Value, /// The session's revision when the request ran the last time. @@ -2173,12 +2200,17 @@ impl SuspendedWorkspaceDiagnosticRequest { tracing::debug!("Resuming workspace diagnostics request after revision bump"); client.queue_action(Action::RetryRequest(lsp_server::Request { id: self.id, - method: WorkspaceDiagnosticRequest::METHOD.to_string(), + method: self.kind.method(), params: self.params, })); None } + + /// answers the request when the server shuts down before it could be run again + pub(crate) fn answer_at_shutdown(self, client: &Client) { + self.kind.answer_at_shutdown(&self.id, client); + } } /// How long a request may wait for its text before it is answered with `ServerCancelled`. @@ -2444,6 +2476,8 @@ impl DocumentHandle { }; session.apply_changes(client, path, &changes); + // the document's text changed, whether or not its file in the database did + session.bump_revision(); } fn set_version(&mut self, version: DocumentVersion) { diff --git a/crates/ty_server/src/session/options.rs b/crates/ty_server/src/session/options.rs index 11d545de86..11448811cd 100644 --- a/crates/ty_server/src/session/options.rs +++ b/crates/ty_server/src/session/options.rs @@ -620,3 +620,48 @@ pub(crate) struct PythonExecutable { uri: Option, sys_prefix: SystemPathBuf, } + +#[cfg(test)] +mod tests { + use strum::IntoEnumIterator; + use ty_ide::{InlayHintKind, InlayHintSettings, TemplateInlayHintKind}; + + use super::InlayHintOptions; + + fn settings_with_off(setting: &str) -> InlayHintSettings { + serde_json::from_value::(serde_json::json!({ setting: false })) + .unwrap_or_else(|error| panic!("`{setting}` is not an `inlayHints` option: {error}")) + .into_settings() + } + + /// the name a hint is tagged with is the option a client turns it off by, and + /// turning it off hides that kind of hint and no other + #[test] + fn each_hint_kind_is_turned_off_by_the_option_it_names() { + for kind in InlayHintKind::iter() { + let settings = settings_with_off(kind.setting()); + for other in InlayHintKind::iter() { + assert_eq!( + settings.enabled(other), + other != kind, + "`{}: false` switched {other:?}", + kind.setting(), + ); + } + assert!(TemplateInlayHintKind::iter().all(|other| settings.template_enabled(other))); + } + + for kind in TemplateInlayHintKind::iter() { + let settings = settings_with_off(kind.setting()); + for other in TemplateInlayHintKind::iter() { + assert_eq!( + settings.template_enabled(other), + other != kind, + "`{}: false` switched {other:?}", + kind.setting(), + ); + } + assert!(InlayHintKind::iter().all(|other| settings.enabled(other))); + } + } +} diff --git a/crates/ty_server/tests/e2e/build_trees.rs b/crates/ty_server/tests/e2e/build_trees.rs new file mode 100644 index 0000000000..902f164b51 --- /dev/null +++ b/crates/ty_server/tests/e2e/build_trees.rs @@ -0,0 +1,139 @@ +//! a tree `by build` wrote is not part of the project it was built from +//! +//! the output is a copy of the project (a `.py` for every `.by`, every hand-written `.py` carried +//! over), so checking it with the workspace would check everything twice and report every +//! diagnostic a second time, against a file nobody edits. it is recognised by the manifest the +//! build leaves at its top, wherever the build wrote it, and not by its name + +use anyhow::Result; +use lsp_types::{WorkspaceDiagnosticReport, WorkspaceDocumentDiagnosticReport}; +use ruff_db::system::SystemPath; +use ty_server::{ClientOptions, DiagnosticMode}; + +use crate::{TestServer, TestServerBuilder}; + +const ERROR: &str = "x: int = \"\"\n"; +const MANIFEST: &str = "# written by `by build`; delete it and stale output stays\napp.py\n"; + +/// the paths, relative to the workspace, of the files the report has a diagnostic for +fn reported_paths(server: &TestServer, report: &WorkspaceDiagnosticReport) -> Vec { + let root = server.file_path("project"); + let mut paths: Vec = report + .items + .iter() + .filter_map(|item| match item { + WorkspaceDocumentDiagnosticReport::WorkspaceFullDocumentDiagnosticReport(full) + if !full.full_document_diagnostic_report.items.is_empty() => + { + let path = full.uri.to_file_path().ok()?; + let path = SystemPath::from_std_path(&path)?.to_path_buf(); + Some(path.strip_prefix(&root).ok()?.as_str().replace('\\', "/")) + } + _ => None, + }) + .collect(); + paths.sort(); + paths +} + +fn workspace_server(builder: TestServerBuilder) -> Result { + Ok(builder + .with_workspace(SystemPath::new("project"), None)? + .with_initialization_options( + &ClientOptions::default().with_diagnostic_mode(DiagnosticMode::Workspace), + ) + .build() + .wait_until_workspaces_are_initialized()) +} + +/// the default output, `build/`, holds a copy of the module with the error. only the source is +/// reported +#[test] +fn the_workspace_does_not_include_the_build_output() -> Result<()> { + let mut server = workspace_server( + TestServerBuilder::new()? + .with_file("project/app.py", ERROR)? + .with_file("project/build/.by-manifest", MANIFEST)? + .with_file("project/build/app.py", ERROR)?, + )?; + + let report = server.workspace_diagnostic_request(None, None); + assert_eq!(reported_paths(&server, &report), ["app.py"]); + + Ok(()) +} + +/// a build told to write elsewhere is left out wherever it wrote, and a directory of the author's +/// own called `build`, with no manifest in it, stays part of the project +#[test] +fn the_output_is_recognised_by_its_manifest_not_its_name() -> Result<()> { + let mut server = workspace_server( + TestServerBuilder::new()? + .with_file("project/app.py", ERROR)? + .with_file("project/build/tool.py", ERROR)? + .with_file("project/elsewhere/.by-manifest", MANIFEST)? + .with_file("project/elsewhere/app.py", ERROR)?, + )?; + + let report = server.workspace_diagnostic_request(None, None); + assert_eq!( + reported_paths(&server, &report), + ["app.py", "build/tool.py"] + ); + + Ok(()) +} + +/// pulls the workspace's diagnostics each time the server says something changed, until `done` +fn reported_after_change(server: &mut TestServer, done: impl Fn(&[String]) -> bool) -> Vec { + loop { + server.await_diagnostic_refresh(); + let report = server.workspace_diagnostic_request(None, None); + let paths = reported_paths(server, &report); + if done(&paths) { + return paths; + } + } +} + +/// a build run while the server watches: the tree it writes does not join the project, and a +/// directory that becomes an output when a manifest appears in it leaves the project +/// +/// these read real file system events, like the other watching tests +#[test] +fn a_build_written_while_the_server_watches_does_not_join_the_project() -> Result<()> { + let mut server = workspace_server( + TestServerBuilder::new()? + .with_file("project/app.py", ERROR)? + .with_file("project/elsewhere/app.py", ERROR)? + .enable_pull_diagnostics(true) + .enable_workspace_diagnostic_refresh(true) + .watch_file_system(), + )?; + let report = server.workspace_diagnostic_request(None, None); + assert_eq!( + reported_paths(&server, &report), + ["app.py", "elsewhere/app.py"], + "a directory with no manifest is part of the project" + ); + + // the build marks its output before the first file lands in it + server.write_file("project/build/.by-manifest", MANIFEST)?; + server.write_file("project/build/app.py", ERROR)?; + // a write the server has to see after the build's, so that once it is reported the build's + // writes have been taken in too + server.write_file("project/later.py", ERROR)?; + let paths = reported_after_change(&mut server, |paths| { + paths.iter().any(|path| path == "later.py") + }); + assert_eq!(paths, ["app.py", "elsewhere/app.py", "later.py"]); + + // a directory already in the project leaves it when a build claims it + server.write_file("project/elsewhere/.by-manifest", MANIFEST)?; + let paths = reported_after_change(&mut server, |paths| { + !paths.iter().any(|path| path == "elsewhere/app.py") + }); + assert_eq!(paths, ["app.py", "later.py"]); + + Ok(()) +} diff --git a/crates/ty_server/tests/e2e/check_workspace.rs b/crates/ty_server/tests/e2e/check_workspace.rs new file mode 100644 index 0000000000..390969c7a8 --- /dev/null +++ b/crates/ty_server/tests/e2e/check_workspace.rs @@ -0,0 +1,417 @@ +//! `by/checkWorkspace` over the wire +//! +//! the report is `workspace/diagnostic`'s, from the same check, and the pull diagnostics tests +//! cover what is in it. what these cover is when it comes back: straight away, whatever it says — +//! which is the difference between the two, since `workspace/diagnostic` holds on to a request +//! whose answer the client already has — and how it comes back + +use std::time::Duration; + +use anyhow::Result; +use lsp_server::ErrorCode; +use lsp_types::{ + LspRequestMethod, MessageDirection, PartialResultParams, PreviousResultId, + ProgressNotification, ProgressToken, Request, WorkDoneProgressParams, + WorkspaceDiagnosticParams, WorkspaceDiagnosticReport, WorkspaceDocumentDiagnosticReport, +}; +use ruff_db::system::SystemPath; +use ty_server::{ClientOptions, DiagnosticMode}; + +use crate::pull_diagnostics::{ + assert_workspace_diagnostics_suspends_for_long_polling, send_workspace_diagnostic_request, + shutdown_and_await_workspace_diagnostic, +}; +use crate::{AwaitResponseError, TestServer, TestServerBuilder}; + +/// the request as a client sends it +enum CheckWorkspace {} + +impl Request for CheckWorkspace { + type Params = WorkspaceDiagnosticParams; + type Result = WorkspaceDiagnosticReport; + const METHOD: LspRequestMethod<'static> = LspRequestMethod::Custom("by/checkWorkspace"); + const MESSAGE_DIRECTION: MessageDirection = MessageDirection::ClientToServer; +} + +fn params(previous_result_ids: Vec) -> WorkspaceDiagnosticParams { + WorkspaceDiagnosticParams { + identifier: None, + previous_result_ids, + work_done_progress_params: WorkDoneProgressParams::default(), + partial_result_params: PartialResultParams::default(), + } +} + +/// asks, and waits no longer than a request that is answered straight away takes +fn check( + server: &mut TestServer, + previous_result_ids: Vec, +) -> Result { + let id = server.send_request::(params(previous_result_ids)); + server.try_await_response::(&id, Some(Duration::from_secs(10))) +} + +fn workspace_server(files: &[(&str, &str)], mode: DiagnosticMode) -> Result { + let mut builder = TestServerBuilder::new()? + .with_workspace(SystemPath::new("src"), None)? + .with_initialization_options(&ClientOptions::default().with_diagnostic_mode(mode)); + for (path, content) in files { + builder = builder.with_file(SystemPath::new(path), *content)?; + } + Ok(builder.build().wait_until_workspaces_are_initialized()) +} + +const CLEAN: &str = "def hello() -> str:\n return \"world\"\n"; +const BROKEN: &str = "def hello() -> str:\n return 42\n"; + +fn result_ids(report: &WorkspaceDiagnosticReport) -> Vec { + report + .items + .iter() + .filter_map(|item| match item { + WorkspaceDocumentDiagnosticReport::WorkspaceFullDocumentDiagnosticReport(full) => { + Some(PreviousResultId { + uri: full.uri.clone(), + value: full.full_document_diagnostic_report.result_id.clone()?, + }) + } + WorkspaceDocumentDiagnosticReport::WorkspaceUnchangedDocumentDiagnosticReport(_) => { + None + } + }) + .collect() +} + +/// a project with nothing wrong in it is an answer too: an empty report, not a request held open +/// until something goes wrong +#[test] +fn a_project_with_no_diagnostics_is_answered() -> Result<()> { + let mut server = workspace_server(&[("src/clean.py", CLEAN)], DiagnosticMode::Workspace)?; + + // the long poll holds on to exactly this request + let long_poll = send_workspace_diagnostic_request(&mut server); + assert_workspace_diagnostics_suspends_for_long_polling(&mut server, &long_poll); + + let report = check(&mut server, Vec::new()).expect("answered straight away"); + assert!(report.items.is_empty(), "{report:#?}"); + + // and the long poll still has nothing to say until the server goes away + let long_poll_report = shutdown_and_await_workspace_diagnostic(server, &long_poll); + assert!(long_poll_report.items.is_empty(), "{long_poll_report:#?}"); + + Ok(()) +} + +/// a check that finds what the client already has says so, file by file, rather than waiting for +/// a change +#[test] +fn an_unchanged_workspace_is_answered_with_unchanged_reports() -> Result<()> { + let mut server = workspace_server(&[("src/broken.py", BROKEN)], DiagnosticMode::Workspace)?; + + let first = check(&mut server, Vec::new()).expect("answered"); + let ids = result_ids(&first); + assert_eq!(ids.len(), 1, "{first:#?}"); + + let second = check(&mut server, ids).expect("answered although nothing changed"); + assert!( + matches!( + second.items.as_slice(), + [WorkspaceDocumentDiagnosticReport::WorkspaceUnchangedDocumentDiagnosticReport(_)] + ), + "{second:#?}" + ); + + Ok(()) +} + +/// in the mode that checks open files only, the workspace is not checked, and an empty report +/// would say it has no diagnostics +#[test] +fn a_workspace_that_is_not_checked_is_refused() -> Result<()> { + let mut server = workspace_server(&[("src/broken.py", BROKEN)], DiagnosticMode::OpenFilesOnly)?; + + match check(&mut server, Vec::new()) { + Err(AwaitResponseError::RequestFailed(error)) => { + assert_eq!(error.code, ErrorCode::RequestFailed as i32, "{error:?}"); + } + other => panic!("expected the request to be refused, got {other:?}"), + } + + Ok(()) +} + +/// the report comes whole, in the response, even when the params ask for partial results and +/// progress under the client's tokens: a check that an edit interrupts starts again, and nothing +/// sent under those tokens can be taken back +#[test] +fn the_report_comes_whole_in_the_response() -> Result<()> { + // the server streams partial results in batches of two in tests, so three files make at + // least one batch + let mut server = workspace_server( + &[ + ("src/a.py", BROKEN), + ("src/b.py", BROKEN), + ("src/c.py", BROKEN), + ], + DiagnosticMode::Workspace, + )?; + + let id = server.send_request::(WorkspaceDiagnosticParams { + work_done_progress_params: WorkDoneProgressParams { + work_done_token: Some(ProgressToken::String("work-done".to_string())), + }, + partial_result_params: PartialResultParams { + partial_result_token: Some(ProgressToken::String("partial-result".to_string())), + }, + ..params(Vec::new()) + }); + let report = server + .try_await_response::(&id, Some(Duration::from_secs(10))) + .expect("answered"); + assert_eq!(result_ids(&report).len(), 3, "{report:#?}"); + + if let Ok(progress) = + server.try_await_notification::(Some(Duration::from_secs(1))) + { + panic!("expected no progress under the client's tokens, got {progress:#?}"); + } + + Ok(()) +} + +/// a workspace whose script is still having its environment set up is not checked yet, because +/// the script would be checked against an environment it does not have. the check is held open +/// until the environment is there, like a long poll, and these cover what happens to it +/// meanwhile +/// +/// `uv` here is a stand-in that holds the setup until the test releases it +#[cfg(unix)] +mod a_script_environment_being_set_up { + use std::fs; + use std::os::unix::fs::PermissionsExt; + use std::time::{Duration, Instant}; + + use anyhow::{Result, bail}; + use lsp_server::{ErrorCode, RequestId}; + use lsp_types::{Uri, WorkspaceDiagnosticReport, WorkspaceDiagnosticRequest}; + use ruff_db::system::{SystemPath, SystemPathBuf}; + use ty_project::UseUv; + use ty_server::{ClientOptions, DiagnosticMode}; + + use super::{CheckWorkspace, params}; + use crate::pull_diagnostics::{ + assert_workspace_diagnostics_suspends_for_long_polling, send_workspace_diagnostic_request, + }; + use crate::{AwaitResponseError, TestServer, TestServerBuilder}; + + /// run beside the script, it says it has started, waits to be released, and fails, which + /// ends the setup with an environment that has an error rather than one that is pending + const UV: &str = "#!/bin/sh +touch started +i=0 +while [ ! -e release ] && [ \"$i\" -lt 3000 ]; do + sleep 0.01 + i=$((i + 1)) +done +exit 1 +"; + + /// the stand-in `uv`'s two signals, as files beside the script + struct Setup { + started: SystemPathBuf, + release: SystemPathBuf, + } + + impl Setup { + fn wait_until_started(&self) -> Result<()> { + let deadline = Instant::now() + Duration::from_secs(10); + while !self.started.as_std_path().exists() { + if Instant::now() > deadline { + bail!("the script's environment setup never started"); + } + std::thread::sleep(Duration::from_millis(10)); + } + Ok(()) + } + + fn release(&self) { + // an error here leaves the setup to the stand-in's own time limit + let _ = fs::write(self.release.as_std_path(), ""); + } + } + + impl Drop for Setup { + // the server waits for the setup when it exits + fn drop(&mut self) { + self.release(); + } + } + + fn server() -> Result<(TestServer, Setup)> { + let builder = TestServerBuilder::new()? + .with_workspace(SystemPath::new("src"), None)? + .with_initialization_options( + &ClientOptions::default().with_diagnostic_mode(DiagnosticMode::Workspace), + ) + .with_use_uv(UseUv::Scripts) + .with_file("src/main.py", "missing\n")? + .with_file( + "src/script.py", + "# /// script\n# dependencies = []\n# ///\n", + )? + .with_file("bin/uv", UV)?; + let uv = builder.file_path("bin/uv"); + fs::set_permissions(uv.as_std_path(), fs::Permissions::from_mode(0o755))?; + let setup = Setup { + started: builder.file_path("src/started"), + release: builder.file_path("src/release"), + }; + let server = builder + .with_env_var("UV", uv.as_str()) + .build() + .wait_until_workspaces_are_initialized(); + setup.wait_until_started()?; + Ok((server, setup)) + } + + fn answer>( + server: &mut TestServer, + id: &RequestId, + ) -> WorkspaceDiagnosticReport { + server + .try_await_response::(id, Some(Duration::from_secs(10))) + .unwrap_or_else(|err| panic!("request {id} was not answered: {err}")) + } + + fn reports_on(report: &WorkspaceDiagnosticReport, uri: &Uri) -> bool { + report.items.iter().any(|item| match item { + lsp_types::WorkspaceDocumentDiagnosticReport::WorkspaceFullDocumentDiagnosticReport( + full, + ) => &full.uri == uri, + lsp_types::WorkspaceDocumentDiagnosticReport::WorkspaceUnchangedDocumentDiagnosticReport( + unchanged, + ) => &unchanged.uri == uri, + }) + } + + fn assert_refused(server: &mut TestServer, id: &RequestId, code: ErrorCode) { + match server.try_await_response::(id, Some(Duration::from_secs(10))) { + Err(AwaitResponseError::RequestFailed(error)) => { + assert_eq!(error.code, code as i32, "{error:?}"); + } + other => panic!("expected the check to be refused with {code:?}, got {other:?}"), + } + } + + #[track_caller] + fn assert_held(server: &mut TestServer, id: &RequestId) { + match server.try_await_response::(id, Some(Duration::from_secs(2))) { + Err(AwaitResponseError::Timeout) => {} + other => panic!("expected the check to be held open, got {other:?}"), + } + } + + fn shutdown(server: &mut TestServer) -> RequestId { + server.send_request::(()) + } + + fn exit(server: &mut TestServer, shutdown: &RequestId) { + server.await_response::(shutdown); + server.send_notification::(()); + } + + /// the check is answered once the environment is there, with the whole workspace checked + #[test] + fn a_check_is_answered_once_the_environment_is_there() -> Result<()> { + let (mut server, setup) = server()?; + + let check = server.send_request::(params(Vec::new())); + assert_held(&mut server, &check); + + setup.release(); + let report = answer::(&mut server, &check); + assert!( + reports_on(&report, &server.file_uri("src/main.py")), + "{report:#?}" + ); + + Ok(()) + } + + /// a long poll waits for the environment too, and holding the one does not drop the other + #[test] + fn a_held_check_and_a_held_long_poll_are_both_answered() -> Result<()> { + let (mut server, setup) = server()?; + + let long_poll = send_workspace_diagnostic_request(&mut server); + assert_workspace_diagnostics_suspends_for_long_polling(&mut server, &long_poll); + let check = server.send_request::(params(Vec::new())); + assert_held(&mut server, &check); + + setup.release(); + let main = server.file_uri("src/main.py"); + let report = answer::(&mut server, &long_poll); + assert!(reports_on(&report, &main), "{report:#?}"); + let report = answer::(&mut server, &check); + assert!(reports_on(&report, &main), "{report:#?}"); + + Ok(()) + } + + /// at shutdown the long poll is answered with an empty report, since nothing changed, but the + /// check is refused: an empty report would say the workspace has no diagnostics, and it was + /// never checked + #[test] + fn a_check_held_at_shutdown_is_refused() -> Result<()> { + let (mut server, _setup) = server()?; + + let long_poll = send_workspace_diagnostic_request(&mut server); + assert_workspace_diagnostics_suspends_for_long_polling(&mut server, &long_poll); + let check = server.send_request::(params(Vec::new())); + assert_held(&mut server, &check); + + let shutdown = shutdown(&mut server); + let report = answer::(&mut server, &long_poll); + assert!(report.items.is_empty(), "{report:#?}"); + assert_refused(&mut server, &check, ErrorCode::RequestFailed); + exit(&mut server, &shutdown); + + Ok(()) + } + + /// a check that has not been held yet when the `shutdown` request comes, because it is still + /// finding out that the workspace is not ready, is refused as a held one is rather than held + /// by a server that will not run it again + #[test] + fn a_check_still_running_at_shutdown_is_refused() -> Result<()> { + let (mut server, _setup) = server()?; + + let check = server.send_request::(params(Vec::new())); + let shutdown = shutdown(&mut server); + assert_refused(&mut server, &check, ErrorCode::RequestFailed); + exit(&mut server, &shutdown); + + Ok(()) + } + + /// a held check the client cancels is answered as cancelled, once, and is not run again when + /// the environment is there + #[test] + fn a_held_check_can_be_cancelled() -> Result<()> { + let (mut server, setup) = server()?; + + let check = server.send_request::(params(Vec::new())); + assert_held(&mut server, &check); + server.cancel(&check); + assert_refused(&mut server, &check, ErrorCode::RequestCanceled); + + // a second check is answered once the environment is there, by which time the cancelled + // one would have been run again too. the test server's drop asserts that no other + // response came + setup.release(); + let second = server.send_request::(params(Vec::new())); + answer::(&mut server, &second); + + Ok(()) + } +} diff --git a/crates/ty_server/tests/e2e/django_templates.rs b/crates/ty_server/tests/e2e/django_templates.rs index 30363882ed..7cf65f0801 100644 --- a/crates/ty_server/tests/e2e/django_templates.rs +++ b/crates/ty_server/tests/e2e/django_templates.rs @@ -278,6 +278,122 @@ fn a_template_created_during_the_session_is_offered_to_an_extends() -> Result<() Ok(()) } +/// a client watching for files reports the file alone, and not the directory it was +/// written into along with it: the walk that found the templates listed the directory +/// above the new one, and that is the directory whose entries changed +#[test] +fn a_template_created_in_a_new_directory_is_offered_to_an_extends() -> Result<()> { + let mut server = server("{% extends '' %}\n")?; + let uri = template_uri(&server); + + let offered = |server: &mut TestServer| -> Vec { + server + .completion_request(&uri, Position::new(0, 12)) + .iter() + .map(|completion| completion.label.clone()) + .collect() + }; + + assert!(!offered(&mut server).contains(&"blog/cards/card.html".to_string())); + + let created = SystemPath::new("src/blog/templates/blog/cards/card.html"); + server.write_file(created, "

a card

\n")?; + server.did_change_watched_files(vec![FileEvent { + uri: server.file_uri(created), + kind: FileChangeType::Created, + }]); + + assert!( + offered(&mut server).contains(&"blog/cards/card.html".to_string()), + "a template in a directory created after the server started is still a template" + ); + + Ok(()) +} + +const BUILD_MANIFEST: &str = "src/build/.by-manifest"; + +/// a project holding a build's output, with a template of its own under `build/` +fn server_with_a_build_output() -> Result { + let mut server = TestServerBuilder::new()? + .with_workspace(SystemPath::new("src"), None)? + .with_file(SystemPath::new("src/blog/views.py"), VIEWS)? + .with_file(SystemPath::new("src/blog/templates/blog/base.html"), BASE)? + .with_file( + SystemPath::new("src/blog/templates/blog/post.html"), + "{% extends '' %}\n", + )? + .with_file(SystemPath::new(BUILD_MANIFEST), "blog/views.py\n")? + .with_file( + SystemPath::new("src/build/blog/templates/blog/built.html"), + BASE, + )? + .build() + .wait_until_workspaces_are_initialized(); + server.open_text_document_as( + SystemPath::new("src/blog/templates/blog/post.html"), + "{% extends '' %}\n", + 1, + LanguageKind::new("django-html"), + ); + Ok(server) +} + +/// the names an `{% extends '' %}` in the template under test is offered +fn offered_to_the_extends(server: &mut TestServer) -> Vec { + let uri = template_uri(server); + server + .completion_request(&uri, Position::new(0, 12)) + .iter() + .map(|completion| completion.label.clone()) + .collect() +} + +/// a build's output is a copy of the project, templates and all, and none of it is the +/// project's own: a template there is not one the project can load by that name +#[test] +fn a_template_in_a_build_output_is_not_offered_to_an_extends() -> Result<()> { + let mut server = server_with_a_build_output()?; + + let offered = offered_to_the_extends(&mut server); + assert!( + offered.contains(&"blog/base.html".to_string()), + "{offered:#?}" + ); + assert!( + !offered.contains(&"blog/built.html".to_string()), + "{offered:#?}" + ); + + Ok(()) +} + +/// deleting the manifest makes the directory an ordinary one of the project's, and the +/// templates in it the project's own +#[test] +fn a_template_in_a_former_build_output_is_offered_to_an_extends() -> Result<()> { + let mut server = server_with_a_build_output()?; + let offered = offered_to_the_extends(&mut server); + assert!( + !offered.contains(&"blog/built.html".to_string()), + "{offered:#?}" + ); + + std::fs::remove_file(server.file_path(BUILD_MANIFEST).as_std_path())?; + server.did_change_watched_files(vec![FileEvent { + uri: server.file_uri(BUILD_MANIFEST), + kind: FileChangeType::Deleted, + }]); + + let offered = offered_to_the_extends(&mut server); + assert!( + offered.contains(&"blog/built.html".to_string()), + "{offered:#?}" + ); + + Ok(()) +} + #[test] fn a_template_is_never_type_checked_as_python() -> Result<()> { // every line of this would be a python syntax error @@ -999,6 +1115,17 @@ fn a_template_hints_a_loop_binding_and_the_file_an_include_resolves_to() -> Resu ["11 : Book", "50 → blog/templates/blog/card.html"] ); + // each is tagged with the `inlayHints` option that switches it, as a python + // hint is + let kinds: Vec<_> = hints.iter().map(|hint| hint.data.clone()).collect(); + assert_eq!( + kinds, + [ + Some(serde_json::json!({ "kind": "templateBindingTypes" })), + Some(serde_json::json!({ "kind": "resolvedTemplates" })), + ] + ); + Ok(()) } diff --git a/crates/ty_server/tests/e2e/file_watching.rs b/crates/ty_server/tests/e2e/file_watching.rs index 5a74ce6b3a..af483ab549 100644 --- a/crates/ty_server/tests/e2e/file_watching.rs +++ b/crates/ty_server/tests/e2e/file_watching.rs @@ -3,15 +3,26 @@ //! These tests read real file system events from the operating system. Like the command line's //! file-watching tests, they do not work inside a sandbox that withholds those events. +use std::time::Duration; + use anyhow::Result; -use lsp_types::{DocumentDiagnosticReport, Message, RegistrationRequest}; +use lsp_types::{ + DocumentDiagnosticReport, Message, RegistrationRequest, WorkspaceDiagnosticRequest, +}; use ruff_db::system::SystemPath; +use ty_project::UseUv; +use ty_server::{ClientOptions, DiagnosticMode}; +use crate::pull_diagnostics::{ + assert_workspace_diagnostics_suspends_for_long_polling, send_workspace_diagnostic_request, +}; use crate::{TestServer, TestServerBuilder}; const LIB_INT: &str = "def make() -> int:\n return 1\n"; const LIB_STR: &str = "def make() -> str:\n return ''\n"; const USER: &str = "from lib import make\n\nx: bytes = make()\n"; +const IMPORTS_OTHER: &str = "from other import make\n\nx: bytes = make()\n"; +const IMPORTS_DEP: &str = "from dep import make\n\nx: bytes = make()\n"; /// The messages of the diagnostics the server has for `path` now. fn diagnostic_messages(server: &mut TestServer, path: &SystemPath) -> Vec { @@ -52,6 +63,19 @@ fn diagnostics_after_change( } } +/// how long a change is given to reach the server and be answered before it is taken to have +/// sent nothing: the watcher settles a batch 10 ms after its last event +const SETTLE: Duration = Duration::from_secs(1); + +/// asserts that the server asks for no refresh within [`SETTLE`] +#[track_caller] +fn assert_no_refresh(server: &mut TestServer, after: &str) { + assert!( + !server.try_await_diagnostic_refresh(SETTLE), + "the server asked the client to refresh after {after}" + ); +} + /// A file that is not open, rewritten on disk, reaches the diagnostics of an open file that /// depends on it, with no `workspace/didChangeWatchedFiles` from the client. The server hears of /// the write from the operating system, which reports it only once it has happened. @@ -177,3 +201,208 @@ fn the_client_is_asked_to_watch_when_the_server_does_not() -> Result<()> { Ok(()) } + +/// a refresh makes the client pull the diagnostics and inlay hints of everything it shows again, +/// so a write that no answer depends on is not one: not a build writing its output a file at a +/// time, nor a write into an excluded or an ignored directory. a write the project reads after +/// them is +#[test] +fn writes_the_project_does_not_read_are_not_a_refresh() -> Result<()> { + let lib = SystemPath::new("project/src/lib.py"); + let user = SystemPath::new("project/src/user.py"); + + let mut server = TestServerBuilder::new()? + .with_workspace(SystemPath::new("project"), None)? + .with_file("project/basedpython.toml", "[src]\nexclude = [\"runs\"]\n")? + .with_file("project/.gitignore", "__pycache__/\n")? + .with_file(lib, LIB_INT)? + .with_file(user, USER)? + // the output of an earlier build, which a rebuild writes over + .with_file("project/build/.by-manifest", "src/lib.py\nsrc/user.py\n")? + .with_file("project/build/lib.py", LIB_INT)? + .with_file("project/build/user.py", USER)? + .with_file("project/runs/0.trace", "")? + .with_file("project/src/__pycache__/lib.cpython-314.pyc", "")? + .enable_pull_diagnostics(true) + .enable_workspace_diagnostic_refresh(true) + .watch_file_system() + .build() + .wait_until_workspaces_are_initialized(); + + server.open_text_document(user, USER, 1); + let before = diagnostic_messages(&mut server, user); + assert!( + before.iter().any(|message| message.contains("`int`")), + "{before:#?}" + ); + assert_no_refresh(&mut server, "starting"); + + // far enough apart for each round to be a batch of its own, as a build's writes are + for round in 0..8 { + server.write_file("project/build/lib.py", LIB_STR)?; + server.write_file(format!("project/build/pkg/module_{round}.py"), LIB_STR)?; + server.write_file(format!("project/runs/{round}.trace"), "")?; + server.write_file( + format!("project/src/__pycache__/user.{round}.cpython-314.pyc"), + "", + )?; + std::thread::sleep(Duration::from_millis(50)); + } + assert_no_refresh( + &mut server, + "writes to a build's output, an excluded directory and an ignored one", + ); + + server.write_file(lib, LIB_STR)?; + diagnostics_after_change(&mut server, user, |messages| { + messages.iter().any(|message| message.contains("`str`")) + }); + + Ok(()) +} + +/// a file appearing and going away changes what an import resolves to, and each is a refresh, +/// though no file the server had read was written +#[test] +fn a_new_file_and_a_deleted_one_are_each_a_refresh() -> Result<()> { + let user = SystemPath::new("project/src/user.py"); + let other = SystemPath::new("project/src/other.py"); + let unresolved = |messages: &[String]| { + messages + .iter() + .any(|message| message.contains("Cannot resolve imported module `other`")) + }; + + let mut server = TestServerBuilder::new()? + .with_workspace(SystemPath::new("project"), None)? + .with_file(user, IMPORTS_OTHER)? + .enable_pull_diagnostics(true) + .enable_workspace_diagnostic_refresh(true) + .watch_file_system() + .build() + .wait_until_workspaces_are_initialized(); + + server.open_text_document(user, IMPORTS_OTHER, 1); + let before = diagnostic_messages(&mut server, user); + assert!(unresolved(&before), "{before:#?}"); + + server.write_file(other, LIB_INT)?; + diagnostics_after_change(&mut server, user, |messages| !unresolved(messages)); + + std::fs::remove_file(server.file_path(other).as_std_path())?; + diagnostics_after_change(&mut server, user, unresolved); + + Ok(()) +} + +/// a dependency is read from outside the project, in the environment's `site-packages`, and a +/// change to it is a refresh like a change to the project's own file +#[cfg(unix)] +#[test] +fn a_change_in_site_packages_is_a_refresh() -> Result<()> { + let user = SystemPath::new("project/user.py"); + let dependency = SystemPath::new("venv/lib/python3.14/site-packages/dep.py"); + + let builder = TestServerBuilder::new()?; + let home = builder.file_path("base/bin"); + let mut server = builder + .with_workspace(SystemPath::new("project"), None)? + .with_file( + "project/basedpython.toml", + "[environment]\npython = \"../venv\"\n", + )? + .with_file(user, IMPORTS_DEP)? + .with_file("base/bin/python", "")? + .with_file("venv/bin/python", "")? + .with_file( + "venv/pyvenv.cfg", + format!("home = {home}\nversion_info = 3.14.0\n"), + )? + .with_file(dependency, LIB_INT)? + .enable_pull_diagnostics(true) + .enable_workspace_diagnostic_refresh(true) + .watch_file_system() + .build() + .wait_until_workspaces_are_initialized(); + + server.open_text_document(user, IMPORTS_DEP, 1); + let before = diagnostic_messages(&mut server, user); + assert!( + before.iter().any(|message| message.contains("`int`")), + "{before:#?}" + ); + + server.write_file(dependency, LIB_STR)?; + diagnostics_after_change(&mut server, user, |messages| { + messages.iter().any(|message| message.contains("`str`")) + }); + + Ok(()) +} + +/// a configuration file changes every answer without changing a source file +#[test] +fn a_configuration_change_is_a_refresh() -> Result<()> { + let lib = SystemPath::new("project/lib.py"); + let user = SystemPath::new("project/user.py"); + let configuration = SystemPath::new("project/basedpython.toml"); + + let mut server = TestServerBuilder::new()? + .with_workspace(SystemPath::new("project"), None)? + .with_file(configuration, "")? + .with_file(lib, LIB_INT)? + .with_file(user, USER)? + .with_use_uv(UseUv::Off) + .enable_pull_diagnostics(true) + .enable_workspace_diagnostic_refresh(true) + .watch_file_system() + .build() + .wait_until_workspaces_are_initialized(); + + server.open_text_document(user, USER, 1); + let before = diagnostic_messages(&mut server, user); + assert!(!before.is_empty(), "{before:#?}"); + + server.write_file(configuration, "[rules]\ninvalid-assignment = \"ignore\"\n")?; + diagnostics_after_change(&mut server, user, <[String]>::is_empty); + + Ok(()) +} + +/// a workspace diagnostic request held open until the workspace's diagnostics change is answered +/// when the server's own watcher sees the change, with nothing from the client +#[test] +fn a_request_held_open_is_answered_after_a_change_the_server_saw() -> Result<()> { + let broken = SystemPath::new("project/broken.py"); + + let mut server = TestServerBuilder::new()? + .with_workspace(SystemPath::new("project"), None)? + .with_file("project/clean.py", LIB_INT)? + .with_initialization_options( + &ClientOptions::default().with_diagnostic_mode(DiagnosticMode::Workspace), + ) + .watch_file_system() + .build() + .wait_until_workspaces_are_initialized(); + + // checking the one file first leaves the workspace check with nothing slow to do, so the long + // poll is held open well within the time `assert_workspace_diagnostics_suspends_for_long_polling` + // gives it, rather than still being checked when the file is written + let clean = SystemPath::new("project/clean.py"); + server.open_text_document(clean, LIB_INT, 1); + assert_eq!( + diagnostic_messages(&mut server, clean), + Vec::::new() + ); + + let long_poll = send_workspace_diagnostic_request(&mut server); + assert_workspace_diagnostics_suspends_for_long_polling(&mut server, &long_poll); + + server.write_file(broken, "x: int = ''\n")?; + let report = server + .try_await_response::(&long_poll, Some(Duration::from_secs(10))) + .unwrap_or_else(|error| panic!("the request held open was not answered: {error}")); + assert_eq!(report.items.len(), 1, "{report:#?}"); + + Ok(()) +} diff --git a/crates/ty_server/tests/e2e/goto_definition.rs b/crates/ty_server/tests/e2e/goto_definition.rs index f457a84835..0fb9c8fead 100644 --- a/crates/ty_server/tests/e2e/goto_definition.rs +++ b/crates/ty_server/tests/e2e/goto_definition.rs @@ -251,12 +251,13 @@ from idna import encode server.write_file(script, updated)?; // A watcher notification can arrive after the file is written but before `didSave`. - // Synchronization must still wait for the save notification. + // Synchronization must still wait for the save notification. the event changes nothing + // the server reads, as an open document's text comes from the client, so it asks for no + // refresh; the request below is answered after the event is handled server.did_change_watched_files(vec![FileEvent { uri: server.file_uri(script), kind: FileChangeType::Changed, }]); - server.await_diagnostic_refresh(); assert!( server @@ -267,14 +268,15 @@ from idna import encode server.save_text_document(script); + // the finished synchronization refreshes diagnostics, and the save itself may refresh + // them before it server.await_diagnostic_refresh(); - - assert!( - server - .goto_definition_request(script, Position::new(4, 19)) - .is_some(), - "saving must synchronize newly declared script dependencies" - ); + while server + .goto_definition_request(script, Position::new(4, 19)) + .is_none() + { + server.await_diagnostic_refresh(); + } Ok(()) } @@ -493,17 +495,24 @@ from idna import encode kind: FileChangeType::Created, }]); - // Watched-file changes refresh diagnostics immediately and again after uv finishes. - server.await_diagnostic_refresh(); - server.await_diagnostic_refresh(); - - server.open_text_document(script, updated, 2); - assert!( - server + // the finished synchronization refreshes diagnostics. whether the event refreshes them + // before it depends on whether the server read the rewritten file when the document + // closed, which races the write, so each refresh is checked until the new dependency + // resolves: a closed script is looked at by opening it, and closed again to go on + // waiting + let mut version = 2; + loop { + server.await_diagnostic_refresh(); + server.open_text_document(script, updated, version); + if server .goto_definition_request(script, Position::new(4, 19)) - .is_some(), - "watched changes must update environments even while scripts are closed" - ); + .is_some() + { + break; + } + server.close_text_document(script); + version += 1; + } Ok(()) } diff --git a/crates/ty_server/tests/e2e/inlay_hints.rs b/crates/ty_server/tests/e2e/inlay_hints.rs index 6216106ffe..9ecca40b8c 100644 --- a/crates/ty_server/tests/e2e/inlay_hints.rs +++ b/crates/ty_server/tests/e2e/inlay_hints.rs @@ -83,7 +83,10 @@ y = foo(Thing()) }, "newText": ": Thing" } - ] + ], + "data": { + "kind": "variableTypes" + } }, { "position": { @@ -126,7 +129,10 @@ y = foo(Thing()) }, "newText": "a=" } - ] + ], + "data": { + "kind": "callArgumentNames" + } } ] "#); @@ -174,7 +180,10 @@ def f(x: int) -> None: ], "kind": 1, "textEdits": [], - "paddingLeft": true + "paddingLeft": true, + "data": { + "kind": "revealedTypes" + } } ] "#); @@ -182,6 +191,68 @@ def f(x: int) -> None: Ok(()) } +/// the basedpython hints are each tagged, as `data.kind`, with the `inlayHints` +/// option that switches them, though LSP's own `kind` cannot tell them apart: +/// an implicit `self` is a `Parameter` like a call's argument name, and an +/// inferred return type is a `Type` like a variable's +#[test] +fn a_basedpython_hint_names_the_setting_that_switches_it() -> Result<()> { + let workspace_root = SystemPath::new("src"); + let foo = SystemPath::new("src/foo.by"); + let foo_content = "\ +class C: + init(a: int) + +def f(): + return 1 +"; + + let mut server = TestServerBuilder::new()? + .with_initialization_options(&ClientOptions::default()) + .with_workspace(workspace_root, None)? + .with_file(foo, foo_content)? + .enable_inlay_hints(true) + .build() + .wait_until_workspaces_are_initialized(); + + server.open_text_document(foo, foo_content, 1); + + let hints = server + .inlay_hints_request(foo, Range::new(Position::new(0, 0), Position::new(5, 0))) + .unwrap(); + + let kinds: Vec<_> = hints + .iter() + .map(|hint| { + let label = match &hint.label { + lsp_types::Label::String(label) => label.clone(), + lsp_types::Label::InlayHintLabelPartList(parts) => { + parts.iter().map(|part| part.value.as_str()).collect() + } + }; + (label, hint.kind, hint.data.clone()) + }) + .collect(); + + assert_eq!( + kinds, + vec![ + ( + "self,".to_string(), + Some(lsp_types::InlayHintKind::Parameter), + Some(serde_json::json!({ "kind": "implicitSelf" })) + ), + ( + "-> 1".to_string(), + Some(lsp_types::InlayHintKind::Type), + Some(serde_json::json!({ "kind": "inferredReturnTypes" })) + ), + ] + ); + + Ok(()) +} + /// Tests that disabling variable types inlay hints works correctly. #[test] fn variable_inlay_hints_disabled() -> Result<()> { @@ -347,7 +418,10 @@ def get_a() -> A: }, "newText": ", A" } - ] + ], + "data": { + "kind": "variableTypes" + } } ] "#); diff --git a/crates/ty_server/tests/e2e/main.rs b/crates/ty_server/tests/e2e/main.rs index a4cd5c184c..f8deb3949e 100644 --- a/crates/ty_server/tests/e2e/main.rs +++ b/crates/ty_server/tests/e2e/main.rs @@ -29,7 +29,9 @@ mod alignment_groups; mod build_output; +mod build_trees; mod call_hierarchy; +mod check_workspace; mod code_actions; mod commands; mod completions; @@ -55,6 +57,7 @@ mod rename; mod script_preparation; mod semantic_tokens; mod signature_help; +mod super_members; mod syntax_outline; mod transpile; mod type_hierarchy; @@ -119,6 +122,21 @@ fn setup_tracing() { }); } +/// The text hash as a client computes it for a request's `textHash`, written out again here +/// rather than borrowed from the server, so that the two are held to the definition rather than +/// to each other: FNV-1a, 64 bits, over UTF-16 code units, every line ending counted as one `\n`. +pub(crate) fn text_hash(text: &str) -> String { + let mut hash: u64 = 0xcbf2_9ce4_8422_2325; + let normalised = text.replace("\r\n", "\n").replace('\r', "\n"); + for unit in normalised.encode_utf16() { + for byte in unit.to_le_bytes() { + hash ^= u64::from(byte); + hash = hash.wrapping_mul(0x0000_0100_0000_01b3); + } + } + format!("{hash:016x}") +} + /// Errors when receiving a notification or request from the server. #[derive(thiserror::Error, Debug)] pub(crate) enum ServerMessageError { @@ -641,6 +659,19 @@ impl TestServer { self.send(Message::Response(Response::new_ok(id, ()))); } + /// Acknowledges a diagnostic refresh if the server requests one within `timeout`, and says + /// whether it did. + pub(crate) fn try_await_diagnostic_refresh(&mut self, timeout: Duration) -> bool { + match self.try_await_request::(Some(timeout)) { + Ok((id, ())) => { + self.send(Message::Response(Response::new_ok(id, ()))); + true + } + Err(ServerMessageError::Timeout) => false, + Err(error) => panic!("Failed to receive a diagnostic refresh: {error}"), + } + } + /// Checks server-created progress with matching begin, report, and end notifications. #[cfg(feature = "test-uv")] #[track_caller] diff --git a/crates/ty_server/tests/e2e/named_text.rs b/crates/ty_server/tests/e2e/named_text.rs index c84b5daf27..99f540fa8b 100644 --- a/crates/ty_server/tests/e2e/named_text.rs +++ b/crates/ty_server/tests/e2e/named_text.rs @@ -17,7 +17,7 @@ use ruff_db::system::SystemPath; use ty_server::ClientOptions; use crate::notebook::NotebookBuilder; -use crate::{AwaitResponseError, TestServer, TestServerBuilder}; +use crate::{AwaitResponseError, TestServer, TestServerBuilder, text_hash}; /// `by/syntaxOutline`, in json throughout, as a client sends it. enum SyntaxOutline {} @@ -50,21 +50,6 @@ impl Request for Hover { const MESSAGE_DIRECTION: MessageDirection = MessageDirection::ClientToServer; } -/// The text hash as a client computes it, written out again here rather than borrowed from the -/// server, so that the two are held to the definition rather than to each other: FNV-1a, 64 -/// bits, over UTF-16 code units, every line ending counted as one `\n`. -fn text_hash(text: &str) -> String { - let mut hash: u64 = 0xcbf2_9ce4_8422_2325; - let normalised = text.replace("\r\n", "\n").replace('\r', "\n"); - for unit in normalised.encode_utf16() { - for byte in unit.to_le_bytes() { - hash ^= u64::from(byte); - hash = hash.wrapping_mul(0x0000_0100_0000_01b3); - } - } - format!("{hash:016x}") -} - const MAIN: &str = "src/main.by"; /// On disk: one statement. diff --git a/crates/ty_server/tests/e2e/publish_diagnostics.rs b/crates/ty_server/tests/e2e/publish_diagnostics.rs index f68a2a384a..18959bcdb4 100644 --- a/crates/ty_server/tests/e2e/publish_diagnostics.rs +++ b/crates/ty_server/tests/e2e/publish_diagnostics.rs @@ -718,6 +718,7 @@ assert_type("test", list[str]) fn on_did_change_watched_files() -> Result<()> { let workspace_root = SystemPath::new("src"); let foo = SystemPath::new("src/foo.py"); + let bar = SystemPath::new("src/bar.py"); let foo_content = "\ def foo() -> str: print(a) @@ -726,14 +727,16 @@ def foo() -> str: let mut server = TestServerBuilder::new()? .with_workspace(workspace_root, None)? .with_file(foo, "")? + .with_file(bar, "")? .enable_pull_diagnostics(false) .build() .wait_until_workspaces_are_initialized(); let foo = server.file_path(foo); let foo_uri = server.file_uri(&foo); + let bar = server.file_path(bar); - server.open_text_document(&foo, "", 1); + server.open_text_document(&foo, "import bar\n", 1); let _open_diagnostics = server.await_notification::(); @@ -744,11 +747,19 @@ def foo() -> str: server.collect_publish_diagnostic_notifications(2); std::fs::write(&foo, foo_content)?; + // a module the open document imports, which the editor does not hold + std::fs::write(&bar, "x = 1\n")?; - server.did_change_watched_files(vec![FileEvent { - uri: foo_uri.clone(), - kind: FileChangeType::Changed, - }]); + server.did_change_watched_files(vec![ + FileEvent { + uri: foo_uri.clone(), + kind: FileChangeType::Changed, + }, + FileEvent { + uri: server.file_uri(&bar), + kind: FileChangeType::Changed, + }, + ]); let mut diagnostics = collect_publish_diagnostic_notifications_with_versions(&mut server, 3); assert_eq!(diagnostics[&first_cell].version, Some(0)); @@ -776,6 +787,39 @@ def foo() -> str: Ok(()) } +/// A change on disk to a document the editor holds changes nothing the server answers from: the +/// editor's text is what it checks. Publishing every open document's diagnostics again would +/// tell the client nothing it did not already have. +#[test] +fn on_did_change_watched_files_for_an_open_document_publishes_nothing() -> Result<()> { + let foo = SystemPath::new("src/foo.py"); + + let mut server = TestServerBuilder::new()? + .with_workspace(SystemPath::new("src"), None)? + .with_file(foo, "")? + .enable_pull_diagnostics(false) + .build() + .wait_until_workspaces_are_initialized(); + + server.open_text_document(foo, "", 1); + let _open_diagnostics = server.await_notification::(); + + server.write_file(foo, "def foo() -> str:\n print(a)\n")?; + server.did_change_watched_files(vec![FileEvent { + uri: server.file_uri(foo), + kind: FileChangeType::Changed, + }]); + + let diagnostics = server + .try_await_notification::(Some(Duration::from_millis(500))); + assert!( + diagnostics.is_err(), + "a change to the disk copy of an open document is not a change: {diagnostics:?}" + ); + + Ok(()) +} + #[test] fn on_did_change_watched_files_pull_diagnostics() -> Result<()> { let workspace_root = SystemPath::new("src"); diff --git a/crates/ty_server/tests/e2e/pull_diagnostics.rs b/crates/ty_server/tests/e2e/pull_diagnostics.rs index 76b0f801d8..50c5b717e9 100644 --- a/crates/ty_server/tests/e2e/pull_diagnostics.rs +++ b/crates/ty_server/tests/e2e/pull_diagnostics.rs @@ -1346,6 +1346,52 @@ def hello() -> str: Ok(()) } +/// every long-polling request held open at once is answered when something changes, not only the +/// last one held +#[test] +fn workspace_diagnostic_long_polling_answers_every_held_request() -> Result<()> { + let workspace_root = SystemPath::new("src"); + let file_path = SystemPath::new("src/test.py"); + let file_content_no_error = "\ +def hello() -> str: + return \"world\" +"; + let file_content_with_error = "\ +def hello() -> str: + return 42 +"; + + let mut server = + create_workspace_server_with_file(workspace_root, file_path, file_content_no_error)?; + server.open_text_document(file_path, file_content_no_error, 1); + + let first = send_workspace_diagnostic_request(&mut server); + assert_workspace_diagnostics_suspends_for_long_polling(&mut server, &first); + let second = send_workspace_diagnostic_request(&mut server); + assert_workspace_diagnostics_suspends_for_long_polling(&mut server, &second); + + server.change_text_document( + file_path, + vec![ + lsp_types::TextDocumentContentChangeEvent::TextDocumentContentChangeWholeDocument( + TextDocumentContentChangeWholeDocument { + text: file_content_with_error.to_string(), + }, + ), + ], + 2, + ); + + for id in [first, second] { + let report = server + .try_await_response::(&id, Some(Duration::from_secs(10))) + .unwrap_or_else(|err| panic!("request {id} was not answered: {err}")); + assert_eq!(report.items.len(), 1, "{report:#?}"); + } + + Ok(()) +} + /// Regression test for diagnostics disappearing in some cases. /// /// The specific way this fails is when a file that was never in the "open diff --git a/crates/ty_server/tests/e2e/rename.rs b/crates/ty_server/tests/e2e/rename.rs index e7aac2108f..9ea093b514 100644 --- a/crates/ty_server/tests/e2e/rename.rs +++ b/crates/ty_server/tests/e2e/rename.rs @@ -1,5 +1,9 @@ -use crate::TestServerBuilder; +use std::time::Duration; + +use lsp_types::ShowMessageNotification; + use crate::notebook::NotebookBuilder; +use crate::{AwaitResponseError, TestServer, TestServerBuilder}; use insta::assert_json_snapshot; #[test] @@ -80,3 +84,237 @@ type Style = Literal["italic", "bold", "underline"]"#, server.collect_publish_diagnostic_notifications(2); Ok(()) } + +/// a name that cannot be renamed is refused with the reason, which the editor +/// shows: a `null` answer gives it nothing to say, and to the user the rename +/// key then looks broken. the refusal is an answer rather than a failure of the +/// server, so no popup says the server hit a problem +#[test] +fn a_builtin_is_refused_with_the_reason() -> anyhow::Result<()> { + let mut server = TestServerBuilder::new()? + .with_file("foo.py", "")? + .build() + .wait_until_workspaces_are_initialized(); + + server.open_text_document("foo.py", "print(1)\n", 1); + + let id = server.send_request::(prepare_rename_params( + server.file_uri("foo.py"), + lsp_types::Position::new(0, 2), + )); + let response = server.try_await_response::(&id, None); + + let Err(AwaitResponseError::RequestFailed(failure)) = response else { + panic!("expected a refusal, got {response:?}"); + }; + assert_eq!(failure.code, lsp_server::ErrorCode::RequestFailed as i32); + assert_eq!(failure.message, "`print` is declared outside this project"); + + let popup = + server.try_await_notification::(Some(Duration::from_millis(100))); + assert!(popup.is_err(), "got {popup:?}"); + + Ok(()) +} + +/// a client that renames without asking first is refused just the same, rather +/// than handed an edit renaming every use of `print` in the project +#[test] +fn a_rename_of_a_builtin_is_refused_without_asking_first() -> anyhow::Result<()> { + let mut server = TestServerBuilder::new()? + .with_file("foo.py", "")? + .build() + .wait_until_workspaces_are_initialized(); + + server.open_text_document("foo.py", "print(1)\n", 1); + + let response = rename_at( + &mut server, + "foo.py", + lsp_types::Position::new(0, 2), + "show", + ); + + assert_eq!( + refusal(response), + "`print` is declared outside this project" + ); + + Ok(()) +} + +/// where there is no name at all there is nothing to explain, and the answer +/// stays `null` +#[test] +fn nothing_to_rename_is_still_a_null_answer() -> anyhow::Result<()> { + let mut server = TestServerBuilder::new()? + .with_file("foo.py", "")? + .build() + .wait_until_workspaces_are_initialized(); + + server.open_text_document("foo.py", "x = 1\n", 1); + + let prepared = server.send_request_await::( + prepare_rename_params(server.file_uri("foo.py"), lsp_types::Position::new(0, 4)), + ); + + assert!(prepared.is_none(), "got {prepared:?}"); + + Ok(()) +} + +/// a keyword argument a `.by` file names with a string is offered for renaming +/// by its name, so the editor does not start the new name with the quotes +#[test] +fn a_quoted_keyword_argument_is_offered_by_its_name() -> anyhow::Result<()> { + let mut server = TestServerBuilder::new()? + .with_file("foo.by", QUOTED_KEYWORD)? + .build() + .wait_until_workspaces_are_initialized(); + + server.open_text_document("foo.by", QUOTED_KEYWORD, 1); + + let prepared = server.send_request_await::( + prepare_rename_params(server.file_uri("foo.by"), lsp_types::Position::new(2, 4)), + ); + + assert_eq!( + prepared, + Some(lsp_types::PrepareRenameResult::PrepareRenamePlaceholder( + lsp_types::PrepareRenamePlaceholder::new( + lsp_types::Range::new( + lsp_types::Position::new(2, 2), + lsp_types::Position::new(2, 11) + ), + "timeout".to_string(), + ) + )) + ); + + Ok(()) +} + +const QUOTED_KEYWORD: &str = "def f(timeout: int): ...\n\nf(\"timeout\"=1)\n"; + +fn prepare_rename_params( + uri: lsp_types::Uri, + position: lsp_types::Position, +) -> lsp_types::PrepareRenameParams { + lsp_types::PrepareRenameParams { + text_document_position_params: lsp_types::TextDocumentPositionParams { + text_document: lsp_types::TextDocumentIdentifier { uri }, + position, + }, + work_done_progress_params: lsp_types::WorkDoneProgressParams::default(), + } +} + +/// the quotes are part of what is replaced, so the argument is left written +/// the plain way, matching the parameter +#[test] +fn a_quoted_keyword_argument_is_renamed_to_a_bare_name() -> anyhow::Result<()> { + let mut server = TestServerBuilder::new()? + .with_file("foo.by", QUOTED_KEYWORD)? + .build() + .wait_until_workspaces_are_initialized(); + + server.open_text_document("foo.by", QUOTED_KEYWORD, 1); + + let edit = rename_at( + &mut server, + "foo.by", + lsp_types::Position::new(2, 4), + "retries", + ) + .expect("the rename is carried out") + .expect("an edit"); + + let edits = edit + .changes + .expect("edits by file") + .remove(&server.file_uri("foo.by")) + .expect("edits to foo.by"); + + assert_eq!( + apply(QUOTED_KEYWORD, edits), + "def f(retries: int): ...\n\nf(retries=1)\n" + ); + + Ok(()) +} + +/// a new name that is not a name, or is a keyword, would leave the code unable +/// to parse, so the rename is refused rather than written +#[test] +fn a_new_name_that_is_not_a_name_is_refused() -> anyhow::Result<()> { + let mut server = TestServerBuilder::new()? + .with_file("foo.by", QUOTED_KEYWORD)? + .build() + .wait_until_workspaces_are_initialized(); + + server.open_text_document("foo.by", QUOTED_KEYWORD, 1); + + let quoted = rename_at( + &mut server, + "foo.by", + lsp_types::Position::new(2, 4), + "\"retries\"", + ); + assert_eq!(refusal(quoted), "`\"retries\"` is not a valid name"); + + let keyword = rename_at( + &mut server, + "foo.by", + lsp_types::Position::new(2, 4), + "class", + ); + assert_eq!(refusal(keyword), "`class` is a keyword"); + + Ok(()) +} + +fn rename_at( + server: &mut TestServer, + path: &str, + position: lsp_types::Position, + new_name: &str, +) -> Result, AwaitResponseError> { + let id = server.send_request::(lsp_types::RenameParams { + text_document_position_params: lsp_types::TextDocumentPositionParams { + text_document: lsp_types::TextDocumentIdentifier { + uri: server.file_uri(path), + }, + position, + }, + new_name: new_name.to_string(), + work_done_progress_params: lsp_types::WorkDoneProgressParams::default(), + }); + server.try_await_response::(&id, None) +} + +/// the reason a request was refused, asserting that it was +#[track_caller] +fn refusal(response: Result) -> String { + let Err(AwaitResponseError::RequestFailed(failure)) = response else { + panic!("expected a refusal, got {response:?}"); + }; + assert_eq!(failure.code, lsp_server::ErrorCode::RequestFailed as i32); + failure.message +} + +/// `source` with `edits` applied, for edits that each stay on one line of +/// ascii text +fn apply(source: &str, mut edits: Vec) -> String { + let mut lines: Vec = source.split_inclusive('\n').map(str::to_string).collect(); + edits + .sort_by_key(|edit| std::cmp::Reverse((edit.range.start.line, edit.range.start.character))); + for edit in edits { + assert_eq!(edit.range.start.line, edit.range.end.line); + let line = &mut lines[edit.range.start.line as usize]; + line.replace_range( + edit.range.start.character as usize..edit.range.end.character as usize, + &edit.new_text, + ); + } + lines.concat() +} diff --git a/crates/ty_server/tests/e2e/script_preparation.rs b/crates/ty_server/tests/e2e/script_preparation.rs index e185bcb5d4..c9f3d7208d 100644 --- a/crates/ty_server/tests/e2e/script_preparation.rs +++ b/crates/ty_server/tests/e2e/script_preparation.rs @@ -74,8 +74,9 @@ fn created_closed_script_is_prepared_after_a_file_event() -> Result<()> { kind: lsp_types::FileChangeType::Created, }]); - // The file event and completed synchronization each refresh diagnostics. - server.await_diagnostic_refresh(); + // the finished synchronization refreshes diagnostics. the file event may not: until + // something has listed the project's files, a new one changes nothing the server has read. + // the workspace request waits for the script's environment either way server.await_diagnostic_refresh(); let report = server.workspace_diagnostic_request(None, None); assert_snapshot!(condensed_workspace_diagnostic_snapshot(report), @" diff --git a/crates/ty_server/tests/e2e/super_members.rs b/crates/ty_server/tests/e2e/super_members.rs new file mode 100644 index 0000000000..76c4f75fbf --- /dev/null +++ b/crates/ty_server/tests/e2e/super_members.rs @@ -0,0 +1,363 @@ +//! `by/superMembers` and `by/documentSuperMembers` over the wire +//! +//! which members a member overrides is `ty_ide`'s and its unit tests cover it. what these cover is +//! the contract a client reads: a position on a declared name in, and out the members in MRO order, +//! each in the file that declares it with the name to land on — and `null` kept apart from an empty +//! list, which a client tells a user two different things about. and for a whole document, every +//! member that overrides something with the same members, and the name each is declared with, about +//! the text the client names + +use anyhow::Result; +use lsp_types::{ + LanguageKind, LspRequestMethod, MessageDirection, Request, TextDocumentContentChangeEvent, + TextDocumentContentChangeWholeDocument, +}; +use ruff_db::system::SystemPath; +use ty_server::ClientOptions; + +use crate::notebook::NotebookBuilder; +use crate::{TestServer, TestServerBuilder, text_hash}; + +/// the request as a client sends it, in json throughout +enum SuperMembers {} + +impl Request for SuperMembers { + type Params = serde_json::Value; + type Result = Option; + const METHOD: LspRequestMethod<'static> = LspRequestMethod::Custom("by/superMembers"); + const MESSAGE_DIRECTION: MessageDirection = MessageDirection::ClientToServer; +} + +/// `by/documentSuperMembers`, in json throughout +enum DocumentSuperMembers {} + +impl Request for DocumentSuperMembers { + type Params = serde_json::Value; + type Result = Option; + const METHOD: LspRequestMethod<'static> = LspRequestMethod::Custom("by/documentSuperMembers"); + const MESSAGE_DIRECTION: MessageDirection = MessageDirection::ClientToServer; +} + +const BASES: &str = "\ +class Root: + def speak(self) -> str: + return '' + +class Loud(Root): + def speak(self) -> str: + return 'HEY' + +class Quiet(Root): + def speak(self) -> str: + return 'hey' +"; + +const MAIN: &str = "\ +from bases import Loud, Quiet + +class Both(Loud, Quiet): + override def speak(self) -> str: + return 'hi' + + def listen(self) -> None: ... + +def free() -> None: ... +"; + +fn server() -> Result { + let mut server = TestServerBuilder::new()? + .with_initialization_options(&ClientOptions::default()) + .with_workspace(SystemPath::new("src"), None)? + .with_file(SystemPath::new("src/bases.by"), BASES)? + .with_file(SystemPath::new("src/main.by"), MAIN)? + .build() + .wait_until_workspaces_are_initialized(); + server.open_text_document(SystemPath::new("src/main.by"), MAIN, 1); + Ok(server) +} + +fn ask(server: &mut TestServer, line: u32, character: u32) -> Option { + let uri = server.file_uri(SystemPath::new("src/main.by")); + server.send_request_await::(serde_json::json!({ + "textDocument": { "uri": uri }, + "position": { "line": line, "character": character }, + })) +} + +#[test] +fn a_method_of_a_class_with_two_bases_overrides_each_in_mro_order() -> Result<()> { + let mut server = server()?; + let bases = server.file_uri(SystemPath::new("src/bases.by")); + + // on `speak` in `override def speak` + let answer = ask(&mut server, 3, 17).expect("a member is declared there"); + let members = answer.as_array().expect("a list of members"); + assert_eq!(members.len(), 2, "{answer:#}"); + + for (member, (container, line)) in members.iter().zip([("Loud", 5), ("Quiet", 9)]) { + assert_eq!(member["name"], "speak"); + assert_eq!(member["containerName"], container); + assert_eq!(member["uri"], bases.as_str()); + assert_eq!(member["synthesized"], false); + assert_eq!(member["abstract"], false); + assert_eq!( + member["selectionRange"], + serde_json::json!({ + "start": { "line": line, "character": 8 }, + "end": { "line": line, "character": 13 }, + }), + ); + assert_eq!(member["range"]["start"]["line"], line); + } + Ok(()) +} + +#[test] +fn a_member_that_overrides_nothing_is_an_empty_list_and_no_member_is_null() -> Result<()> { + let mut server = server()?; + + // on `listen` + assert_eq!(ask(&mut server, 6, 9), Some(serde_json::json!([]))); + // on `free`, a function no class declares + assert_eq!(ask(&mut server, 8, 5), None); + // inside a method body rather than on a declared name + assert_eq!(ask(&mut server, 4, 9), None); + Ok(()) +} + +fn ask_document(server: &mut TestServer, text_hash: Option<&str>) -> serde_json::Value { + let mut params = serde_json::json!({ + "textDocument": { "uri": server.file_uri(SystemPath::new("src/main.by")) }, + }); + if let Some(hash) = text_hash { + params["textHash"] = serde_json::json!(hash); + } + server + .send_request_await::(params) + .expect("an answer for a document") +} + +#[test] +fn a_document_lists_the_members_that_override_something_as_super_members_answers_each() -> Result<()> +{ + let mut server = server()?; + + let answer = ask_document(&mut server, None); + let members = answer.as_array().expect("a list of members"); + // `listen` overrides nothing, and `free` is no member + assert_eq!(members.len(), 1, "{answer:#}"); + let speak = &members[0]; + assert_eq!(speak["name"], "speak"); + assert_eq!(speak["containerName"], "Both"); + assert_eq!(speak["abstract"], false); + assert_eq!( + speak["selectionRange"], + serde_json::json!({ + "start": { "line": 3, "character": 17 }, + "end": { "line": 3, "character": 22 }, + }), + ); + // the very members `by/superMembers` answers at the name + assert_eq!( + Some(speak["superMembers"].clone()), + ask(&mut server, 3, 17), + "{answer:#}" + ); + Ok(()) +} + +/// a protocol member, and a member two classes up: what an editor marks as implemented rather +/// than overridden, and what it goes to past a class that does not declare it +#[test] +fn a_protocol_member_is_abstract_and_a_member_two_levels_up_is_found() -> Result<()> { + let main = "\ +from typing import Protocol + +class Greeter(Protocol): + def greet(self) -> str: ... + +class A: + def f(self) -> None: ... + +class B(A): ... + +class C(B, Greeter): + def f(self) -> None: ... + + def greet(self) -> str: + return 'hi' +"; + let mut server = TestServerBuilder::new()? + .with_initialization_options(&ClientOptions::default()) + .with_workspace(SystemPath::new("src"), None)? + .with_file(SystemPath::new("src/main.by"), main)? + .build() + .wait_until_workspaces_are_initialized(); + server.open_text_document(SystemPath::new("src/main.by"), main, 1); + + let answer = ask_document(&mut server, None); + let members = answer.as_array().expect("a list of members"); + let summary: Vec<_> = members + .iter() + .map(|member| { + let overridden = &member["superMembers"][0]; + ( + member["name"].as_str().unwrap().to_string(), + member["selectionRange"]["start"]["line"].as_u64().unwrap(), + overridden["containerName"].as_str().unwrap().to_string(), + overridden["selectionRange"]["start"]["line"] + .as_u64() + .unwrap(), + overridden["abstract"].as_bool().unwrap(), + ) + }) + .collect(); + assert_eq!( + summary, + vec![ + ("f".to_string(), 11, "A".to_string(), 6, false), + ("greet".to_string(), 13, "Greeter".to_string(), 3, true), + ], + "{answer:#}" + ); + Ok(()) +} + +/// the same text-naming every document request takes: a document the client has not opened is +/// answered from the file when that is the text named, and a text the server does not hold yet is +/// waited for +#[test] +fn a_document_request_is_answered_about_the_text_it_names() -> Result<()> { + let mut server = TestServerBuilder::new()? + .with_initialization_options(&ClientOptions::default()) + .with_workspace(SystemPath::new("src"), None)? + .with_file(SystemPath::new("src/bases.by"), BASES)? + .with_file(SystemPath::new("src/main.by"), MAIN)? + .build() + .wait_until_workspaces_are_initialized(); + + // not opened: the file on disk, which is the text named + let answer = ask_document(&mut server, Some(&text_hash(MAIN))); + assert_eq!( + answer[0]["selectionRange"]["start"]["line"], 3, + "{answer:#}" + ); + + // an edit the server has not been sent moves `speak` down a line, and is waited for. the server + // takes its messages in order, so an answer made when the request arrived would be about the + // text before the edit, with `speak` on line 3 + server.open_text_document(SystemPath::new("src/main.by"), MAIN, 1); + let edited = format!("\n{MAIN}"); + let id = server.send_request::(serde_json::json!({ + "textDocument": { "uri": server.file_uri(SystemPath::new("src/main.by")) }, + "textHash": text_hash(&edited), + })); + server.change_text_document( + SystemPath::new("src/main.by"), + vec![ + TextDocumentContentChangeEvent::TextDocumentContentChangeWholeDocument( + TextDocumentContentChangeWholeDocument { text: edited }, + ), + ], + 2, + ); + let answer = server + .await_response::(&id) + .expect("an answer about the edit"); + assert_eq!( + answer[0]["selectionRange"]["start"]["line"], 4, + "{answer:#}" + ); + Ok(()) +} + +/// `by/superMembers` names its text as the document request does: answered from the file for a +/// document the client has not opened, and held for an edit the server has not been sent +#[test] +fn a_member_request_is_answered_about_the_text_it_names() -> Result<()> { + let mut server = TestServerBuilder::new()? + .with_initialization_options(&ClientOptions::default()) + .with_workspace(SystemPath::new("src"), None)? + .with_file(SystemPath::new("src/bases.by"), BASES)? + .with_file(SystemPath::new("src/main.by"), MAIN)? + .build() + .wait_until_workspaces_are_initialized(); + let uri = server.file_uri(SystemPath::new("src/main.by")); + + // not opened: the file on disk, which is the text named + let answer = server + .send_request_await::(serde_json::json!({ + "textDocument": { "uri": uri }, + "position": { "line": 3, "character": 17 }, + "textHash": text_hash(MAIN), + })) + .expect("the members `speak` overrides"); + assert_eq!(answer.as_array().map(Vec::len), Some(2), "{answer:#}"); + + // `speak` a line down in an edit not sent yet: held until it is. the server takes its messages + // in order, so an answer made when the request arrived would be about the text before the + // edit, where that position is inside a method body and has no member + server.open_text_document(SystemPath::new("src/main.by"), MAIN, 1); + let edited = format!("\n{MAIN}"); + let id = server.send_request::(serde_json::json!({ + "textDocument": { "uri": uri }, + "position": { "line": 4, "character": 17 }, + "textHash": text_hash(&edited), + })); + server.change_text_document( + SystemPath::new("src/main.by"), + vec![ + TextDocumentContentChangeEvent::TextDocumentContentChangeWholeDocument( + TextDocumentContentChangeWholeDocument { text: edited }, + ), + ], + 2, + ); + let answer = server + .await_response::(&id) + .expect("the members `speak` overrides, in the edit"); + assert_eq!(answer.as_array().map(Vec::len), Some(2), "{answer:#}"); + Ok(()) +} + +/// a notebook's ranges are per cell and a template is not python: neither has its members answered +/// for the whole document, and a template has none at a position either, even one whose text +/// would read as python +#[test] +fn a_notebook_or_a_template_is_answered_null() -> Result<()> { + let template = "class A:\n x = 1\n\nclass B(A):\n x = 2\n"; + let mut server = TestServerBuilder::new()? + .with_initialization_options(&ClientOptions::default()) + .with_workspace(SystemPath::new("src"), None)? + .with_file(SystemPath::new("src/templates/page.html"), template)? + .build() + .wait_until_workspaces_are_initialized(); + + let mut notebook = NotebookBuilder::virtual_file("test.ipynb"); + notebook.add_python_cell("class A:\n def f(self) -> None: ...\n"); + let cell = notebook.add_python_cell("class B(A):\n def f(self) -> None: ...\n"); + notebook.open(&mut server); + server.collect_publish_diagnostic_notifications(2); + let answer = server.send_request_await::(serde_json::json!({ + "textDocument": { "uri": cell }, + })); + assert_eq!(answer, None); + + server.open_text_document_as( + SystemPath::new("src/templates/page.html"), + template, + 1, + LanguageKind::new("django-html"), + ); + let uri = server.file_uri(SystemPath::new("src/templates/page.html")); + let answer = server.send_request_await::(serde_json::json!({ + "textDocument": { "uri": uri }, + })); + assert_eq!(answer, None); + let answer = server.send_request_await::(serde_json::json!({ + "textDocument": { "uri": uri }, + "position": { "line": 4, "character": 4 }, + })); + assert_eq!(answer, None); + Ok(()) +} diff --git a/crates/ty_wasm/src/lib.rs b/crates/ty_wasm/src/lib.rs index c801570226..c75dd45859 100644 --- a/crates/ty_wasm/src/lib.rs +++ b/crates/ty_wasm/src/lib.rs @@ -20,7 +20,7 @@ use ruff_text_size::{Ranged, TextSize}; use ty_ide::{ CompletionCapabilities, Hint as IdeHint, InlayHintSettings, MarkupKind, RangedValue, can_rename, document_highlights, find_references, goto_declaration, goto_definition, - goto_type_definition, hover, inlay_hints, rename, + goto_type_definition, hover, inlay_hints, invalid_new_name, prepare_rename, rename, }; use ty_ide::{NavigationTarget, NavigationTargets, hints, signature_help}; use ty_project::metadata::options::Options; @@ -444,22 +444,36 @@ impl Workspace { &self, file_id: &FileHandle, position: Position, - ) -> Result, Error> { + ) -> Result { let source = source_text(&self.db, file_id.file); let index = line_index(&self.db, file_id.file); let offset = position.to_text_size(&source, &index, self.position_encoding)?; - let Some(range) = can_rename(&self.db, self.db.program_file(file_id.file), offset) else { - return Ok(None); - }; - - Ok(Some(Range::from_text_range( - range, - &index, - &source, - self.position_encoding, - ))) + Ok( + match prepare_rename(&self.db, self.db.program_file(file_id.file), offset) { + ty_ide::PreparedRename::Ready { range, placeholder } => PreparedRename { + range: Some(Range::from_text_range( + range, + &index, + &source, + self.position_encoding, + )), + placeholder: Some(placeholder), + refusal: None, + }, + ty_ide::PreparedRename::Refused(why) => PreparedRename { + range: None, + placeholder: None, + refusal: Some(why), + }, + ty_ide::PreparedRename::NoSymbol => PreparedRename { + range: None, + placeholder: None, + refusal: None, + }, + }, + ) } #[wasm_bindgen] @@ -479,6 +493,10 @@ impl Workspace { return Ok(Vec::new()); } + if let Some(why) = invalid_new_name(new_name) { + return Err(Error::new(&why)); + } + let Some(rename_results) = rename(&self.db, program_file, offset, new_name) else { return Ok(Vec::new()); }; @@ -1450,6 +1468,19 @@ pub struct TextEdit { pub new_text: String, } +/// what a rename at a position would be: the range of the name to rename and +/// the name as it is offered for editing, or, when there is a name that cannot +/// be renamed, why not +#[wasm_bindgen] +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct PreparedRename { + pub range: Option, + #[wasm_bindgen(getter_with_clone)] + pub placeholder: Option, + #[wasm_bindgen(getter_with_clone)] + pub refusal: Option, +} + #[wasm_bindgen] #[derive(Debug, Clone, PartialEq, Eq)] pub struct RenameEdit { @@ -1469,25 +1500,10 @@ pub enum InlayHintKind { impl From for InlayHintKind { fn from(kind: ty_ide::InlayHintKind) -> Self { - match kind { - ty_ide::InlayHintKind::Type - // basedpython: an inferred exception set is a type, like a return type - | ty_ide::InlayHintKind::Raises - | ty_ide::InlayHintKind::Variance - | ty_ide::InlayHintKind::Reification - | ty_ide::InlayHintKind::TypeArgument - | ty_ide::InlayHintKind::Override - | ty_ide::InlayHintKind::Reads - | ty_ide::InlayHintKind::Stability - | ty_ide::InlayHintKind::DerivedDeps - | ty_ide::InlayHintKind::Invalidates - | ty_ide::InlayHintKind::NumericPromotion - | ty_ide::InlayHintKind::RevealedType - | ty_ide::InlayHintKind::EnumValue => Self::Type, - ty_ide::InlayHintKind::CallArgumentName - | ty_ide::InlayHintKind::ImplicitParameter - | ty_ide::InlayHintKind::ImplicitArgument - | ty_ide::InlayHintKind::InheritedDefault => Self::Parameter, + if kind.is_parameter() { + Self::Parameter + } else { + Self::Type } } } diff --git a/docs/basedpython/features/editor.md b/docs/basedpython/features/editor.md index fa5c16de26..e676cff7ac 100644 --- a/docs/basedpython/features/editor.md +++ b/docs/basedpython/features/editor.md @@ -229,6 +229,18 @@ on | `templateBindingTypes` | a django template `{% for %}` binding's element type | | `resolvedTemplates` | the file a django `{% extends %}` name resolves to | +every hint also carries its setting's name as `data.kind` — +`{"kind": "revealedTypes"}` — so an editor can show each kind in its own way, +or only while a key is held, without telling kinds apart by their text. lsp's +own `kind` is only `Type` or `Parameter`, and a revealed `list[int]` and a +call's `[int]` differ by a bracket + +a hint is shown or hidden by its own setting alone, but one setting also +changes what other hints say: in a `.by` file, `typeArgumentNames` names the +type arguments of the types that `variableTypes` and `callTypeArguments` hints +show, so `x = identity({"a": 1})` is hinted `: dict[Key=str, Value=int]` with it +on and `: dict[str, int]` with it off + ### keeping a hand-aligned block aligned a hint takes up room. drawn after the name, it pushes the rest of the line along @@ -375,6 +387,87 @@ the pairing is read off the parse tree, so `match = 1` is an assignment and nothing lights up on it. a position that is not a keyword is answered the way it always was +## go to super + +`textDocument/implementation` on a method goes down, to the methods that +override it. `by/superMembers` goes up: given a position on the name a class +member is declared with — a `def` or a nested class, a name the class body +assigns, annotates or captures, or an import's alias — it answers the +superclass members that member overrides, each with its class, its file and the +name to land on. a class goes up with `typeHierarchy/supertypes` + +what counts as an override is the override checks' own walk up the MRO, from +the class as it is written rather than as a decorator returns it. of what that +walk finds, a member goes to the nearest declaration along each branch, in MRO +order: a method overriding `B.f`, where `B.f` overrides `A.f`, goes to `B.f`, +and a class with two bases that each declare the method goes to both. the first +of them is the one `missing-override-decorator` names. a `private` member +overrides nothing, since it is emitted under a name of its own, and a member a +superclass synthesizes rather than writes — a dataclass's `__init__` — goes to +that superclass, marked `synthesized`. each is marked `abstract` when it is +abstract where it is declared — an `@abstractmethod`, or a protocol method with +no implementation — so that a member overriding it implements it. the answer is +`null` when no class member is declared at the position, and an empty list for +one that overrides nothing + +an answer depends on more than the text of the document it is about: on every +file the document's classes inherit from, and the server announces no change to +it. a client that keeps one asks again after it opens, edits or closes any +document of the workspace, not only that one, and on +`workspace/inlayHint/refresh`, which the server sends, to a client that supports +it, after a change on disk or to the environment changes what it answers from + +`by/documentSuperMembers` answers the same for a whole document in one request, +from the same code: every class member that overrides something, with its +class, the range of the name it is declared with, whether it is `abstract` +itself, and its `superMembers` as `by/superMembers` gives them. it is what an +editor asks to draw an "overrides" marker beside each member, on every pass over +the document. a property with `get` and `set` blocks is one member, listed once. +constructors and the methods they call — `__init__`, `__new__`, +`__post_init__`, `__init_subclass__` — are left out, since neither +`invalid-method-override` nor `missing-override-decorator` holds one to what it +overrides, and `by/superMembers` still answers one. like the other document +requests it takes `textHash`, and answers a document the client has not opened +from the file on disk + +## the whole project's diagnostics + +with `diagnosticMode` set to `workspace`, the server checks every file in the +project, open or not, and reports what it finds two ways: + +| request | answers | +| ---------------------- | ---------------------------------------------------------------------- | +| `workspace/diagnostic` | once something differs from the result ids sent — held open until then | +| `by/checkWorkspace` | as soon as the check is done, whether or not anything differs | + +both take the same parameters and report the same thing, file by file, with an +unchanged report for a file whose result id still matches. `workspace/diagnostic` +is the one an editor keeps a problems list current with: it asks again as soon +as it is answered, and the server answers when an edit or a change on disk +changes something. `by/checkWorkspace` is for a client that asks once — an +inspection over the whole project, a batch run with no editor — and would +otherwise wait forever on a project with nothing wrong in it. it is refused, +rather than answered empty, when the diagnostic mode does not check the +workspace + +they differ in how the report arrives. `workspace/diagnostic` streams it as +partial results when the client sends a `partialResultToken`, and reports +progress under the client's `workDoneToken`. `by/checkWorkspace` ignores both +tokens: its report comes whole, in the response, and its progress, when the +client supports `window/workDoneProgress`, comes under a token the server +creates + +`by/checkWorkspace` can take a while to answer, for two reasons: + +- an edit or a change on disk that lands while the workspace is being checked + starts the check again, on the changed workspace. the answer describes the + workspace as it was when a check finished, and a workspace that never stops + changing is never answered +- while a PEP 723 script's environment is being set up for the first time, the + workspace is not checked, and the request waits until the environment is + there. the first setup installs the script's dependencies, so it can take as + long as that does + ## the program model a run configuration, a test list and a "go to generated file" are all questions @@ -413,10 +506,11 @@ which text a document request is about, by adding `textHash` to the request's params: FNV-1a, 64 bits, over the text's UTF-16 code units with every line ending counted as one `\n`, written as sixteen lowercase hex digits. for a notebook cell that is the cell's own text. the request is then answered about that text and no -other. when the server already holds it — the -open buffer, or for `by/syntaxOutline`, `by/injections`, -`textDocument/semanticTokens/full` and `textDocument/documentSymbol` the file on -disk of a document the client has not opened — it answers at once; otherwise the +other. when the server already holds it — the open buffer, or for +`by/syntaxOutline`, `by/injections`, `by/superMembers`, +`by/documentSuperMembers`, `textDocument/semanticTokens/full` and +`textDocument/documentSymbol` the file on disk of a document the client has not +opened — it answers at once; otherwise the request waits for the `didOpen`, `didChange` or file write that brings the text, and is answered with `ServerCancelled` if nothing has after ten seconds. so a client need not know whether its `didOpen` has gone out before it asks, and an diff --git a/docs/basedpython/features/sound-types.md b/docs/basedpython/features/sound-types.md index 47e275e16b..f020ea825d 100644 --- a/docs/basedpython/features/sound-types.md +++ b/docs/basedpython/features/sound-types.md @@ -344,7 +344,7 @@ contradiction at every call site and never where it lives ## unannotated return types a function with no return annotation returns what its body returns: the union of every `return` -expression, plus `None` when control can also fall off the end +expression that can be reached, plus `None` when control can also fall off the end ```python def f(): @@ -361,6 +361,25 @@ class X(Protocol): def f(self): ... # (self) -> None ``` +a statement that calls a function which returns `Never` ends the flow, as a call to a `NoReturn` +function does, and the code after it is unreachable. a function returns `Never` when no path through +its body reaches a `return` or its end without passing a `raise`, a loop that never exits, or a +call that cannot return either. a function that only ever calls itself never returns, since each +call makes another until `RecursionError` + +```python +import sys + +def die(): + sys.exit(1) + +def f() -> int: + die() # no `invalid-return-type`: the end of `f` is unreachable +``` + +a `return` whose value never finishes evaluating, such as `return sys.exit(1)`, is the exception: the +function returns `Never`, but a call to it is still treated as one that can return + a generator returns a generator: its `yield` expressions supply the yield type and its `return` statements the third type argument. the send type is the one thing the body does not determine, so it stays gradual diff --git a/playground/ty/src/Editor/Editor.tsx b/playground/ty/src/Editor/Editor.tsx index 08537d918e..dd5473fe7c 100644 --- a/playground/ty/src/Editor/Editor.tsx +++ b/playground/ty/src/Editor/Editor.tsx @@ -265,11 +265,11 @@ class PlaygroundServer signatureHelpTriggerCharacters: string[] = ["(", ","]; signatureHelpRetriggerCharacters: string[] = [")"]; - private renameRejection(): T & languages.Rejection { + private renameRejection(reason?: string): T & languages.Rejection { // This assertion is intentionally unchecked: Monaco checks `rejectReason` // before reading the success fields, but its RenameProvider declaration // models rejections as part of the success type. - return { rejectReason: "this element can't be renamed" } as T & + return { rejectReason: reason ?? "this element can't be renamed" } as T & languages.Rejection; } @@ -879,20 +879,21 @@ class PlaygroundServer return this.renameRejection(); } - const range = this.props.workspace.prepareRename( + const prepared = this.props.workspace.prepareRename( fileHandle, new TyPosition(position.lineNumber, position.column), ); + const range = prepared.range; if (range == null) { - return this.renameRejection(); + return this.renameRejection(prepared.refusal); } const monacoRange = tyRangeToMonacoRange(range); return { range: monacoRange, - text: model.getValueInRange(monacoRange), + text: prepared.placeholder ?? model.getValueInRange(monacoRange), }; } @@ -908,11 +909,19 @@ class PlaygroundServer return this.renameRejection(); } - const renameEdits = this.props.workspace.rename( - fileHandle, - new TyPosition(position.lineNumber, position.column), - newName, - ); + let renameEdits; + try { + renameEdits = this.props.workspace.rename( + fileHandle, + new TyPosition(position.lineNumber, position.column), + newName, + ); + } catch (error) { + // a new name that is not a valid name is refused with the reason + return this.renameRejection( + error instanceof Error ? error.message : undefined, + ); + } if (renameEdits.length === 0) { return this.renameRejection();