From 88ed708e8d23ea31a82f577046f2cb9995a211b7 Mon Sep 17 00:00:00 2001 From: joboet Date: Mon, 3 Aug 2026 13:46:42 +0200 Subject: [PATCH 01/15] core: generalize `BorrowedCursor::ensure_init` --- library/core/src/io/borrowed_buf.rs | 54 +++++++++++++++++++++++------ 1 file changed, 43 insertions(+), 11 deletions(-) diff --git a/library/core/src/io/borrowed_buf.rs b/library/core/src/io/borrowed_buf.rs index 7ca6f6d8a02e8..7dac09f14e64e 100644 --- a/library/core/src/io/borrowed_buf.rs +++ b/library/core/src/io/borrowed_buf.rs @@ -2,7 +2,6 @@ use crate::fmt::{self, Debug, Formatter}; use crate::mem::{self, MaybeUninit}; -use crate::ptr; /// A borrowed buffer of initially uninitialized elements, which is incrementally filled. /// @@ -357,24 +356,57 @@ impl<'a, T: Copy> BorrowedCursor<'a, T> { } } -impl<'a> BorrowedCursor<'a, u8> { - /// Initializes all bytes in the cursor and returns them. +impl<'a, T: Default + Copy> BorrowedCursor<'a, T> { + /// Initializes all elements in the cursor with their default value and + /// returns them. #[unstable(feature = "borrowed_buf_init", issue = "160476")] #[inline] - pub fn ensure_init(&mut self) -> &mut [u8] { - // SAFETY: always in bounds and we never uninitialize these bytes. + pub fn ensure_init(&mut self) -> &mut [T] { + trait InitSpec: Default + Copy { + fn initialize(buf: &mut [MaybeUninit]); + } + + impl InitSpec for T { + default fn initialize(buf: &mut [MaybeUninit]) { + buf.write_with(|_| Self::default()); + } + } + + macro_rules! spec_zero_init { + ($ty:ty) => { + impl InitSpec for $ty { + fn initialize(buf: &mut [MaybeUninit]) { + // SAFETY: all these types can be zero-initialized. + unsafe { + buf.as_mut_ptr().write_bytes(0, buf.len()); + } + } + } + }; + } + + spec_zero_init!(i8); + spec_zero_init!(u8); + spec_zero_init!(i16); + spec_zero_init!(u16); + spec_zero_init!(i32); + spec_zero_init!(u32); + spec_zero_init!(i64); + spec_zero_init!(u64); + spec_zero_init!(i128); + spec_zero_init!(u128); + spec_zero_init!(isize); + spec_zero_init!(usize); + + // SAFETY: always in bounds and we never uninitialize these elements. let unfilled = unsafe { self.buf.buf.get_unchecked_mut(self.buf.filled..) }; if !self.buf.init { - // SAFETY: 0 is a valid value for MaybeUninit and the length matches the allocation - // since it is comes from a slice reference. - unsafe { - ptr::write_bytes(unfilled.as_mut_ptr(), 0, unfilled.len()); - } + InitSpec::initialize(unfilled); self.buf.init = true; } - // SAFETY: these bytes have just been initialized if they weren't before + // SAFETY: these elements have just been initialized if they weren't before unsafe { unfilled.assume_init_mut() } } } From 0100285a6e9d69bbc16c81af8d9037f9712c0186 Mon Sep 17 00:00:00 2001 From: joboet Date: Tue, 4 Aug 2026 16:35:31 +0200 Subject: [PATCH 02/15] core: add `slice::write_default` --- library/core/src/io/borrowed_buf.rs | 38 +-------------- library/core/src/mem/maybe_uninit.rs | 71 +++++++++++++++++++++++++++- 2 files changed, 70 insertions(+), 39 deletions(-) diff --git a/library/core/src/io/borrowed_buf.rs b/library/core/src/io/borrowed_buf.rs index 7dac09f14e64e..f2926f1ce7cf9 100644 --- a/library/core/src/io/borrowed_buf.rs +++ b/library/core/src/io/borrowed_buf.rs @@ -362,47 +362,11 @@ impl<'a, T: Default + Copy> BorrowedCursor<'a, T> { #[unstable(feature = "borrowed_buf_init", issue = "160476")] #[inline] pub fn ensure_init(&mut self) -> &mut [T] { - trait InitSpec: Default + Copy { - fn initialize(buf: &mut [MaybeUninit]); - } - - impl InitSpec for T { - default fn initialize(buf: &mut [MaybeUninit]) { - buf.write_with(|_| Self::default()); - } - } - - macro_rules! spec_zero_init { - ($ty:ty) => { - impl InitSpec for $ty { - fn initialize(buf: &mut [MaybeUninit]) { - // SAFETY: all these types can be zero-initialized. - unsafe { - buf.as_mut_ptr().write_bytes(0, buf.len()); - } - } - } - }; - } - - spec_zero_init!(i8); - spec_zero_init!(u8); - spec_zero_init!(i16); - spec_zero_init!(u16); - spec_zero_init!(i32); - spec_zero_init!(u32); - spec_zero_init!(i64); - spec_zero_init!(u64); - spec_zero_init!(i128); - spec_zero_init!(u128); - spec_zero_init!(isize); - spec_zero_init!(usize); - // SAFETY: always in bounds and we never uninitialize these elements. let unfilled = unsafe { self.buf.buf.get_unchecked_mut(self.buf.filled..) }; if !self.buf.init { - InitSpec::initialize(unfilled); + unfilled.write_default(); self.buf.init = true; } diff --git a/library/core/src/mem/maybe_uninit.rs b/library/core/src/mem/maybe_uninit.rs index 9bc8b0d128d2b..6275d7cd59a2c 100644 --- a/library/core/src/mem/maybe_uninit.rs +++ b/library/core/src/mem/maybe_uninit.rs @@ -1286,8 +1286,8 @@ impl [MaybeUninit] { /// Fills a slice with elements returned by calling a closure for each index. /// /// This method uses a closure to create new values. If you'd rather `Clone` a given value, use - /// [slice::write_filled]. If you want to use the `Default` trait to generate values, you can - /// pass [`|_| Default::default()`][Default::default] as the argument. + /// [`slice::write_filled`]. If you want to use the `Default` trait to generate values, use + /// [`slice::write_default`]. /// /// # Panics /// @@ -1324,6 +1324,73 @@ impl [MaybeUninit] { unsafe { self.assume_init_mut() } } + /// Fills a slice with elements returned by calling [`Default::default`] for each index. + /// + /// # Panics + /// + /// This function will panic if any call to [`Default::default`] panics. + /// + /// If such a panic occurs, any elements previously initialized during this operation will be + /// dropped. + /// + /// # Examples + /// + /// ``` + /// #![feature(maybe_uninit_fill)] + /// use std::mem::MaybeUninit; + /// + /// let mut buf = [const { MaybeUninit::::uninit() }; 5]; + /// let initialized = buf.write_default(); + /// assert_eq!(initialized, &mut [0, 0, 0, 0, 0]); + /// ``` + #[unstable(feature = "maybe_uninit_fill", issue = "117428")] + pub fn write_default(&mut self) -> &mut [T] + where + T: Default, + { + trait DefaultSpec: Default { + fn write_default(buf: &mut [MaybeUninit]) -> &mut [Self]; + } + + impl DefaultSpec for T { + default fn write_default(buf: &mut [MaybeUninit]) -> &mut [Self] { + buf.write_with(|_| T::default()) + } + } + + macro_rules! spec_default_zero { + ($ty:ty) => { + impl DefaultSpec for $ty { + fn write_default(buf: &mut [MaybeUninit]) -> &mut [Self] { + // SAFETY: + // `Default::default` is equivalent to zero-initialization + // for all these types, and this initializes the entire + // slice. + unsafe { + buf.as_mut_ptr().write_bytes(0, buf.len()); + buf.assume_init_mut() + } + } + } + }; + } + + spec_default_zero!(i8); + spec_default_zero!(u8); + spec_default_zero!(i16); + spec_default_zero!(u16); + spec_default_zero!(i32); + spec_default_zero!(u32); + spec_default_zero!(i64); + spec_default_zero!(u64); + spec_default_zero!(i128); + spec_default_zero!(u128); + spec_default_zero!(isize); + spec_default_zero!(usize); + + T::write_default(self) + } + /// Fills a slice with elements yielded by an iterator until either all elements have been /// initialized or the iterator is empty. /// From e52a10e4b505a54ac642d16476f9f1fef8d3cafe Mon Sep 17 00:00:00 2001 From: Makai Date: Tue, 11 Aug 2026 00:32:47 +0800 Subject: [PATCH 03/15] use `static` instead of `const` for derive(Debug) --- .../rustc_builtin_macros/src/deriving/debug.rs | 18 +++++++++--------- tests/ui/derives/deriving-all-codegen.stdout | 4 ++-- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/compiler/rustc_builtin_macros/src/deriving/debug.rs b/compiler/rustc_builtin_macros/src/deriving/debug.rs index 59b50d8ba969d..c41a4e4fd6b8d 100644 --- a/compiler/rustc_builtin_macros/src/deriving/debug.rs +++ b/compiler/rustc_builtin_macros/src/deriving/debug.rs @@ -261,8 +261,8 @@ fn show_fieldless_enum( /// ```text /// impl ::core::fmt::Debug for A { /// fn fmt(&self, f: &mut ::core::fmt::Formatter) -> ::core::fmt::Result { -/// const __NAMES: &str = "ABBBCC"; -/// const __OFFSET: [usize; 4] =[0, 1, 4, 6]; +/// static __NAMES: &str = "ABBBCC"; +/// static __OFFSET: [usize; 4] =[0, 1, 4, 6]; /// let __d = ::core::intrinsics::discriminant_value(self) as usize; /// ::core::fmt::Formatter::debug_c_like_enums_write_str(f, __NAMES, &__OFFSET, __d) /// } @@ -314,8 +314,8 @@ fn show_fieldless_enum_concat_str( ), ); let names_str_body = cx.expr_str(span, Symbol::intern(&concatenated_names)); - let names_const_item = - cx.item_const(span, names_ident, str_ty, Some(names_str_body), ast::ConstItemKind::Body); + let names_static_item = + cx.item_static(span, names_ident, str_ty, ast::Mutability::Not, names_str_body); // Create the constant offset array let offset_ident = Ident::from_str_and_span("__OFFSET", span); @@ -332,12 +332,12 @@ fn show_fieldless_enum_concat_str( None, )), ); - let offset_const_item = cx.item_const( + let offset_static_item = cx.item_static( span, offset_ident, cx.ty(span, TyKind::Array(usize_ty, offset_array_len_expr)), - Some(starts_array_body), - ast::ConstItemKind::Body, + ast::Mutability::Not, + starts_array_body, ); // let __d = ::core::intrinsics::discriminant_value(self) as usize; @@ -372,8 +372,8 @@ fn show_fieldless_enum_concat_str( Some(( thin_vec![ - cx.stmt_item(span, names_const_item), - cx.stmt_item(span, offset_const_item), + cx.stmt_item(span, names_static_item), + cx.stmt_item(span, offset_static_item), discriminant_let_stmt, ], call_expr, diff --git a/tests/ui/derives/deriving-all-codegen.stdout b/tests/ui/derives/deriving-all-codegen.stdout index 6b72b08aefa5a..d0b387dbc8b25 100644 --- a/tests/ui/derives/deriving-all-codegen.stdout +++ b/tests/ui/derives/deriving-all-codegen.stdout @@ -1275,9 +1275,9 @@ enum Fieldless10 { impl ::core::fmt::Debug for Fieldless10 { #[inline] fn fmt(&self, f: &mut ::core::fmt::Formatter) -> ::core::fmt::Result { - const __NAMES: &str = + static __NAMES: &str = "AAAAABBBBCCDDDDDDDDEFFFFFFFFFFFFFGGGGGGHatsuneIIIIIIIJJJJJJJJJ"; - const __OFFSET: [usize; 11] = + static __OFFSET: [usize; 11] = [0usize, 5usize, 9usize, 11usize, 19usize, 20usize, 33usize, 39usize, 46usize, 53usize, 62usize]; let __d = ::core::intrinsics::discriminant_value(self) as usize; From c02de72260be982ab1bd9313ac2297e555c61358 Mon Sep 17 00:00:00 2001 From: rustbot <47979223+rustbot@users.noreply.github.com> Date: Mon, 10 Aug 2026 19:00:44 +0200 Subject: [PATCH 04/15] Update books --- src/doc/reference | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/doc/reference b/src/doc/reference index afdc77bab886d..82da570bafd9e 160000 --- a/src/doc/reference +++ b/src/doc/reference @@ -1 +1 @@ -Subproject commit afdc77bab886d4455c11247cdd32391bfab636ae +Subproject commit 82da570bafd9efed421bdbae0f8603ede8dd308b From 34b02c630adec84516ba517df7f542d71a2462b3 Mon Sep 17 00:00:00 2001 From: malezjaa Date: Tue, 11 Aug 2026 00:46:20 +0200 Subject: [PATCH 05/15] implement ::new_init --- library/core/src/cell/once.rs | 25 +++++++++++++++++++++++++ library/std/src/sync/once_lock.rs | 29 +++++++++++++++++++++++++++++ 2 files changed, 54 insertions(+) diff --git a/library/core/src/cell/once.rs b/library/core/src/cell/once.rs index 799e19cce67e6..01ba604e67949 100644 --- a/library/core/src/cell/once.rs +++ b/library/core/src/cell/once.rs @@ -47,6 +47,31 @@ impl OnceCell { OnceCell { inner: UnsafeCell::new(None) } } + /// Creates a new initialized cell. + /// + /// This is equivalent to `OnceCell::from(value)`, but can be used in + /// const contexts, unlike the `From` implementation. + /// + /// # Examples + /// + /// ``` + /// #![feature(once_cell_new_init)] + /// use std::cell::OnceCell; + /// + /// const CELL: OnceCell = OnceCell::new_init(1); + /// assert_eq!(CELL.get(), Some(&1)); + /// + /// let cell = OnceCell::new_init(String::from("kitty")); + /// assert_eq!(cell.set(String::from("puppy")), Err(String::from("puppy"))); + /// assert_eq!(cell.get(), Some(&"kitty".to_string())); + /// ``` + #[inline] + #[must_use] + #[unstable(feature = "once_cell_new_init", issue = "159859")] + pub const fn new_init(init_value: T) -> OnceCell { + OnceCell { inner: UnsafeCell::new(Some(init_value)) } + } + /// Gets the reference to the underlying value. /// /// Returns `None` if the cell is uninitialized. diff --git a/library/std/src/sync/once_lock.rs b/library/std/src/sync/once_lock.rs index de80164ed4f85..4b41fc4587829 100644 --- a/library/std/src/sync/once_lock.rs +++ b/library/std/src/sync/once_lock.rs @@ -145,6 +145,35 @@ impl OnceLock { } } + /// Creates a new initialized cell. + /// + /// This is equivalent to `OnceLock::from(value)`, but can be used in + /// const contexts, unlike the `From` implementation. + /// + /// # Examples + /// + /// ``` + /// #![feature(once_lock_new_init)] + /// use std::sync::OnceLock; + /// + /// static CELL: OnceLock = OnceLock::new_init(1); + /// + /// assert_eq!(CELL.get(), Some(&1)); + /// + /// // Already initialized, so this closure never runs. + /// assert_eq!(CELL.get_or_init(|| panic!("Kaboom!")), &1); + /// ``` + #[inline] + #[must_use] + #[unstable(feature = "once_lock_new_init", issue = "159860")] + pub const fn new_init(init_value: T) -> OnceLock { + OnceLock { + once: Once::new_complete(), + value: UnsafeCell::new(MaybeUninit::new(init_value)), + _marker: PhantomData, + } + } + /// Gets the reference to the underlying value. /// /// Returns `None` if the cell is uninitialized, or being initialized. From 0bcefa1ec78c6a25f9ef636a0f85f01e7184785a Mon Sep 17 00:00:00 2001 From: Frank Steffahn Date: Mon, 10 Aug 2026 20:30:38 +0200 Subject: [PATCH 06/15] Add back homu-ignore markers around the PR template Also clarify how the homu-ignore markers actually work. --- .github/pull_request_template.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index be37f27b4c25a..8abe1d818de1c 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,8 +1,8 @@ + + From 0f7dbc062cc2300be2b110e37301420d975ea630 Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Mon, 10 Aug 2026 17:00:21 +1000 Subject: [PATCH 07/15] Fix a `use` item `lib.rs` imports `DepNode`/`DepNodeIndex` even though it doesn't use them directly. And then `execution.rs` uses that one. Weird. This commit fixes things to be more normal. --- compiler/rustc_query_impl/src/execution.rs | 5 +++-- compiler/rustc_query_impl/src/lib.rs | 1 - 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/compiler/rustc_query_impl/src/execution.rs b/compiler/rustc_query_impl/src/execution.rs index 190bfaec8887a..c8562fe534a8f 100644 --- a/compiler/rustc_query_impl/src/execution.rs +++ b/compiler/rustc_query_impl/src/execution.rs @@ -6,7 +6,9 @@ use rustc_data_structures::hash_table::{Entry, HashTable}; use rustc_data_structures::sync::{DynSend, DynSync}; use rustc_data_structures::{defer, outline, sharded, sync}; use rustc_errors::FatalError; -use rustc_middle::dep_graph::{DepGraphData, DepNodeKey, SerializedDepNodeIndex}; +use rustc_middle::dep_graph::{ + DepGraphData, DepNode, DepNodeIndex, DepNodeKey, SerializedDepNodeIndex, +}; use rustc_middle::query::{ ActiveKeyStatus, Cycle, QueryCache, QueryJob, QueryJobId, QueryKey, QueryLatch, QueryMode, QueryState, QueryVTable, @@ -16,7 +18,6 @@ use rustc_middle::verify_ich::incremental_verify_ich; use rustc_span::{DUMMY_SP, Span}; use tracing::debug; -use crate::dep_graph::{DepNode, DepNodeIndex}; use crate::handle_cycle_error; use crate::job::{QueryJobInfo, QueryJobMap, create_cycle_error, find_cycle_in_stack}; use crate::plumbing::{current_query_job, next_job_id, start_query}; diff --git a/compiler/rustc_query_impl/src/lib.rs b/compiler/rustc_query_impl/src/lib.rs index 0a8b25c2fa878..e9a13bea1ddd8 100644 --- a/compiler/rustc_query_impl/src/lib.rs +++ b/compiler/rustc_query_impl/src/lib.rs @@ -9,7 +9,6 @@ // tidy-alphabetical-end use rustc_data_structures::sync::{AtomicU64, Lock}; -use rustc_middle::dep_graph; use rustc_middle::queries::{ExternProviders, Providers}; use rustc_middle::query::on_disk_cache::OnDiskCache; use rustc_middle::query::{QueryCache, QuerySystem, QueryVTable}; From 9e2147f559288f9199aa4cda00cf6c65c12f86c0 Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Mon, 10 Aug 2026 16:08:15 +1000 Subject: [PATCH 08/15] Move `create_cycle_error` to `mod handle_cycle_error` It's a more sensible home than `mod job`. --- compiler/rustc_query_impl/src/execution.rs | 4 +- .../src/handle_cycle_error.rs | 92 ++++++++++++++++++- compiler/rustc_query_impl/src/job.rs | 91 +----------------- 3 files changed, 92 insertions(+), 95 deletions(-) diff --git a/compiler/rustc_query_impl/src/execution.rs b/compiler/rustc_query_impl/src/execution.rs index c8562fe534a8f..c5a34838cc957 100644 --- a/compiler/rustc_query_impl/src/execution.rs +++ b/compiler/rustc_query_impl/src/execution.rs @@ -19,7 +19,7 @@ use rustc_span::{DUMMY_SP, Span}; use tracing::debug; use crate::handle_cycle_error; -use crate::job::{QueryJobInfo, QueryJobMap, create_cycle_error, find_cycle_in_stack}; +use crate::job::{QueryJobInfo, QueryJobMap, find_cycle_in_stack}; use crate::plumbing::{current_query_job, next_job_id, start_query}; use crate::query_impl::for_each_query_vtable; @@ -136,7 +136,7 @@ fn handle_cycle<'tcx, C: QueryCache>( } let _guard = defer(|| *tcx.query_system.cycle_handler_nesting.lock() -= 1); - let error = create_cycle_error(tcx, &cycle, nested); + let error = handle_cycle_error::create_cycle_error(tcx, &cycle, nested); if nested { // Avoid custom handlers and only use the robust `create_cycle_error` for nested cycle errors diff --git a/compiler/rustc_query_impl/src/handle_cycle_error.rs b/compiler/rustc_query_impl/src/handle_cycle_error.rs index 7e7ed3ab3b525..6bc7bfc59b08b 100644 --- a/compiler/rustc_query_impl/src/handle_cycle_error.rs +++ b/compiler/rustc_query_impl/src/handle_cycle_error.rs @@ -13,9 +13,7 @@ use rustc_middle::queries::TaggedQueryKey; use rustc_middle::query::Cycle; use rustc_middle::ty::{self, Ty, TyCtxt}; use rustc_span::def_id::{DefId, LocalDefId}; -use rustc_span::{ErrorGuaranteed, Span}; - -use crate::job::create_cycle_error; +use rustc_span::{DUMMY_SP, ErrorGuaranteed, Span}; // Default cycle handler used for all queries that don't use the `handle_cycle_error` query // modifier. @@ -343,3 +341,91 @@ fn find_item_ty_spans( _ => {} } } + +#[inline(never)] +#[cold] +pub(crate) fn create_cycle_error<'tcx>( + tcx: TyCtxt<'tcx>, + Cycle { usage, frames }: &Cycle<'tcx>, + nested: bool, +) -> Diag<'tcx> { + assert!(!frames.is_empty()); + + let span = frames[0].tagged_key.catch_default_span(tcx, frames[1 % frames.len()].span); + + let mut cycle_stack = Vec::new(); + + use crate::diagnostics::StackCount; + let stack_bottom = frames[0].tagged_key.catch_description(tcx); + let stack_count = if frames.len() == 1 { + StackCount::Single { stack_bottom: stack_bottom.clone() } + } else { + StackCount::Multiple { stack_bottom: stack_bottom.clone() } + }; + + let mut prev = span; + for i in 1..frames.len() { + let frame = &frames[i]; + let span = frame.tagged_key.catch_default_span(tcx, frames[(i + 1) % frames.len()].span); + cycle_stack.push(crate::diagnostics::CycleStack { + span: if span == prev { DUMMY_SP } else { span }, + desc: frame.tagged_key.catch_description(tcx), + }); + prev = span; + } + + let cycle_usage = usage.as_ref().map(|usage| { + let cycle_span = usage.tagged_key.catch_default_span(tcx, usage.span); + crate::diagnostics::CycleUsage { + span: if cycle_span != span { cycle_span } else { DUMMY_SP }, + usage: usage.tagged_key.catch_description(tcx), + } + }); + + let is_all_def_kind = |def_kind| { + // Trivial type alias and trait alias cycles consists of `type_of` and + // `explicit_implied_clauses_of` queries, so we just check just these here. + frames.iter().all(|frame| match frame.tagged_key { + TaggedQueryKey::type_of(def_id) + | TaggedQueryKey::explicit_implied_clauses_of(def_id) + if tcx.def_kind(def_id) == def_kind => + { + true + } + _ => false, + }) + }; + + let alias = if !nested { + if is_all_def_kind(DefKind::TyAlias) { + Some(crate::diagnostics::Alias::Ty) + } else if is_all_def_kind(DefKind::TraitAlias) { + Some(crate::diagnostics::Alias::Trait) + } else { + None + } + } else { + None + }; + + if nested { + tcx.sess.dcx().create_err(crate::diagnostics::NestedCycle { + span, + cycle_stack, + stack_bottom: crate::diagnostics::NestedCycleBottom { stack_bottom }, + cycle_usage, + stack_count, + note_span: (), + }) + } else { + tcx.sess.dcx().create_err(crate::diagnostics::Cycle { + span, + cycle_stack, + stack_bottom, + alias, + cycle_usage, + stack_count, + note_span: (), + }) + } +} diff --git a/compiler/rustc_query_impl/src/job.rs b/compiler/rustc_query_impl/src/job.rs index 1b604409f38a6..57601e1781d5f 100644 --- a/compiler/rustc_query_impl/src/job.rs +++ b/compiler/rustc_query_impl/src/job.rs @@ -4,8 +4,7 @@ use std::sync::Arc; use std::{iter, mem}; use rustc_data_structures::fx::{FxHashMap, FxHashSet}; -use rustc_errors::{Diag, DiagCtxtHandle}; -use rustc_hir::def::DefKind; +use rustc_errors::DiagCtxtHandle; use rustc_middle::queries::TaggedQueryKey; use rustc_middle::query::{Cycle, QueryJob, QueryJobId, QueryLatch, QueryStackFrame, QueryWaiter}; use rustc_middle::ty::TyCtxt; @@ -407,91 +406,3 @@ pub fn print_query_stack<'tcx>( } count_total } - -#[inline(never)] -#[cold] -pub(crate) fn create_cycle_error<'tcx>( - tcx: TyCtxt<'tcx>, - Cycle { usage, frames }: &Cycle<'tcx>, - nested: bool, -) -> Diag<'tcx> { - assert!(!frames.is_empty()); - - let span = frames[0].tagged_key.catch_default_span(tcx, frames[1 % frames.len()].span); - - let mut cycle_stack = Vec::new(); - - use crate::diagnostics::StackCount; - let stack_bottom = frames[0].tagged_key.catch_description(tcx); - let stack_count = if frames.len() == 1 { - StackCount::Single { stack_bottom: stack_bottom.clone() } - } else { - StackCount::Multiple { stack_bottom: stack_bottom.clone() } - }; - - let mut prev = span; - for i in 1..frames.len() { - let frame = &frames[i]; - let span = frame.tagged_key.catch_default_span(tcx, frames[(i + 1) % frames.len()].span); - cycle_stack.push(crate::diagnostics::CycleStack { - span: if span == prev { DUMMY_SP } else { span }, - desc: frame.tagged_key.catch_description(tcx), - }); - prev = span; - } - - let cycle_usage = usage.as_ref().map(|usage| { - let cycle_span = usage.tagged_key.catch_default_span(tcx, usage.span); - crate::diagnostics::CycleUsage { - span: if cycle_span != span { cycle_span } else { DUMMY_SP }, - usage: usage.tagged_key.catch_description(tcx), - } - }); - - let is_all_def_kind = |def_kind| { - // Trivial type alias and trait alias cycles consists of `type_of` and - // `explicit_implied_clauses_of` queries, so we just check just these here. - frames.iter().all(|frame| match frame.tagged_key { - TaggedQueryKey::type_of(def_id) - | TaggedQueryKey::explicit_implied_clauses_of(def_id) - if tcx.def_kind(def_id) == def_kind => - { - true - } - _ => false, - }) - }; - - let alias = if !nested { - if is_all_def_kind(DefKind::TyAlias) { - Some(crate::diagnostics::Alias::Ty) - } else if is_all_def_kind(DefKind::TraitAlias) { - Some(crate::diagnostics::Alias::Trait) - } else { - None - } - } else { - None - }; - - if nested { - tcx.sess.dcx().create_err(crate::diagnostics::NestedCycle { - span, - cycle_stack, - stack_bottom: crate::diagnostics::NestedCycleBottom { stack_bottom }, - cycle_usage, - stack_count, - note_span: (), - }) - } else { - tcx.sess.dcx().create_err(crate::diagnostics::Cycle { - span, - cycle_stack, - stack_bottom, - alias, - cycle_usage, - stack_count, - note_span: (), - }) - } -} From 9e37f3950e13684401274fca50c5a581d806d6dd Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Mon, 10 Aug 2026 16:11:35 +1000 Subject: [PATCH 09/15] Rename `query_impl.rs` as `query_vtables.rs` The current name is very generic and the file is mostly about query vtables. --- compiler/rustc_query_impl/src/dep_kind_vtables.rs | 2 +- compiler/rustc_query_impl/src/execution.rs | 2 +- compiler/rustc_query_impl/src/lib.rs | 6 +++--- compiler/rustc_query_impl/src/plumbing.rs | 2 +- compiler/rustc_query_impl/src/profiling_support.rs | 2 +- .../src/{query_impl.rs => query_vtables.rs} | 8 ++++---- 6 files changed, 11 insertions(+), 11 deletions(-) rename compiler/rustc_query_impl/src/{query_impl.rs => query_vtables.rs} (97%) diff --git a/compiler/rustc_query_impl/src/dep_kind_vtables.rs b/compiler/rustc_query_impl/src/dep_kind_vtables.rs index 5adcf6c7bb576..a71f919c282e8 100644 --- a/compiler/rustc_query_impl/src/dep_kind_vtables.rs +++ b/compiler/rustc_query_impl/src/dep_kind_vtables.rs @@ -166,7 +166,7 @@ macro_rules! define_dep_kind_vtables { let q_vtables: [DepKindVTable<'tcx>; _] = [ $( $crate::dep_kind_vtables::make_dep_kind_vtable_for_query::< - $crate::query_impl::$name::VTableGetter, + $crate::query_vtables::$name::VTableGetter, >( $cache_on_disk, $eval_always, diff --git a/compiler/rustc_query_impl/src/execution.rs b/compiler/rustc_query_impl/src/execution.rs index c5a34838cc957..691c5c66aa023 100644 --- a/compiler/rustc_query_impl/src/execution.rs +++ b/compiler/rustc_query_impl/src/execution.rs @@ -21,7 +21,7 @@ use tracing::debug; use crate::handle_cycle_error; use crate::job::{QueryJobInfo, QueryJobMap, find_cycle_in_stack}; use crate::plumbing::{current_query_job, next_job_id, start_query}; -use crate::query_impl::for_each_query_vtable; +use crate::query_vtables::for_each_query_vtable; #[inline] fn equivalent_key(k: K) -> impl Fn(&(K, V)) -> bool { diff --git a/compiler/rustc_query_impl/src/lib.rs b/compiler/rustc_query_impl/src/lib.rs index e9a13bea1ddd8..12094ca567f52 100644 --- a/compiler/rustc_query_impl/src/lib.rs +++ b/compiler/rustc_query_impl/src/lib.rs @@ -25,7 +25,7 @@ mod handle_cycle_error; mod job; mod plumbing; mod profiling_support; -mod query_impl; +mod query_vtables; /// Trait that knows how to look up the [`QueryVTable`] for a particular query. /// @@ -34,7 +34,7 @@ mod query_impl; /// expansion. /// /// There is one macro-generated implementation of this trait for each query, -/// on the type `rustc_query_impl::query_impl::$name::VTableGetter`. +/// on the type `rustc_query_impl::query_vtables::$name::VTableGetter`. trait GetQueryVTable<'tcx> { type Cache: QueryCache + 'tcx; @@ -49,7 +49,7 @@ pub fn query_system<'tcx>( ) -> QuerySystem<'tcx> { QuerySystem { arenas: Default::default(), - query_vtables: query_impl::make_query_vtables(incremental), + query_vtables: query_vtables::make_query_vtables(incremental), side_effects: Default::default(), on_disk_cache, local_providers, diff --git a/compiler/rustc_query_impl/src/plumbing.rs b/compiler/rustc_query_impl/src/plumbing.rs index e358be327f240..bc085916afee3 100644 --- a/compiler/rustc_query_impl/src/plumbing.rs +++ b/compiler/rustc_query_impl/src/plumbing.rs @@ -18,7 +18,7 @@ use rustc_span::def_id::LOCAL_CRATE; use crate::diagnostics::{QueryOverflow, QueryOverflowNote}; use crate::execution::{all_inactive, should_verify_loaded_value}; use crate::job::find_dep_kind_root; -use crate::query_impl::for_each_query_vtable; +use crate::query_vtables::for_each_query_vtable; use crate::{CollectActiveJobsKind, collect_active_query_jobs}; fn depth_limit_error<'tcx>(tcx: TyCtxt<'tcx>, job: QueryJobId) { diff --git a/compiler/rustc_query_impl/src/profiling_support.rs b/compiler/rustc_query_impl/src/profiling_support.rs index 980e2b1305245..53734b84a9a10 100644 --- a/compiler/rustc_query_impl/src/profiling_support.rs +++ b/compiler/rustc_query_impl/src/profiling_support.rs @@ -9,7 +9,7 @@ use rustc_hir::definitions::DefPathData; use rustc_middle::query::{QueryCache, QueryVTable}; use rustc_middle::ty::TyCtxt; -use crate::query_impl::for_each_query_vtable; +use crate::query_vtables::for_each_query_vtable; pub(crate) struct QueryKeyStringCache { def_id_cache: FxHashMap, diff --git a/compiler/rustc_query_impl/src/query_impl.rs b/compiler/rustc_query_impl/src/query_vtables.rs similarity index 97% rename from compiler/rustc_query_impl/src/query_impl.rs rename to compiler/rustc_query_impl/src/query_vtables.rs index 3720d9fd80547..e7c8ff9bd537b 100644 --- a/compiler/rustc_query_impl/src/query_impl.rs +++ b/compiler/rustc_query_impl/src/query_vtables.rs @@ -33,7 +33,7 @@ macro_rules! define_queries { // Non-queries are unused here. non_queries { $($_:tt)* } ) => { - // This macro expects to be expanded into `crate::query_impl`, which is this file. + // This macro expects to be expanded into `crate::query_vtables`, which is this file. $( pub(crate) mod $name { use super::*; @@ -181,9 +181,9 @@ macro_rules! define_queries { }, create_tagged_key: TaggedQueryKey::$name, execute_query_fn: if incremental { - crate::query_impl::$name::execute_query_incr::__rust_end_short_backtrace + crate::query_vtables::$name::execute_query_incr::__rust_end_short_backtrace } else { - crate::query_impl::$name::execute_query_non_incr::__rust_end_short_backtrace + crate::query_vtables::$name::execute_query_non_incr::__rust_end_short_backtrace }, } } @@ -207,7 +207,7 @@ macro_rules! define_queries { { rustc_middle::queries::QueryVTables { $( - $name: crate::query_impl::$name::make_query_vtable(incremental), + $name: crate::query_vtables::$name::make_query_vtable(incremental), )* } } From c0b0065f0644e2e3d49289d7fb3741ce9ae1ec5d Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Mon, 10 Aug 2026 16:17:54 +1000 Subject: [PATCH 10/15] Move four functions from `plumbing.rs` to `execution.rs` These four functions are all only used within `execution.rs`, so they no longer need `pub(crate)`. This is the first step toward eliminating `plumbing.rs`. --- compiler/rustc_query_impl/src/execution.rs | 59 +++++++++++++++++++- compiler/rustc_query_impl/src/plumbing.rs | 64 +--------------------- 2 files changed, 57 insertions(+), 66 deletions(-) diff --git a/compiler/rustc_query_impl/src/execution.rs b/compiler/rustc_query_impl/src/execution.rs index 691c5c66aa023..aceb8dfca94e5 100644 --- a/compiler/rustc_query_impl/src/execution.rs +++ b/compiler/rustc_query_impl/src/execution.rs @@ -1,10 +1,11 @@ use std::hash::Hash; use std::mem::ManuallyDrop; +use std::num::NonZero; use rustc_data_structures::fingerprint::{Fingerprint, PackedFingerprint}; use rustc_data_structures::hash_table::{Entry, HashTable}; use rustc_data_structures::sync::{DynSend, DynSync}; -use rustc_data_structures::{defer, outline, sharded, sync}; +use rustc_data_structures::{Limit, defer, outline, sharded, sync}; use rustc_errors::FatalError; use rustc_middle::dep_graph::{ DepGraphData, DepNode, DepNodeIndex, DepNodeKey, SerializedDepNodeIndex, @@ -14,13 +15,15 @@ use rustc_middle::query::{ QueryState, QueryVTable, }; use rustc_middle::ty::TyCtxt; +use rustc_middle::ty::tls::{self, ImplicitCtxt}; use rustc_middle::verify_ich::incremental_verify_ich; +use rustc_span::def_id::LOCAL_CRATE; use rustc_span::{DUMMY_SP, Span}; use tracing::debug; +use crate::diagnostics::{QueryOverflow, QueryOverflowNote}; use crate::handle_cycle_error; -use crate::job::{QueryJobInfo, QueryJobMap, find_cycle_in_stack}; -use crate::plumbing::{current_query_job, next_job_id, start_query}; +use crate::job::{QueryJobInfo, QueryJobMap, find_cycle_in_stack, find_dep_kind_root}; use crate::query_vtables::for_each_query_vtable; #[inline] @@ -282,6 +285,19 @@ fn wait_for_query<'tcx, C: QueryCache>( } } +#[inline] +fn next_job_id<'tcx>(tcx: TyCtxt<'tcx>) -> QueryJobId { + QueryJobId( + NonZero::new(tcx.query_system.jobs.fetch_add(1, std::sync::atomic::Ordering::Relaxed)) + .unwrap(), + ) +} + +#[inline] +fn current_query_job() -> Option { + tls::with_context(|icx| icx.query) +} + /// Shared main part of both [`execute_query_incr_inner`] and [`execute_query_non_incr_inner`]. #[inline(never)] fn try_execute_query<'tcx, C: QueryCache, const INCR: bool>( @@ -408,6 +424,43 @@ fn check_feedable_consistency<'tcx, C: QueryCache>( } } +fn depth_limit_error<'tcx>(tcx: TyCtxt<'tcx>, job: QueryJobId) { + let job_map = collect_active_query_jobs(tcx, CollectActiveJobsKind::Full); + let (span, desc, depth) = find_dep_kind_root(tcx, job, job_map); + + let suggested_limit = match tcx.recursion_limit() { + Limit(0) => Limit(2), + limit => limit * 2, + }; + + tcx.dcx().emit_fatal(QueryOverflow { + span, + note: QueryOverflowNote { desc, depth }, + suggested_limit, + crate_name: tcx.crate_name(LOCAL_CRATE), + }); +} + +/// Executes a job by changing the `ImplicitCtxt` to point to the new query job while it executes. +#[inline(always)] +fn start_query(job_id: QueryJobId, depth_limit: bool, compute: impl FnOnce() -> R) -> R { + tls::with_context(move |icx| { + if depth_limit && !icx.tcx.recursion_limit().value_within_limit(icx.query_depth) { + depth_limit_error(icx.tcx, job_id); + } + + // Update the `ImplicitCtxt` to point to our new query job. + let icx = ImplicitCtxt { + query: Some(job_id), + query_depth: icx.query_depth + if depth_limit { 1 } else { 0 }, + ..*icx + }; + + // Use the `ImplicitCtxt` while we execute the query. + tls::enter_context(&icx, compute) + }) +} + // Fast path for when incr. comp. is off. #[inline(always)] fn execute_job_non_incr<'tcx, C: QueryCache>( diff --git a/compiler/rustc_query_impl/src/plumbing.rs b/compiler/rustc_query_impl/src/plumbing.rs index bc085916afee3..83b08fcd23cc6 100644 --- a/compiler/rustc_query_impl/src/plumbing.rs +++ b/compiler/rustc_query_impl/src/plumbing.rs @@ -1,6 +1,3 @@ -use std::num::NonZero; - -use rustc_data_structures::Limit; use rustc_data_structures::unord::UnordMap; use rustc_middle::bug; #[expect(unused_imports, reason = "used by doc comments")] @@ -8,72 +5,13 @@ use rustc_middle::dep_graph::DepKindVTable; use rustc_middle::dep_graph::{DepNode, DepNodeIndex, DepNodeKey, SerializedDepNodeIndex}; use rustc_middle::query::erase::{Erasable, Erased}; use rustc_middle::query::on_disk_cache::{CacheDecoder, CacheEncoder}; -use rustc_middle::query::{QueryCache, QueryJobId, QueryVTable, erase}; +use rustc_middle::query::{QueryCache, QueryVTable, erase}; use rustc_middle::ty::TyCtxt; -use rustc_middle::ty::tls::{self, ImplicitCtxt}; use rustc_middle::verify_ich::incremental_verify_ich; use rustc_serialize::{Decodable, Encodable}; -use rustc_span::def_id::LOCAL_CRATE; -use crate::diagnostics::{QueryOverflow, QueryOverflowNote}; use crate::execution::{all_inactive, should_verify_loaded_value}; -use crate::job::find_dep_kind_root; use crate::query_vtables::for_each_query_vtable; -use crate::{CollectActiveJobsKind, collect_active_query_jobs}; - -fn depth_limit_error<'tcx>(tcx: TyCtxt<'tcx>, job: QueryJobId) { - let job_map = collect_active_query_jobs(tcx, CollectActiveJobsKind::Full); - let (span, desc, depth) = find_dep_kind_root(tcx, job, job_map); - - let suggested_limit = match tcx.recursion_limit() { - Limit(0) => Limit(2), - limit => limit * 2, - }; - - tcx.dcx().emit_fatal(QueryOverflow { - span, - note: QueryOverflowNote { desc, depth }, - suggested_limit, - crate_name: tcx.crate_name(LOCAL_CRATE), - }); -} - -#[inline] -pub(crate) fn next_job_id<'tcx>(tcx: TyCtxt<'tcx>) -> QueryJobId { - QueryJobId( - NonZero::new(tcx.query_system.jobs.fetch_add(1, std::sync::atomic::Ordering::Relaxed)) - .unwrap(), - ) -} - -#[inline] -pub(crate) fn current_query_job() -> Option { - tls::with_context(|icx| icx.query) -} - -/// Executes a job by changing the `ImplicitCtxt` to point to the new query job while it executes. -#[inline(always)] -pub(crate) fn start_query( - job_id: QueryJobId, - depth_limit: bool, - compute: impl FnOnce() -> R, -) -> R { - tls::with_context(move |icx| { - if depth_limit && !icx.tcx.recursion_limit().value_within_limit(icx.query_depth) { - depth_limit_error(icx.tcx, job_id); - } - - // Update the `ImplicitCtxt` to point to our new query job. - let icx = ImplicitCtxt { - query: Some(job_id), - query_depth: icx.query_depth + if depth_limit { 1 } else { 0 }, - ..*icx - }; - - // Use the `ImplicitCtxt` while we execute the query. - tls::enter_context(&icx, compute) - }) -} pub(crate) fn encode_query_values<'tcx>(tcx: TyCtxt<'tcx>, encoder: &mut CacheEncoder<'_, 'tcx>) { for_each_query_vtable!(CACHE_ON_DISK, tcx, |query| { From 04df355d86445d45ae8a92fc6c07523d6d22b60d Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Mon, 10 Aug 2026 16:40:26 +1000 Subject: [PATCH 11/15] Rename `plumbing.rs` as `incremental.rs` Because it now contains only things related to incremental. Also move two functions relating to incremental from `execution.rs` to `incremental.rs`. --- .../rustc_query_impl/src/dep_kind_vtables.rs | 2 +- compiler/rustc_query_impl/src/execution.rs | 29 +-------------- .../src/{plumbing.rs => incremental.rs} | 35 +++++++++++++++++-- compiler/rustc_query_impl/src/lib.rs | 6 ++-- .../rustc_query_impl/src/query_vtables.rs | 2 +- 5 files changed, 38 insertions(+), 36 deletions(-) rename compiler/rustc_query_impl/src/{plumbing.rs => incremental.rs} (76%) diff --git a/compiler/rustc_query_impl/src/dep_kind_vtables.rs b/compiler/rustc_query_impl/src/dep_kind_vtables.rs index a71f919c282e8..5d5aca98db0a0 100644 --- a/compiler/rustc_query_impl/src/dep_kind_vtables.rs +++ b/compiler/rustc_query_impl/src/dep_kind_vtables.rs @@ -4,7 +4,7 @@ use rustc_middle::dep_graph::{DepKindVTable, DepNodeKey, KeyFingerprintStyle}; use rustc_middle::query::QueryCache; use crate::GetQueryVTable; -use crate::plumbing::promote_from_disk_inner; +use crate::incremental::promote_from_disk_inner; /// [`DepKindVTable`] constructors for special dep kinds that aren't queries. #[expect(non_snake_case, reason = "use non-snake case to avoid collision with query names")] diff --git a/compiler/rustc_query_impl/src/execution.rs b/compiler/rustc_query_impl/src/execution.rs index aceb8dfca94e5..03e5d588374ca 100644 --- a/compiler/rustc_query_impl/src/execution.rs +++ b/compiler/rustc_query_impl/src/execution.rs @@ -2,7 +2,6 @@ use std::hash::Hash; use std::mem::ManuallyDrop; use std::num::NonZero; -use rustc_data_structures::fingerprint::{Fingerprint, PackedFingerprint}; use rustc_data_structures::hash_table::{Entry, HashTable}; use rustc_data_structures::sync::{DynSend, DynSync}; use rustc_data_structures::{Limit, defer, outline, sharded, sync}; @@ -23,6 +22,7 @@ use tracing::debug; use crate::diagnostics::{QueryOverflow, QueryOverflowNote}; use crate::handle_cycle_error; +use crate::incremental::should_verify_loaded_value; use crate::job::{QueryJobInfo, QueryJobMap, find_cycle_in_stack, find_dep_kind_root}; use crate::query_vtables::for_each_query_vtable; @@ -31,10 +31,6 @@ fn equivalent_key(k: K) -> impl Fn(&(K, V)) -> bool { move |x| x.0 == k } -pub(crate) fn all_inactive<'tcx, K>(state: &QueryState<'tcx, K>) -> bool { - state.active.lock_shards().all(|shard| shard.is_empty()) -} - #[derive(Clone, Copy)] pub enum CollectActiveJobsKind { /// We need the full query job map, and we are willing to wait to obtain the query state @@ -538,29 +534,6 @@ fn execute_job_incr<'tcx, C: QueryCache>( (result, dep_node_index) } -/// Whether a value loaded from the on-disk cache should have its fingerprint -/// verified with `incremental_verify_ich`. If `-Zincremental-verify-ich` is -/// specified, re-hash results from the cache and make sure that they have the -/// expected fingerprint. -/// -/// If not, we still verify a subset: re-hashing is too expensive to do for -/// every value. The subset rotates with the session count, covering the whole -/// cache every 32 sessions, and is deterministic so that a verification -/// failure reproduces on retry. -/// -/// `to_smaller_hash` mixes both fingerprint halves because neither half is -/// evenly distributed on its own (`DefPathHash` keys share the -/// `StableCrateId`, `HirId` keys contain a sequential id). -pub(crate) fn should_verify_loaded_value( - tcx: TyCtxt<'_>, - dep_graph_data: &DepGraphData, - key_fingerprint: PackedFingerprint, -) -> bool { - let hash = Fingerprint::from(key_fingerprint).to_smaller_hash().as_u64(); - hash % 32 == dep_graph_data.session_count() % 32 - || tcx.sess.opts.unstable_opts.incremental_verify_ich -} - /// Given that the dep node for this query+key is green, obtain a value for it /// by loading one from disk if possible, or by invoking its query provider if /// necessary. diff --git a/compiler/rustc_query_impl/src/plumbing.rs b/compiler/rustc_query_impl/src/incremental.rs similarity index 76% rename from compiler/rustc_query_impl/src/plumbing.rs rename to compiler/rustc_query_impl/src/incremental.rs index 83b08fcd23cc6..341c9f5e5068d 100644 --- a/compiler/rustc_query_impl/src/plumbing.rs +++ b/compiler/rustc_query_impl/src/incremental.rs @@ -1,18 +1,24 @@ +use rustc_data_structures::fingerprint::{Fingerprint, PackedFingerprint}; use rustc_data_structures::unord::UnordMap; use rustc_middle::bug; #[expect(unused_imports, reason = "used by doc comments")] use rustc_middle::dep_graph::DepKindVTable; -use rustc_middle::dep_graph::{DepNode, DepNodeIndex, DepNodeKey, SerializedDepNodeIndex}; +use rustc_middle::dep_graph::{ + DepGraphData, DepNode, DepNodeIndex, DepNodeKey, SerializedDepNodeIndex, +}; use rustc_middle::query::erase::{Erasable, Erased}; use rustc_middle::query::on_disk_cache::{CacheDecoder, CacheEncoder}; -use rustc_middle::query::{QueryCache, QueryVTable, erase}; +use rustc_middle::query::{QueryCache, QueryState, QueryVTable, erase}; use rustc_middle::ty::TyCtxt; use rustc_middle::verify_ich::incremental_verify_ich; use rustc_serialize::{Decodable, Encodable}; -use crate::execution::{all_inactive, should_verify_loaded_value}; use crate::query_vtables::for_each_query_vtable; +fn all_inactive<'tcx, K>(state: &QueryState<'tcx, K>) -> bool { + state.active.lock_shards().all(|shard| shard.is_empty()) +} + pub(crate) fn encode_query_values<'tcx>(tcx: TyCtxt<'tcx>, encoder: &mut CacheEncoder<'_, 'tcx>) { for_each_query_vtable!(CACHE_ON_DISK, tcx, |query| { encode_query_values_inner(tcx, query, encoder) @@ -73,6 +79,29 @@ fn verify_query_key_hashes_inner<'tcx, C: QueryCache>( }); } +/// Whether a value loaded from the on-disk cache should have its fingerprint +/// verified with `incremental_verify_ich`. If `-Zincremental-verify-ich` is +/// specified, re-hash results from the cache and make sure that they have the +/// expected fingerprint. +/// +/// If not, we still verify a subset: re-hashing is too expensive to do for +/// every value. The subset rotates with the session count, covering the whole +/// cache every 32 sessions, and is deterministic so that a verification +/// failure reproduces on retry. +/// +/// `to_smaller_hash` mixes both fingerprint halves because neither half is +/// evenly distributed on its own (`DefPathHash` keys share the +/// `StableCrateId`, `HirId` keys contain a sequential id). +pub(crate) fn should_verify_loaded_value( + tcx: TyCtxt<'_>, + dep_graph_data: &DepGraphData, + key_fingerprint: PackedFingerprint, +) -> bool { + let hash = Fingerprint::from(key_fingerprint).to_smaller_hash().as_u64(); + hash % 32 == dep_graph_data.session_count() % 32 + || tcx.sess.opts.unstable_opts.incremental_verify_ich +} + /// Inner implementation of [`DepKindVTable::promote_from_disk_fn`] for queries. pub(crate) fn promote_from_disk_inner<'tcx, C: QueryCache>( tcx: TyCtxt<'tcx>, diff --git a/compiler/rustc_query_impl/src/lib.rs b/compiler/rustc_query_impl/src/lib.rs index 12094ca567f52..621b56cd242ed 100644 --- a/compiler/rustc_query_impl/src/lib.rs +++ b/compiler/rustc_query_impl/src/lib.rs @@ -22,8 +22,8 @@ mod dep_kind_vtables; mod diagnostics; mod execution; mod handle_cycle_error; +mod incremental; mod job; -mod plumbing; mod profiling_support; mod query_vtables; @@ -62,6 +62,6 @@ pub fn query_system<'tcx>( pub fn provide(providers: &mut rustc_middle::util::Providers) { providers.hooks.alloc_self_profile_query_strings = profiling_support::alloc_self_profile_query_strings; - providers.hooks.verify_query_key_hashes = plumbing::verify_query_key_hashes; - providers.hooks.encode_query_values = plumbing::encode_query_values; + providers.hooks.verify_query_key_hashes = incremental::verify_query_key_hashes; + providers.hooks.encode_query_values = incremental::encode_query_values; } diff --git a/compiler/rustc_query_impl/src/query_vtables.rs b/compiler/rustc_query_impl/src/query_vtables.rs index e7c8ff9bd537b..8976f5d9f499a 100644 --- a/compiler/rustc_query_impl/src/query_vtables.rs +++ b/compiler/rustc_query_impl/src/query_vtables.rs @@ -149,7 +149,7 @@ macro_rules! define_queries { use rustc_middle::queries::$name::{ProvidedValue, provided_to_erased}; let loaded_value: ProvidedValue<'tcx> = - $crate::plumbing::try_load_from_disk(tcx, prev_index)?; + $crate::incremental::try_load_from_disk(tcx, prev_index)?; // Arena-alloc the value if appropriate, and erase it. Some(provided_to_erased(tcx, loaded_value)) From b99843c7a25954bdd0ac784b8579edf9e37675b1 Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Mon, 10 Aug 2026 18:07:35 +1000 Subject: [PATCH 12/15] Move `GetQueryVTable` into `query_vtables.rs` It's a more sensible home than `lib.rs`. --- .../rustc_query_impl/src/dep_kind_vtables.rs | 2 +- compiler/rustc_query_impl/src/lib.rs | 17 +---------------- compiler/rustc_query_impl/src/query_vtables.rs | 16 ++++++++++++++-- 3 files changed, 16 insertions(+), 19 deletions(-) diff --git a/compiler/rustc_query_impl/src/dep_kind_vtables.rs b/compiler/rustc_query_impl/src/dep_kind_vtables.rs index 5d5aca98db0a0..495524c3f142a 100644 --- a/compiler/rustc_query_impl/src/dep_kind_vtables.rs +++ b/compiler/rustc_query_impl/src/dep_kind_vtables.rs @@ -3,8 +3,8 @@ use rustc_middle::bug; use rustc_middle::dep_graph::{DepKindVTable, DepNodeKey, KeyFingerprintStyle}; use rustc_middle::query::QueryCache; -use crate::GetQueryVTable; use crate::incremental::promote_from_disk_inner; +use crate::query_vtables::GetQueryVTable; /// [`DepKindVTable`] constructors for special dep kinds that aren't queries. #[expect(non_snake_case, reason = "use non-snake case to avoid collision with query names")] diff --git a/compiler/rustc_query_impl/src/lib.rs b/compiler/rustc_query_impl/src/lib.rs index 621b56cd242ed..256def36bc4a0 100644 --- a/compiler/rustc_query_impl/src/lib.rs +++ b/compiler/rustc_query_impl/src/lib.rs @@ -10,9 +10,8 @@ use rustc_data_structures::sync::{AtomicU64, Lock}; use rustc_middle::queries::{ExternProviders, Providers}; +use rustc_middle::query::QuerySystem; use rustc_middle::query::on_disk_cache::OnDiskCache; -use rustc_middle::query::{QueryCache, QuerySystem, QueryVTable}; -use rustc_middle::ty::TyCtxt; pub use crate::dep_kind_vtables::make_dep_kind_vtables; pub use crate::execution::{CollectActiveJobsKind, collect_active_query_jobs}; @@ -27,20 +26,6 @@ mod job; mod profiling_support; mod query_vtables; -/// Trait that knows how to look up the [`QueryVTable`] for a particular query. -/// -/// This trait allows some per-query code to be defined in generic functions -/// with a trait bound, instead of having to be defined inline within a macro -/// expansion. -/// -/// There is one macro-generated implementation of this trait for each query, -/// on the type `rustc_query_impl::query_vtables::$name::VTableGetter`. -trait GetQueryVTable<'tcx> { - type Cache: QueryCache + 'tcx; - - fn query_vtable(tcx: TyCtxt<'tcx>) -> &'tcx QueryVTable<'tcx, Self::Cache>; -} - pub fn query_system<'tcx>( local_providers: Providers, extern_providers: ExternProviders, diff --git a/compiler/rustc_query_impl/src/query_vtables.rs b/compiler/rustc_query_impl/src/query_vtables.rs index 8976f5d9f499a..0d05be18233a8 100644 --- a/compiler/rustc_query_impl/src/query_vtables.rs +++ b/compiler/rustc_query_impl/src/query_vtables.rs @@ -1,10 +1,22 @@ use rustc_middle::queries::TaggedQueryKey; use rustc_middle::query::erase::{self, Erased}; -use rustc_middle::query::{QueryKey, QueryMode, QueryVTable}; +use rustc_middle::query::{QueryCache, QueryKey, QueryMode, QueryVTable}; use rustc_middle::ty::TyCtxt; use rustc_span::Span; -use crate::GetQueryVTable; +/// Trait that knows how to look up the [`QueryVTable`] for a particular query. +/// +/// This trait allows some per-query code to be defined in generic functions +/// with a trait bound, instead of having to be defined inline within a macro +/// expansion. +/// +/// There is one macro-generated implementation of this trait for each query, +/// on the type `rustc_query_impl::query_vtables::$name::VTableGetter`. +pub(crate) trait GetQueryVTable<'tcx> { + type Cache: QueryCache + 'tcx; + + fn query_vtable(tcx: TyCtxt<'tcx>) -> &'tcx QueryVTable<'tcx, Self::Cache>; +} macro_rules! define_queries { ( From ae1b655c80c4aff6045a3d9ee2b5b40e263853ad Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Mon, 10 Aug 2026 16:47:28 +1000 Subject: [PATCH 13/15] Move some job-related stuff into `job.rs` --- compiler/rustc_query_impl/src/execution.rs | 94 ++-------------------- compiler/rustc_query_impl/src/job.rs | 91 ++++++++++++++++++++- compiler/rustc_query_impl/src/lib.rs | 6 +- 3 files changed, 99 insertions(+), 92 deletions(-) diff --git a/compiler/rustc_query_impl/src/execution.rs b/compiler/rustc_query_impl/src/execution.rs index 03e5d588374ca..3e5ee960e2772 100644 --- a/compiler/rustc_query_impl/src/execution.rs +++ b/compiler/rustc_query_impl/src/execution.rs @@ -2,116 +2,34 @@ use std::hash::Hash; use std::mem::ManuallyDrop; use std::num::NonZero; -use rustc_data_structures::hash_table::{Entry, HashTable}; -use rustc_data_structures::sync::{DynSend, DynSync}; +use rustc_data_structures::hash_table::Entry; use rustc_data_structures::{Limit, defer, outline, sharded, sync}; use rustc_errors::FatalError; use rustc_middle::dep_graph::{ DepGraphData, DepNode, DepNodeIndex, DepNodeKey, SerializedDepNodeIndex, }; use rustc_middle::query::{ - ActiveKeyStatus, Cycle, QueryCache, QueryJob, QueryJobId, QueryKey, QueryLatch, QueryMode, - QueryState, QueryVTable, + ActiveKeyStatus, Cycle, QueryCache, QueryJob, QueryJobId, QueryLatch, QueryMode, QueryState, + QueryVTable, }; use rustc_middle::ty::TyCtxt; use rustc_middle::ty::tls::{self, ImplicitCtxt}; use rustc_middle::verify_ich::incremental_verify_ich; use rustc_span::def_id::LOCAL_CRATE; use rustc_span::{DUMMY_SP, Span}; -use tracing::debug; use crate::diagnostics::{QueryOverflow, QueryOverflowNote}; use crate::handle_cycle_error; use crate::incremental::should_verify_loaded_value; -use crate::job::{QueryJobInfo, QueryJobMap, find_cycle_in_stack, find_dep_kind_root}; -use crate::query_vtables::for_each_query_vtable; +use crate::job::{ + CollectActiveJobsKind, collect_active_query_jobs, find_cycle_in_stack, find_dep_kind_root, +}; #[inline] fn equivalent_key(k: K) -> impl Fn(&(K, V)) -> bool { move |x| x.0 == k } -#[derive(Clone, Copy)] -pub enum CollectActiveJobsKind { - /// We need the full query job map, and we are willing to wait to obtain the query state - /// shard lock(s). - Full, - - /// We need the full query job map, and we shouldn't need to wait to obtain the shard lock(s), - /// because we are in a place where nothing else could hold the shard lock(s). - FullNoContention, - - /// We can get by without the full query job map, so we won't bother waiting to obtain the - /// shard lock(s) if they're not already unlocked. - PartialAllowed, -} - -/// Returns a map of currently active query jobs, collected from all queries. -pub fn collect_active_query_jobs<'tcx>( - tcx: TyCtxt<'tcx>, - collect_kind: CollectActiveJobsKind, -) -> QueryJobMap<'tcx> { - let mut job_map = QueryJobMap::default(); - - for_each_query_vtable!(ALL, tcx, |query| { - collect_active_query_jobs_inner(query, collect_kind, &mut job_map); - }); - - job_map -} - -/// Internal plumbing for collecting the set of active jobs for this query. -/// -/// Aborts if jobs can't be gathered as specified by `collect_kind`. -fn collect_active_query_jobs_inner<'tcx, C>( - query: &'tcx QueryVTable<'tcx, C>, - collect_kind: CollectActiveJobsKind, - job_map: &mut QueryJobMap<'tcx>, -) where - C: QueryCache, - QueryVTable<'tcx, C>: DynSync, -{ - let mut collect_shard_jobs = |shard: &HashTable<(C::Key, ActiveKeyStatus<'tcx>)>| { - for (key, status) in shard.iter() { - if let ActiveKeyStatus::Started(job) = status { - // It's fine to call `create_tagged_key` with the shard locked, - // because it's just a `TaggedQueryKey` variant constructor. - let tagged_key = (query.create_tagged_key)(*key); - job_map.insert(job.id, QueryJobInfo { tagged_key, job: job.clone() }); - } - } - }; - - match collect_kind { - CollectActiveJobsKind::Full => { - for shard in query.state.active.lock_shards() { - collect_shard_jobs(&shard); - } - } - CollectActiveJobsKind::FullNoContention => { - for shard in query.state.active.try_lock_shards() { - match shard { - Some(shard) => collect_shard_jobs(&shard), - None => panic!("Failed to collect active jobs for query `{}`!", query.name), - } - } - } - CollectActiveJobsKind::PartialAllowed => { - for shard in query.state.active.try_lock_shards() { - match shard { - Some(shard) => collect_shard_jobs(&shard), - // This collection is best-effort (it is only used to print the query - // stack on panic), so a contended shard is expected and fine to skip. - // Emitting this at `warn!` would leak nondeterministically into the - // panic output under the parallel front-end, where another thread may - // still hold a shard lock, so keep it at `debug!`. - None => debug!("Failed to collect active jobs for query `{}`!", query.name), - } - } - } - } -} - #[cold] #[inline(never)] fn handle_cycle<'tcx, C: QueryCache>( diff --git a/compiler/rustc_query_impl/src/job.rs b/compiler/rustc_query_impl/src/job.rs index 57601e1781d5f..1f9b04278bd07 100644 --- a/compiler/rustc_query_impl/src/job.rs +++ b/compiler/rustc_query_impl/src/job.rs @@ -4,13 +4,19 @@ use std::sync::Arc; use std::{iter, mem}; use rustc_data_structures::fx::{FxHashMap, FxHashSet}; +use rustc_data_structures::hash_table::HashTable; +use rustc_data_structures::sync::{DynSend, DynSync}; use rustc_errors::DiagCtxtHandle; use rustc_middle::queries::TaggedQueryKey; -use rustc_middle::query::{Cycle, QueryJob, QueryJobId, QueryLatch, QueryStackFrame, QueryWaiter}; +use rustc_middle::query::{ + ActiveKeyStatus, Cycle, QueryCache, QueryJob, QueryJobId, QueryKey, QueryLatch, + QueryStackFrame, QueryVTable, QueryWaiter, +}; use rustc_middle::ty::TyCtxt; use rustc_span::{DUMMY_SP, Span}; +use tracing::debug; -use crate::{CollectActiveJobsKind, collect_active_query_jobs}; +use crate::query_vtables::for_each_query_vtable; /// Map from query job IDs to job information collected by /// `collect_active_query_jobs`. @@ -50,6 +56,87 @@ pub(crate) struct QueryJobInfo<'tcx> { pub(crate) job: QueryJob<'tcx>, } +#[derive(Clone, Copy)] +pub enum CollectActiveJobsKind { + /// We need the full query job map, and we are willing to wait to obtain the query state + /// shard lock(s). + Full, + + /// We need the full query job map, and we shouldn't need to wait to obtain the shard lock(s), + /// because we are in a place where nothing else could hold the shard lock(s). + FullNoContention, + + /// We can get by without the full query job map, so we won't bother waiting to obtain the + /// shard lock(s) if they're not already unlocked. + PartialAllowed, +} + +/// Returns a map of currently active query jobs, collected from all queries. +pub fn collect_active_query_jobs<'tcx>( + tcx: TyCtxt<'tcx>, + collect_kind: CollectActiveJobsKind, +) -> QueryJobMap<'tcx> { + let mut job_map = QueryJobMap::default(); + + for_each_query_vtable!(ALL, tcx, |query| { + collect_active_query_jobs_inner(query, collect_kind, &mut job_map); + }); + + job_map +} + +/// Internal plumbing for collecting the set of active jobs for this query. +/// +/// Aborts if jobs can't be gathered as specified by `collect_kind`. +fn collect_active_query_jobs_inner<'tcx, C>( + query: &'tcx QueryVTable<'tcx, C>, + collect_kind: CollectActiveJobsKind, + job_map: &mut QueryJobMap<'tcx>, +) where + C: QueryCache, + QueryVTable<'tcx, C>: DynSync, +{ + let mut collect_shard_jobs = |shard: &HashTable<(C::Key, ActiveKeyStatus<'tcx>)>| { + for (key, status) in shard.iter() { + if let ActiveKeyStatus::Started(job) = status { + // It's fine to call `create_tagged_key` with the shard locked, + // because it's just a `TaggedQueryKey` variant constructor. + let tagged_key = (query.create_tagged_key)(*key); + job_map.insert(job.id, QueryJobInfo { tagged_key, job: job.clone() }); + } + } + }; + + match collect_kind { + CollectActiveJobsKind::Full => { + for shard in query.state.active.lock_shards() { + collect_shard_jobs(&shard); + } + } + CollectActiveJobsKind::FullNoContention => { + for shard in query.state.active.try_lock_shards() { + match shard { + Some(shard) => collect_shard_jobs(&shard), + None => panic!("Failed to collect active jobs for query `{}`!", query.name), + } + } + } + CollectActiveJobsKind::PartialAllowed => { + for shard in query.state.active.try_lock_shards() { + match shard { + Some(shard) => collect_shard_jobs(&shard), + // This collection is best-effort (it is only used to print the query + // stack on panic), so a contended shard is expected and fine to skip. + // Emitting this at `warn!` would leak nondeterministically into the + // panic output under the parallel front-end, where another thread may + // still hold a shard lock, so keep it at `debug!`. + None => debug!("Failed to collect active jobs for query `{}`!", query.name), + } + } + } + } +} + pub(crate) fn find_cycle_in_stack<'tcx>( id: QueryJobId, job_map: QueryJobMap<'tcx>, diff --git a/compiler/rustc_query_impl/src/lib.rs b/compiler/rustc_query_impl/src/lib.rs index 256def36bc4a0..37daac6a4a48f 100644 --- a/compiler/rustc_query_impl/src/lib.rs +++ b/compiler/rustc_query_impl/src/lib.rs @@ -14,8 +14,10 @@ use rustc_middle::query::QuerySystem; use rustc_middle::query::on_disk_cache::OnDiskCache; pub use crate::dep_kind_vtables::make_dep_kind_vtables; -pub use crate::execution::{CollectActiveJobsKind, collect_active_query_jobs}; -pub use crate::job::{QueryJobMap, break_query_cycle, print_query_stack}; +pub use crate::job::{ + CollectActiveJobsKind, QueryJobMap, break_query_cycle, collect_active_query_jobs, + print_query_stack, +}; mod dep_kind_vtables; mod diagnostics; From d6c550356a42376ffb79e8001f555fc22c3da917 Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Mon, 10 Aug 2026 17:58:01 +1000 Subject: [PATCH 14/15] Rename `profiling_support.rs` as `self_profile.rs` It's a clearer name. --- compiler/rustc_query_impl/src/lib.rs | 4 ++-- .../src/{profiling_support.rs => self_profile.rs} | 0 2 files changed, 2 insertions(+), 2 deletions(-) rename compiler/rustc_query_impl/src/{profiling_support.rs => self_profile.rs} (100%) diff --git a/compiler/rustc_query_impl/src/lib.rs b/compiler/rustc_query_impl/src/lib.rs index 37daac6a4a48f..a5ad875a915c2 100644 --- a/compiler/rustc_query_impl/src/lib.rs +++ b/compiler/rustc_query_impl/src/lib.rs @@ -25,8 +25,8 @@ mod execution; mod handle_cycle_error; mod incremental; mod job; -mod profiling_support; mod query_vtables; +mod self_profile; pub fn query_system<'tcx>( local_providers: Providers, @@ -48,7 +48,7 @@ pub fn query_system<'tcx>( pub fn provide(providers: &mut rustc_middle::util::Providers) { providers.hooks.alloc_self_profile_query_strings = - profiling_support::alloc_self_profile_query_strings; + self_profile::alloc_self_profile_query_strings; providers.hooks.verify_query_key_hashes = incremental::verify_query_key_hashes; providers.hooks.encode_query_values = incremental::encode_query_values; } diff --git a/compiler/rustc_query_impl/src/profiling_support.rs b/compiler/rustc_query_impl/src/self_profile.rs similarity index 100% rename from compiler/rustc_query_impl/src/profiling_support.rs rename to compiler/rustc_query_impl/src/self_profile.rs From 86c2ea09e1ce400be63c9b26d0c547599e743c9b Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Mon, 10 Aug 2026 18:11:54 +1000 Subject: [PATCH 15/15] Remove the crate's doc comment It's out of date. --- compiler/rustc_query_impl/src/lib.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/compiler/rustc_query_impl/src/lib.rs b/compiler/rustc_query_impl/src/lib.rs index a5ad875a915c2..259810b59c435 100644 --- a/compiler/rustc_query_impl/src/lib.rs +++ b/compiler/rustc_query_impl/src/lib.rs @@ -1,5 +1,3 @@ -//! Support for serializing the dep-graph and reloading it. - // tidy-alphabetical-start #![allow(internal_features)] #![feature(core_intrinsics)]