diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index be37f27b4c25a..8abe1d818de1c 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,8 +1,8 @@ + + diff --git a/compiler/rustc_builtin_macros/src/deriving/debug.rs b/compiler/rustc_builtin_macros/src/deriving/debug.rs index 59b50d8ba969d..c41a4e4fd6b8d 100644 --- a/compiler/rustc_builtin_macros/src/deriving/debug.rs +++ b/compiler/rustc_builtin_macros/src/deriving/debug.rs @@ -261,8 +261,8 @@ fn show_fieldless_enum( /// ```text /// impl ::core::fmt::Debug for A { /// fn fmt(&self, f: &mut ::core::fmt::Formatter) -> ::core::fmt::Result { -/// const __NAMES: &str = "ABBBCC"; -/// const __OFFSET: [usize; 4] =[0, 1, 4, 6]; +/// static __NAMES: &str = "ABBBCC"; +/// static __OFFSET: [usize; 4] =[0, 1, 4, 6]; /// let __d = ::core::intrinsics::discriminant_value(self) as usize; /// ::core::fmt::Formatter::debug_c_like_enums_write_str(f, __NAMES, &__OFFSET, __d) /// } @@ -314,8 +314,8 @@ fn show_fieldless_enum_concat_str( ), ); let names_str_body = cx.expr_str(span, Symbol::intern(&concatenated_names)); - let names_const_item = - cx.item_const(span, names_ident, str_ty, Some(names_str_body), ast::ConstItemKind::Body); + let names_static_item = + cx.item_static(span, names_ident, str_ty, ast::Mutability::Not, names_str_body); // Create the constant offset array let offset_ident = Ident::from_str_and_span("__OFFSET", span); @@ -332,12 +332,12 @@ fn show_fieldless_enum_concat_str( None, )), ); - let offset_const_item = cx.item_const( + let offset_static_item = cx.item_static( span, offset_ident, cx.ty(span, TyKind::Array(usize_ty, offset_array_len_expr)), - Some(starts_array_body), - ast::ConstItemKind::Body, + ast::Mutability::Not, + starts_array_body, ); // let __d = ::core::intrinsics::discriminant_value(self) as usize; @@ -372,8 +372,8 @@ fn show_fieldless_enum_concat_str( Some(( thin_vec![ - cx.stmt_item(span, names_const_item), - cx.stmt_item(span, offset_const_item), + cx.stmt_item(span, names_static_item), + cx.stmt_item(span, offset_static_item), discriminant_let_stmt, ], call_expr, diff --git a/compiler/rustc_query_impl/src/dep_kind_vtables.rs b/compiler/rustc_query_impl/src/dep_kind_vtables.rs index 5adcf6c7bb576..495524c3f142a 100644 --- a/compiler/rustc_query_impl/src/dep_kind_vtables.rs +++ b/compiler/rustc_query_impl/src/dep_kind_vtables.rs @@ -3,8 +3,8 @@ use rustc_middle::bug; use rustc_middle::dep_graph::{DepKindVTable, DepNodeKey, KeyFingerprintStyle}; use rustc_middle::query::QueryCache; -use crate::GetQueryVTable; -use crate::plumbing::promote_from_disk_inner; +use crate::incremental::promote_from_disk_inner; +use crate::query_vtables::GetQueryVTable; /// [`DepKindVTable`] constructors for special dep kinds that aren't queries. #[expect(non_snake_case, reason = "use non-snake case to avoid collision with query names")] @@ -166,7 +166,7 @@ macro_rules! define_dep_kind_vtables { let q_vtables: [DepKindVTable<'tcx>; _] = [ $( $crate::dep_kind_vtables::make_dep_kind_vtable_for_query::< - $crate::query_impl::$name::VTableGetter, + $crate::query_vtables::$name::VTableGetter, >( $cache_on_disk, $eval_always, diff --git a/compiler/rustc_query_impl/src/execution.rs b/compiler/rustc_query_impl/src/execution.rs index 190bfaec8887a..3e5ee960e2772 100644 --- a/compiler/rustc_query_impl/src/execution.rs +++ b/compiler/rustc_query_impl/src/execution.rs @@ -1,117 +1,35 @@ use std::hash::Hash; use std::mem::ManuallyDrop; +use std::num::NonZero; -use rustc_data_structures::fingerprint::{Fingerprint, PackedFingerprint}; -use rustc_data_structures::hash_table::{Entry, HashTable}; -use rustc_data_structures::sync::{DynSend, DynSync}; -use rustc_data_structures::{defer, outline, sharded, sync}; +use rustc_data_structures::hash_table::Entry; +use rustc_data_structures::{Limit, defer, outline, sharded, sync}; use rustc_errors::FatalError; -use rustc_middle::dep_graph::{DepGraphData, DepNodeKey, SerializedDepNodeIndex}; +use rustc_middle::dep_graph::{ + DepGraphData, DepNode, DepNodeIndex, DepNodeKey, SerializedDepNodeIndex, +}; use rustc_middle::query::{ - ActiveKeyStatus, Cycle, QueryCache, QueryJob, QueryJobId, QueryKey, QueryLatch, QueryMode, - QueryState, QueryVTable, + ActiveKeyStatus, Cycle, QueryCache, QueryJob, QueryJobId, QueryLatch, QueryMode, QueryState, + QueryVTable, }; use rustc_middle::ty::TyCtxt; +use rustc_middle::ty::tls::{self, ImplicitCtxt}; use rustc_middle::verify_ich::incremental_verify_ich; +use rustc_span::def_id::LOCAL_CRATE; use rustc_span::{DUMMY_SP, Span}; -use tracing::debug; -use crate::dep_graph::{DepNode, DepNodeIndex}; +use crate::diagnostics::{QueryOverflow, QueryOverflowNote}; use crate::handle_cycle_error; -use crate::job::{QueryJobInfo, QueryJobMap, create_cycle_error, find_cycle_in_stack}; -use crate::plumbing::{current_query_job, next_job_id, start_query}; -use crate::query_impl::for_each_query_vtable; +use crate::incremental::should_verify_loaded_value; +use crate::job::{ + CollectActiveJobsKind, collect_active_query_jobs, find_cycle_in_stack, find_dep_kind_root, +}; #[inline] fn equivalent_key(k: K) -> impl Fn(&(K, V)) -> bool { move |x| x.0 == k } -pub(crate) fn all_inactive<'tcx, K>(state: &QueryState<'tcx, K>) -> bool { - state.active.lock_shards().all(|shard| shard.is_empty()) -} - -#[derive(Clone, Copy)] -pub enum CollectActiveJobsKind { - /// We need the full query job map, and we are willing to wait to obtain the query state - /// shard lock(s). - Full, - - /// We need the full query job map, and we shouldn't need to wait to obtain the shard lock(s), - /// because we are in a place where nothing else could hold the shard lock(s). - FullNoContention, - - /// We can get by without the full query job map, so we won't bother waiting to obtain the - /// shard lock(s) if they're not already unlocked. - PartialAllowed, -} - -/// Returns a map of currently active query jobs, collected from all queries. -pub fn collect_active_query_jobs<'tcx>( - tcx: TyCtxt<'tcx>, - collect_kind: CollectActiveJobsKind, -) -> QueryJobMap<'tcx> { - let mut job_map = QueryJobMap::default(); - - for_each_query_vtable!(ALL, tcx, |query| { - collect_active_query_jobs_inner(query, collect_kind, &mut job_map); - }); - - job_map -} - -/// Internal plumbing for collecting the set of active jobs for this query. -/// -/// Aborts if jobs can't be gathered as specified by `collect_kind`. -fn collect_active_query_jobs_inner<'tcx, C>( - query: &'tcx QueryVTable<'tcx, C>, - collect_kind: CollectActiveJobsKind, - job_map: &mut QueryJobMap<'tcx>, -) where - C: QueryCache, - QueryVTable<'tcx, C>: DynSync, -{ - let mut collect_shard_jobs = |shard: &HashTable<(C::Key, ActiveKeyStatus<'tcx>)>| { - for (key, status) in shard.iter() { - if let ActiveKeyStatus::Started(job) = status { - // It's fine to call `create_tagged_key` with the shard locked, - // because it's just a `TaggedQueryKey` variant constructor. - let tagged_key = (query.create_tagged_key)(*key); - job_map.insert(job.id, QueryJobInfo { tagged_key, job: job.clone() }); - } - } - }; - - match collect_kind { - CollectActiveJobsKind::Full => { - for shard in query.state.active.lock_shards() { - collect_shard_jobs(&shard); - } - } - CollectActiveJobsKind::FullNoContention => { - for shard in query.state.active.try_lock_shards() { - match shard { - Some(shard) => collect_shard_jobs(&shard), - None => panic!("Failed to collect active jobs for query `{}`!", query.name), - } - } - } - CollectActiveJobsKind::PartialAllowed => { - for shard in query.state.active.try_lock_shards() { - match shard { - Some(shard) => collect_shard_jobs(&shard), - // This collection is best-effort (it is only used to print the query - // stack on panic), so a contended shard is expected and fine to skip. - // Emitting this at `warn!` would leak nondeterministically into the - // panic output under the parallel front-end, where another thread may - // still hold a shard lock, so keep it at `debug!`. - None => debug!("Failed to collect active jobs for query `{}`!", query.name), - } - } - } - } -} - #[cold] #[inline(never)] fn handle_cycle<'tcx, C: QueryCache>( @@ -135,7 +53,7 @@ fn handle_cycle<'tcx, C: QueryCache>( } let _guard = defer(|| *tcx.query_system.cycle_handler_nesting.lock() -= 1); - let error = create_cycle_error(tcx, &cycle, nested); + let error = handle_cycle_error::create_cycle_error(tcx, &cycle, nested); if nested { // Avoid custom handlers and only use the robust `create_cycle_error` for nested cycle errors @@ -281,6 +199,19 @@ fn wait_for_query<'tcx, C: QueryCache>( } } +#[inline] +fn next_job_id<'tcx>(tcx: TyCtxt<'tcx>) -> QueryJobId { + QueryJobId( + NonZero::new(tcx.query_system.jobs.fetch_add(1, std::sync::atomic::Ordering::Relaxed)) + .unwrap(), + ) +} + +#[inline] +fn current_query_job() -> Option { + tls::with_context(|icx| icx.query) +} + /// Shared main part of both [`execute_query_incr_inner`] and [`execute_query_non_incr_inner`]. #[inline(never)] fn try_execute_query<'tcx, C: QueryCache, const INCR: bool>( @@ -407,6 +338,43 @@ fn check_feedable_consistency<'tcx, C: QueryCache>( } } +fn depth_limit_error<'tcx>(tcx: TyCtxt<'tcx>, job: QueryJobId) { + let job_map = collect_active_query_jobs(tcx, CollectActiveJobsKind::Full); + let (span, desc, depth) = find_dep_kind_root(tcx, job, job_map); + + let suggested_limit = match tcx.recursion_limit() { + Limit(0) => Limit(2), + limit => limit * 2, + }; + + tcx.dcx().emit_fatal(QueryOverflow { + span, + note: QueryOverflowNote { desc, depth }, + suggested_limit, + crate_name: tcx.crate_name(LOCAL_CRATE), + }); +} + +/// Executes a job by changing the `ImplicitCtxt` to point to the new query job while it executes. +#[inline(always)] +fn start_query(job_id: QueryJobId, depth_limit: bool, compute: impl FnOnce() -> R) -> R { + tls::with_context(move |icx| { + if depth_limit && !icx.tcx.recursion_limit().value_within_limit(icx.query_depth) { + depth_limit_error(icx.tcx, job_id); + } + + // Update the `ImplicitCtxt` to point to our new query job. + let icx = ImplicitCtxt { + query: Some(job_id), + query_depth: icx.query_depth + if depth_limit { 1 } else { 0 }, + ..*icx + }; + + // Use the `ImplicitCtxt` while we execute the query. + tls::enter_context(&icx, compute) + }) +} + // Fast path for when incr. comp. is off. #[inline(always)] fn execute_job_non_incr<'tcx, C: QueryCache>( @@ -484,29 +452,6 @@ fn execute_job_incr<'tcx, C: QueryCache>( (result, dep_node_index) } -/// Whether a value loaded from the on-disk cache should have its fingerprint -/// verified with `incremental_verify_ich`. If `-Zincremental-verify-ich` is -/// specified, re-hash results from the cache and make sure that they have the -/// expected fingerprint. -/// -/// If not, we still verify a subset: re-hashing is too expensive to do for -/// every value. The subset rotates with the session count, covering the whole -/// cache every 32 sessions, and is deterministic so that a verification -/// failure reproduces on retry. -/// -/// `to_smaller_hash` mixes both fingerprint halves because neither half is -/// evenly distributed on its own (`DefPathHash` keys share the -/// `StableCrateId`, `HirId` keys contain a sequential id). -pub(crate) fn should_verify_loaded_value( - tcx: TyCtxt<'_>, - dep_graph_data: &DepGraphData, - key_fingerprint: PackedFingerprint, -) -> bool { - let hash = Fingerprint::from(key_fingerprint).to_smaller_hash().as_u64(); - hash % 32 == dep_graph_data.session_count() % 32 - || tcx.sess.opts.unstable_opts.incremental_verify_ich -} - /// Given that the dep node for this query+key is green, obtain a value for it /// by loading one from disk if possible, or by invoking its query provider if /// necessary. diff --git a/compiler/rustc_query_impl/src/handle_cycle_error.rs b/compiler/rustc_query_impl/src/handle_cycle_error.rs index 7e7ed3ab3b525..6bc7bfc59b08b 100644 --- a/compiler/rustc_query_impl/src/handle_cycle_error.rs +++ b/compiler/rustc_query_impl/src/handle_cycle_error.rs @@ -13,9 +13,7 @@ use rustc_middle::queries::TaggedQueryKey; use rustc_middle::query::Cycle; use rustc_middle::ty::{self, Ty, TyCtxt}; use rustc_span::def_id::{DefId, LocalDefId}; -use rustc_span::{ErrorGuaranteed, Span}; - -use crate::job::create_cycle_error; +use rustc_span::{DUMMY_SP, ErrorGuaranteed, Span}; // Default cycle handler used for all queries that don't use the `handle_cycle_error` query // modifier. @@ -343,3 +341,91 @@ fn find_item_ty_spans( _ => {} } } + +#[inline(never)] +#[cold] +pub(crate) fn create_cycle_error<'tcx>( + tcx: TyCtxt<'tcx>, + Cycle { usage, frames }: &Cycle<'tcx>, + nested: bool, +) -> Diag<'tcx> { + assert!(!frames.is_empty()); + + let span = frames[0].tagged_key.catch_default_span(tcx, frames[1 % frames.len()].span); + + let mut cycle_stack = Vec::new(); + + use crate::diagnostics::StackCount; + let stack_bottom = frames[0].tagged_key.catch_description(tcx); + let stack_count = if frames.len() == 1 { + StackCount::Single { stack_bottom: stack_bottom.clone() } + } else { + StackCount::Multiple { stack_bottom: stack_bottom.clone() } + }; + + let mut prev = span; + for i in 1..frames.len() { + let frame = &frames[i]; + let span = frame.tagged_key.catch_default_span(tcx, frames[(i + 1) % frames.len()].span); + cycle_stack.push(crate::diagnostics::CycleStack { + span: if span == prev { DUMMY_SP } else { span }, + desc: frame.tagged_key.catch_description(tcx), + }); + prev = span; + } + + let cycle_usage = usage.as_ref().map(|usage| { + let cycle_span = usage.tagged_key.catch_default_span(tcx, usage.span); + crate::diagnostics::CycleUsage { + span: if cycle_span != span { cycle_span } else { DUMMY_SP }, + usage: usage.tagged_key.catch_description(tcx), + } + }); + + let is_all_def_kind = |def_kind| { + // Trivial type alias and trait alias cycles consists of `type_of` and + // `explicit_implied_clauses_of` queries, so we just check just these here. + frames.iter().all(|frame| match frame.tagged_key { + TaggedQueryKey::type_of(def_id) + | TaggedQueryKey::explicit_implied_clauses_of(def_id) + if tcx.def_kind(def_id) == def_kind => + { + true + } + _ => false, + }) + }; + + let alias = if !nested { + if is_all_def_kind(DefKind::TyAlias) { + Some(crate::diagnostics::Alias::Ty) + } else if is_all_def_kind(DefKind::TraitAlias) { + Some(crate::diagnostics::Alias::Trait) + } else { + None + } + } else { + None + }; + + if nested { + tcx.sess.dcx().create_err(crate::diagnostics::NestedCycle { + span, + cycle_stack, + stack_bottom: crate::diagnostics::NestedCycleBottom { stack_bottom }, + cycle_usage, + stack_count, + note_span: (), + }) + } else { + tcx.sess.dcx().create_err(crate::diagnostics::Cycle { + span, + cycle_stack, + stack_bottom, + alias, + cycle_usage, + stack_count, + note_span: (), + }) + } +} diff --git a/compiler/rustc_query_impl/src/plumbing.rs b/compiler/rustc_query_impl/src/incremental.rs similarity index 68% rename from compiler/rustc_query_impl/src/plumbing.rs rename to compiler/rustc_query_impl/src/incremental.rs index e358be327f240..341c9f5e5068d 100644 --- a/compiler/rustc_query_impl/src/plumbing.rs +++ b/compiler/rustc_query_impl/src/incremental.rs @@ -1,78 +1,22 @@ -use std::num::NonZero; - -use rustc_data_structures::Limit; +use rustc_data_structures::fingerprint::{Fingerprint, PackedFingerprint}; use rustc_data_structures::unord::UnordMap; use rustc_middle::bug; #[expect(unused_imports, reason = "used by doc comments")] use rustc_middle::dep_graph::DepKindVTable; -use rustc_middle::dep_graph::{DepNode, DepNodeIndex, DepNodeKey, SerializedDepNodeIndex}; +use rustc_middle::dep_graph::{ + DepGraphData, DepNode, DepNodeIndex, DepNodeKey, SerializedDepNodeIndex, +}; use rustc_middle::query::erase::{Erasable, Erased}; use rustc_middle::query::on_disk_cache::{CacheDecoder, CacheEncoder}; -use rustc_middle::query::{QueryCache, QueryJobId, QueryVTable, erase}; +use rustc_middle::query::{QueryCache, QueryState, QueryVTable, erase}; use rustc_middle::ty::TyCtxt; -use rustc_middle::ty::tls::{self, ImplicitCtxt}; use rustc_middle::verify_ich::incremental_verify_ich; use rustc_serialize::{Decodable, Encodable}; -use rustc_span::def_id::LOCAL_CRATE; - -use crate::diagnostics::{QueryOverflow, QueryOverflowNote}; -use crate::execution::{all_inactive, should_verify_loaded_value}; -use crate::job::find_dep_kind_root; -use crate::query_impl::for_each_query_vtable; -use crate::{CollectActiveJobsKind, collect_active_query_jobs}; - -fn depth_limit_error<'tcx>(tcx: TyCtxt<'tcx>, job: QueryJobId) { - let job_map = collect_active_query_jobs(tcx, CollectActiveJobsKind::Full); - let (span, desc, depth) = find_dep_kind_root(tcx, job, job_map); - - let suggested_limit = match tcx.recursion_limit() { - Limit(0) => Limit(2), - limit => limit * 2, - }; - - tcx.dcx().emit_fatal(QueryOverflow { - span, - note: QueryOverflowNote { desc, depth }, - suggested_limit, - crate_name: tcx.crate_name(LOCAL_CRATE), - }); -} -#[inline] -pub(crate) fn next_job_id<'tcx>(tcx: TyCtxt<'tcx>) -> QueryJobId { - QueryJobId( - NonZero::new(tcx.query_system.jobs.fetch_add(1, std::sync::atomic::Ordering::Relaxed)) - .unwrap(), - ) -} - -#[inline] -pub(crate) fn current_query_job() -> Option { - tls::with_context(|icx| icx.query) -} - -/// Executes a job by changing the `ImplicitCtxt` to point to the new query job while it executes. -#[inline(always)] -pub(crate) fn start_query( - job_id: QueryJobId, - depth_limit: bool, - compute: impl FnOnce() -> R, -) -> R { - tls::with_context(move |icx| { - if depth_limit && !icx.tcx.recursion_limit().value_within_limit(icx.query_depth) { - depth_limit_error(icx.tcx, job_id); - } - - // Update the `ImplicitCtxt` to point to our new query job. - let icx = ImplicitCtxt { - query: Some(job_id), - query_depth: icx.query_depth + if depth_limit { 1 } else { 0 }, - ..*icx - }; +use crate::query_vtables::for_each_query_vtable; - // Use the `ImplicitCtxt` while we execute the query. - tls::enter_context(&icx, compute) - }) +fn all_inactive<'tcx, K>(state: &QueryState<'tcx, K>) -> bool { + state.active.lock_shards().all(|shard| shard.is_empty()) } pub(crate) fn encode_query_values<'tcx>(tcx: TyCtxt<'tcx>, encoder: &mut CacheEncoder<'_, 'tcx>) { @@ -135,6 +79,29 @@ fn verify_query_key_hashes_inner<'tcx, C: QueryCache>( }); } +/// Whether a value loaded from the on-disk cache should have its fingerprint +/// verified with `incremental_verify_ich`. If `-Zincremental-verify-ich` is +/// specified, re-hash results from the cache and make sure that they have the +/// expected fingerprint. +/// +/// If not, we still verify a subset: re-hashing is too expensive to do for +/// every value. The subset rotates with the session count, covering the whole +/// cache every 32 sessions, and is deterministic so that a verification +/// failure reproduces on retry. +/// +/// `to_smaller_hash` mixes both fingerprint halves because neither half is +/// evenly distributed on its own (`DefPathHash` keys share the +/// `StableCrateId`, `HirId` keys contain a sequential id). +pub(crate) fn should_verify_loaded_value( + tcx: TyCtxt<'_>, + dep_graph_data: &DepGraphData, + key_fingerprint: PackedFingerprint, +) -> bool { + let hash = Fingerprint::from(key_fingerprint).to_smaller_hash().as_u64(); + hash % 32 == dep_graph_data.session_count() % 32 + || tcx.sess.opts.unstable_opts.incremental_verify_ich +} + /// Inner implementation of [`DepKindVTable::promote_from_disk_fn`] for queries. pub(crate) fn promote_from_disk_inner<'tcx, C: QueryCache>( tcx: TyCtxt<'tcx>, diff --git a/compiler/rustc_query_impl/src/job.rs b/compiler/rustc_query_impl/src/job.rs index 1b604409f38a6..1f9b04278bd07 100644 --- a/compiler/rustc_query_impl/src/job.rs +++ b/compiler/rustc_query_impl/src/job.rs @@ -4,14 +4,19 @@ use std::sync::Arc; use std::{iter, mem}; use rustc_data_structures::fx::{FxHashMap, FxHashSet}; -use rustc_errors::{Diag, DiagCtxtHandle}; -use rustc_hir::def::DefKind; +use rustc_data_structures::hash_table::HashTable; +use rustc_data_structures::sync::{DynSend, DynSync}; +use rustc_errors::DiagCtxtHandle; use rustc_middle::queries::TaggedQueryKey; -use rustc_middle::query::{Cycle, QueryJob, QueryJobId, QueryLatch, QueryStackFrame, QueryWaiter}; +use rustc_middle::query::{ + ActiveKeyStatus, Cycle, QueryCache, QueryJob, QueryJobId, QueryKey, QueryLatch, + QueryStackFrame, QueryVTable, QueryWaiter, +}; use rustc_middle::ty::TyCtxt; use rustc_span::{DUMMY_SP, Span}; +use tracing::debug; -use crate::{CollectActiveJobsKind, collect_active_query_jobs}; +use crate::query_vtables::for_each_query_vtable; /// Map from query job IDs to job information collected by /// `collect_active_query_jobs`. @@ -51,6 +56,87 @@ pub(crate) struct QueryJobInfo<'tcx> { pub(crate) job: QueryJob<'tcx>, } +#[derive(Clone, Copy)] +pub enum CollectActiveJobsKind { + /// We need the full query job map, and we are willing to wait to obtain the query state + /// shard lock(s). + Full, + + /// We need the full query job map, and we shouldn't need to wait to obtain the shard lock(s), + /// because we are in a place where nothing else could hold the shard lock(s). + FullNoContention, + + /// We can get by without the full query job map, so we won't bother waiting to obtain the + /// shard lock(s) if they're not already unlocked. + PartialAllowed, +} + +/// Returns a map of currently active query jobs, collected from all queries. +pub fn collect_active_query_jobs<'tcx>( + tcx: TyCtxt<'tcx>, + collect_kind: CollectActiveJobsKind, +) -> QueryJobMap<'tcx> { + let mut job_map = QueryJobMap::default(); + + for_each_query_vtable!(ALL, tcx, |query| { + collect_active_query_jobs_inner(query, collect_kind, &mut job_map); + }); + + job_map +} + +/// Internal plumbing for collecting the set of active jobs for this query. +/// +/// Aborts if jobs can't be gathered as specified by `collect_kind`. +fn collect_active_query_jobs_inner<'tcx, C>( + query: &'tcx QueryVTable<'tcx, C>, + collect_kind: CollectActiveJobsKind, + job_map: &mut QueryJobMap<'tcx>, +) where + C: QueryCache, + QueryVTable<'tcx, C>: DynSync, +{ + let mut collect_shard_jobs = |shard: &HashTable<(C::Key, ActiveKeyStatus<'tcx>)>| { + for (key, status) in shard.iter() { + if let ActiveKeyStatus::Started(job) = status { + // It's fine to call `create_tagged_key` with the shard locked, + // because it's just a `TaggedQueryKey` variant constructor. + let tagged_key = (query.create_tagged_key)(*key); + job_map.insert(job.id, QueryJobInfo { tagged_key, job: job.clone() }); + } + } + }; + + match collect_kind { + CollectActiveJobsKind::Full => { + for shard in query.state.active.lock_shards() { + collect_shard_jobs(&shard); + } + } + CollectActiveJobsKind::FullNoContention => { + for shard in query.state.active.try_lock_shards() { + match shard { + Some(shard) => collect_shard_jobs(&shard), + None => panic!("Failed to collect active jobs for query `{}`!", query.name), + } + } + } + CollectActiveJobsKind::PartialAllowed => { + for shard in query.state.active.try_lock_shards() { + match shard { + Some(shard) => collect_shard_jobs(&shard), + // This collection is best-effort (it is only used to print the query + // stack on panic), so a contended shard is expected and fine to skip. + // Emitting this at `warn!` would leak nondeterministically into the + // panic output under the parallel front-end, where another thread may + // still hold a shard lock, so keep it at `debug!`. + None => debug!("Failed to collect active jobs for query `{}`!", query.name), + } + } + } + } +} + pub(crate) fn find_cycle_in_stack<'tcx>( id: QueryJobId, job_map: QueryJobMap<'tcx>, @@ -407,91 +493,3 @@ pub fn print_query_stack<'tcx>( } count_total } - -#[inline(never)] -#[cold] -pub(crate) fn create_cycle_error<'tcx>( - tcx: TyCtxt<'tcx>, - Cycle { usage, frames }: &Cycle<'tcx>, - nested: bool, -) -> Diag<'tcx> { - assert!(!frames.is_empty()); - - let span = frames[0].tagged_key.catch_default_span(tcx, frames[1 % frames.len()].span); - - let mut cycle_stack = Vec::new(); - - use crate::diagnostics::StackCount; - let stack_bottom = frames[0].tagged_key.catch_description(tcx); - let stack_count = if frames.len() == 1 { - StackCount::Single { stack_bottom: stack_bottom.clone() } - } else { - StackCount::Multiple { stack_bottom: stack_bottom.clone() } - }; - - let mut prev = span; - for i in 1..frames.len() { - let frame = &frames[i]; - let span = frame.tagged_key.catch_default_span(tcx, frames[(i + 1) % frames.len()].span); - cycle_stack.push(crate::diagnostics::CycleStack { - span: if span == prev { DUMMY_SP } else { span }, - desc: frame.tagged_key.catch_description(tcx), - }); - prev = span; - } - - let cycle_usage = usage.as_ref().map(|usage| { - let cycle_span = usage.tagged_key.catch_default_span(tcx, usage.span); - crate::diagnostics::CycleUsage { - span: if cycle_span != span { cycle_span } else { DUMMY_SP }, - usage: usage.tagged_key.catch_description(tcx), - } - }); - - let is_all_def_kind = |def_kind| { - // Trivial type alias and trait alias cycles consists of `type_of` and - // `explicit_implied_clauses_of` queries, so we just check just these here. - frames.iter().all(|frame| match frame.tagged_key { - TaggedQueryKey::type_of(def_id) - | TaggedQueryKey::explicit_implied_clauses_of(def_id) - if tcx.def_kind(def_id) == def_kind => - { - true - } - _ => false, - }) - }; - - let alias = if !nested { - if is_all_def_kind(DefKind::TyAlias) { - Some(crate::diagnostics::Alias::Ty) - } else if is_all_def_kind(DefKind::TraitAlias) { - Some(crate::diagnostics::Alias::Trait) - } else { - None - } - } else { - None - }; - - if nested { - tcx.sess.dcx().create_err(crate::diagnostics::NestedCycle { - span, - cycle_stack, - stack_bottom: crate::diagnostics::NestedCycleBottom { stack_bottom }, - cycle_usage, - stack_count, - note_span: (), - }) - } else { - tcx.sess.dcx().create_err(crate::diagnostics::Cycle { - span, - cycle_stack, - stack_bottom, - alias, - cycle_usage, - stack_count, - note_span: (), - }) - } -} diff --git a/compiler/rustc_query_impl/src/lib.rs b/compiler/rustc_query_impl/src/lib.rs index 0a8b25c2fa878..259810b59c435 100644 --- a/compiler/rustc_query_impl/src/lib.rs +++ b/compiler/rustc_query_impl/src/lib.rs @@ -1,5 +1,3 @@ -//! Support for serializing the dep-graph and reloading it. - // tidy-alphabetical-start #![allow(internal_features)] #![feature(core_intrinsics)] @@ -9,38 +7,24 @@ // tidy-alphabetical-end use rustc_data_structures::sync::{AtomicU64, Lock}; -use rustc_middle::dep_graph; use rustc_middle::queries::{ExternProviders, Providers}; +use rustc_middle::query::QuerySystem; use rustc_middle::query::on_disk_cache::OnDiskCache; -use rustc_middle::query::{QueryCache, QuerySystem, QueryVTable}; -use rustc_middle::ty::TyCtxt; pub use crate::dep_kind_vtables::make_dep_kind_vtables; -pub use crate::execution::{CollectActiveJobsKind, collect_active_query_jobs}; -pub use crate::job::{QueryJobMap, break_query_cycle, print_query_stack}; +pub use crate::job::{ + CollectActiveJobsKind, QueryJobMap, break_query_cycle, collect_active_query_jobs, + print_query_stack, +}; mod dep_kind_vtables; mod diagnostics; mod execution; mod handle_cycle_error; +mod incremental; mod job; -mod plumbing; -mod profiling_support; -mod query_impl; - -/// Trait that knows how to look up the [`QueryVTable`] for a particular query. -/// -/// This trait allows some per-query code to be defined in generic functions -/// with a trait bound, instead of having to be defined inline within a macro -/// expansion. -/// -/// There is one macro-generated implementation of this trait for each query, -/// on the type `rustc_query_impl::query_impl::$name::VTableGetter`. -trait GetQueryVTable<'tcx> { - type Cache: QueryCache + 'tcx; - - fn query_vtable(tcx: TyCtxt<'tcx>) -> &'tcx QueryVTable<'tcx, Self::Cache>; -} +mod query_vtables; +mod self_profile; pub fn query_system<'tcx>( local_providers: Providers, @@ -50,7 +34,7 @@ pub fn query_system<'tcx>( ) -> QuerySystem<'tcx> { QuerySystem { arenas: Default::default(), - query_vtables: query_impl::make_query_vtables(incremental), + query_vtables: query_vtables::make_query_vtables(incremental), side_effects: Default::default(), on_disk_cache, local_providers, @@ -62,7 +46,7 @@ pub fn query_system<'tcx>( pub fn provide(providers: &mut rustc_middle::util::Providers) { providers.hooks.alloc_self_profile_query_strings = - profiling_support::alloc_self_profile_query_strings; - providers.hooks.verify_query_key_hashes = plumbing::verify_query_key_hashes; - providers.hooks.encode_query_values = plumbing::encode_query_values; + self_profile::alloc_self_profile_query_strings; + providers.hooks.verify_query_key_hashes = incremental::verify_query_key_hashes; + providers.hooks.encode_query_values = incremental::encode_query_values; } diff --git a/compiler/rustc_query_impl/src/query_impl.rs b/compiler/rustc_query_impl/src/query_vtables.rs similarity index 91% rename from compiler/rustc_query_impl/src/query_impl.rs rename to compiler/rustc_query_impl/src/query_vtables.rs index 3720d9fd80547..0d05be18233a8 100644 --- a/compiler/rustc_query_impl/src/query_impl.rs +++ b/compiler/rustc_query_impl/src/query_vtables.rs @@ -1,10 +1,22 @@ use rustc_middle::queries::TaggedQueryKey; use rustc_middle::query::erase::{self, Erased}; -use rustc_middle::query::{QueryKey, QueryMode, QueryVTable}; +use rustc_middle::query::{QueryCache, QueryKey, QueryMode, QueryVTable}; use rustc_middle::ty::TyCtxt; use rustc_span::Span; -use crate::GetQueryVTable; +/// Trait that knows how to look up the [`QueryVTable`] for a particular query. +/// +/// This trait allows some per-query code to be defined in generic functions +/// with a trait bound, instead of having to be defined inline within a macro +/// expansion. +/// +/// There is one macro-generated implementation of this trait for each query, +/// on the type `rustc_query_impl::query_vtables::$name::VTableGetter`. +pub(crate) trait GetQueryVTable<'tcx> { + type Cache: QueryCache + 'tcx; + + fn query_vtable(tcx: TyCtxt<'tcx>) -> &'tcx QueryVTable<'tcx, Self::Cache>; +} macro_rules! define_queries { ( @@ -33,7 +45,7 @@ macro_rules! define_queries { // Non-queries are unused here. non_queries { $($_:tt)* } ) => { - // This macro expects to be expanded into `crate::query_impl`, which is this file. + // This macro expects to be expanded into `crate::query_vtables`, which is this file. $( pub(crate) mod $name { use super::*; @@ -149,7 +161,7 @@ macro_rules! define_queries { use rustc_middle::queries::$name::{ProvidedValue, provided_to_erased}; let loaded_value: ProvidedValue<'tcx> = - $crate::plumbing::try_load_from_disk(tcx, prev_index)?; + $crate::incremental::try_load_from_disk(tcx, prev_index)?; // Arena-alloc the value if appropriate, and erase it. Some(provided_to_erased(tcx, loaded_value)) @@ -181,9 +193,9 @@ macro_rules! define_queries { }, create_tagged_key: TaggedQueryKey::$name, execute_query_fn: if incremental { - crate::query_impl::$name::execute_query_incr::__rust_end_short_backtrace + crate::query_vtables::$name::execute_query_incr::__rust_end_short_backtrace } else { - crate::query_impl::$name::execute_query_non_incr::__rust_end_short_backtrace + crate::query_vtables::$name::execute_query_non_incr::__rust_end_short_backtrace }, } } @@ -207,7 +219,7 @@ macro_rules! define_queries { { rustc_middle::queries::QueryVTables { $( - $name: crate::query_impl::$name::make_query_vtable(incremental), + $name: crate::query_vtables::$name::make_query_vtable(incremental), )* } } diff --git a/compiler/rustc_query_impl/src/profiling_support.rs b/compiler/rustc_query_impl/src/self_profile.rs similarity index 99% rename from compiler/rustc_query_impl/src/profiling_support.rs rename to compiler/rustc_query_impl/src/self_profile.rs index 980e2b1305245..53734b84a9a10 100644 --- a/compiler/rustc_query_impl/src/profiling_support.rs +++ b/compiler/rustc_query_impl/src/self_profile.rs @@ -9,7 +9,7 @@ use rustc_hir::definitions::DefPathData; use rustc_middle::query::{QueryCache, QueryVTable}; use rustc_middle::ty::TyCtxt; -use crate::query_impl::for_each_query_vtable; +use crate::query_vtables::for_each_query_vtable; pub(crate) struct QueryKeyStringCache { def_id_cache: FxHashMap, diff --git a/library/core/src/cell/once.rs b/library/core/src/cell/once.rs index 799e19cce67e6..01ba604e67949 100644 --- a/library/core/src/cell/once.rs +++ b/library/core/src/cell/once.rs @@ -47,6 +47,31 @@ impl OnceCell { OnceCell { inner: UnsafeCell::new(None) } } + /// Creates a new initialized cell. + /// + /// This is equivalent to `OnceCell::from(value)`, but can be used in + /// const contexts, unlike the `From` implementation. + /// + /// # Examples + /// + /// ``` + /// #![feature(once_cell_new_init)] + /// use std::cell::OnceCell; + /// + /// const CELL: OnceCell = OnceCell::new_init(1); + /// assert_eq!(CELL.get(), Some(&1)); + /// + /// let cell = OnceCell::new_init(String::from("kitty")); + /// assert_eq!(cell.set(String::from("puppy")), Err(String::from("puppy"))); + /// assert_eq!(cell.get(), Some(&"kitty".to_string())); + /// ``` + #[inline] + #[must_use] + #[unstable(feature = "once_cell_new_init", issue = "159859")] + pub const fn new_init(init_value: T) -> OnceCell { + OnceCell { inner: UnsafeCell::new(Some(init_value)) } + } + /// Gets the reference to the underlying value. /// /// Returns `None` if the cell is uninitialized. diff --git a/library/core/src/io/borrowed_buf.rs b/library/core/src/io/borrowed_buf.rs index 7ca6f6d8a02e8..f2926f1ce7cf9 100644 --- a/library/core/src/io/borrowed_buf.rs +++ b/library/core/src/io/borrowed_buf.rs @@ -2,7 +2,6 @@ use crate::fmt::{self, Debug, Formatter}; use crate::mem::{self, MaybeUninit}; -use crate::ptr; /// A borrowed buffer of initially uninitialized elements, which is incrementally filled. /// @@ -357,24 +356,21 @@ impl<'a, T: Copy> BorrowedCursor<'a, T> { } } -impl<'a> BorrowedCursor<'a, u8> { - /// Initializes all bytes in the cursor and returns them. +impl<'a, T: Default + Copy> BorrowedCursor<'a, T> { + /// Initializes all elements in the cursor with their default value and + /// returns them. #[unstable(feature = "borrowed_buf_init", issue = "160476")] #[inline] - pub fn ensure_init(&mut self) -> &mut [u8] { - // SAFETY: always in bounds and we never uninitialize these bytes. + pub fn ensure_init(&mut self) -> &mut [T] { + // SAFETY: always in bounds and we never uninitialize these elements. let unfilled = unsafe { self.buf.buf.get_unchecked_mut(self.buf.filled..) }; if !self.buf.init { - // SAFETY: 0 is a valid value for MaybeUninit and the length matches the allocation - // since it is comes from a slice reference. - unsafe { - ptr::write_bytes(unfilled.as_mut_ptr(), 0, unfilled.len()); - } + unfilled.write_default(); self.buf.init = true; } - // SAFETY: these bytes have just been initialized if they weren't before + // SAFETY: these elements have just been initialized if they weren't before unsafe { unfilled.assume_init_mut() } } } diff --git a/library/core/src/mem/maybe_uninit.rs b/library/core/src/mem/maybe_uninit.rs index 9bc8b0d128d2b..6275d7cd59a2c 100644 --- a/library/core/src/mem/maybe_uninit.rs +++ b/library/core/src/mem/maybe_uninit.rs @@ -1286,8 +1286,8 @@ impl [MaybeUninit] { /// Fills a slice with elements returned by calling a closure for each index. /// /// This method uses a closure to create new values. If you'd rather `Clone` a given value, use - /// [slice::write_filled]. If you want to use the `Default` trait to generate values, you can - /// pass [`|_| Default::default()`][Default::default] as the argument. + /// [`slice::write_filled`]. If you want to use the `Default` trait to generate values, use + /// [`slice::write_default`]. /// /// # Panics /// @@ -1324,6 +1324,73 @@ impl [MaybeUninit] { unsafe { self.assume_init_mut() } } + /// Fills a slice with elements returned by calling [`Default::default`] for each index. + /// + /// # Panics + /// + /// This function will panic if any call to [`Default::default`] panics. + /// + /// If such a panic occurs, any elements previously initialized during this operation will be + /// dropped. + /// + /// # Examples + /// + /// ``` + /// #![feature(maybe_uninit_fill)] + /// use std::mem::MaybeUninit; + /// + /// let mut buf = [const { MaybeUninit::::uninit() }; 5]; + /// let initialized = buf.write_default(); + /// assert_eq!(initialized, &mut [0, 0, 0, 0, 0]); + /// ``` + #[unstable(feature = "maybe_uninit_fill", issue = "117428")] + pub fn write_default(&mut self) -> &mut [T] + where + T: Default, + { + trait DefaultSpec: Default { + fn write_default(buf: &mut [MaybeUninit]) -> &mut [Self]; + } + + impl DefaultSpec for T { + default fn write_default(buf: &mut [MaybeUninit]) -> &mut [Self] { + buf.write_with(|_| T::default()) + } + } + + macro_rules! spec_default_zero { + ($ty:ty) => { + impl DefaultSpec for $ty { + fn write_default(buf: &mut [MaybeUninit]) -> &mut [Self] { + // SAFETY: + // `Default::default` is equivalent to zero-initialization + // for all these types, and this initializes the entire + // slice. + unsafe { + buf.as_mut_ptr().write_bytes(0, buf.len()); + buf.assume_init_mut() + } + } + } + }; + } + + spec_default_zero!(i8); + spec_default_zero!(u8); + spec_default_zero!(i16); + spec_default_zero!(u16); + spec_default_zero!(i32); + spec_default_zero!(u32); + spec_default_zero!(i64); + spec_default_zero!(u64); + spec_default_zero!(i128); + spec_default_zero!(u128); + spec_default_zero!(isize); + spec_default_zero!(usize); + + T::write_default(self) + } + /// Fills a slice with elements yielded by an iterator until either all elements have been /// initialized or the iterator is empty. /// diff --git a/library/std/src/sync/once_lock.rs b/library/std/src/sync/once_lock.rs index de80164ed4f85..4b41fc4587829 100644 --- a/library/std/src/sync/once_lock.rs +++ b/library/std/src/sync/once_lock.rs @@ -145,6 +145,35 @@ impl OnceLock { } } + /// Creates a new initialized cell. + /// + /// This is equivalent to `OnceLock::from(value)`, but can be used in + /// const contexts, unlike the `From` implementation. + /// + /// # Examples + /// + /// ``` + /// #![feature(once_lock_new_init)] + /// use std::sync::OnceLock; + /// + /// static CELL: OnceLock = OnceLock::new_init(1); + /// + /// assert_eq!(CELL.get(), Some(&1)); + /// + /// // Already initialized, so this closure never runs. + /// assert_eq!(CELL.get_or_init(|| panic!("Kaboom!")), &1); + /// ``` + #[inline] + #[must_use] + #[unstable(feature = "once_lock_new_init", issue = "159860")] + pub const fn new_init(init_value: T) -> OnceLock { + OnceLock { + once: Once::new_complete(), + value: UnsafeCell::new(MaybeUninit::new(init_value)), + _marker: PhantomData, + } + } + /// Gets the reference to the underlying value. /// /// Returns `None` if the cell is uninitialized, or being initialized. diff --git a/src/doc/reference b/src/doc/reference index afdc77bab886d..82da570bafd9e 160000 --- a/src/doc/reference +++ b/src/doc/reference @@ -1 +1 @@ -Subproject commit afdc77bab886d4455c11247cdd32391bfab636ae +Subproject commit 82da570bafd9efed421bdbae0f8603ede8dd308b diff --git a/tests/ui/derives/deriving-all-codegen.stdout b/tests/ui/derives/deriving-all-codegen.stdout index 6b72b08aefa5a..d0b387dbc8b25 100644 --- a/tests/ui/derives/deriving-all-codegen.stdout +++ b/tests/ui/derives/deriving-all-codegen.stdout @@ -1275,9 +1275,9 @@ enum Fieldless10 { impl ::core::fmt::Debug for Fieldless10 { #[inline] fn fmt(&self, f: &mut ::core::fmt::Formatter) -> ::core::fmt::Result { - const __NAMES: &str = + static __NAMES: &str = "AAAAABBBBCCDDDDDDDDEFFFFFFFFFFFFFGGGGGGHatsuneIIIIIIIJJJJJJJJJ"; - const __OFFSET: [usize; 11] = + static __OFFSET: [usize; 11] = [0usize, 5usize, 9usize, 11usize, 19usize, 20usize, 33usize, 39usize, 46usize, 53usize, 62usize]; let __d = ::core::intrinsics::discriminant_value(self) as usize;