diff --git a/Cargo.lock b/Cargo.lock index 873182dde9102..47c4276449ee4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1345,9 +1345,9 @@ checksum = "edd0f118536f44f5ccd48bcb8b111bdc3de888b58c74639dfb034a357d0f206d" [[package]] name = "ena" -version = "0.14.3" +version = "0.14.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d248bdd43ce613d87415282f69b9bb99d947d290b10962dd6c56233312c2ad5" +checksum = "eabffdaee24bd1bf95c5ef7cec31260444317e72ea56c4c91750e8b7ee58d5f1" dependencies = [ "log", ] diff --git a/compiler/rustc_data_structures/Cargo.toml b/compiler/rustc_data_structures/Cargo.toml index 6db3a624500a9..66bfb07d51dfd 100644 --- a/compiler/rustc_data_structures/Cargo.toml +++ b/compiler/rustc_data_structures/Cargo.toml @@ -9,7 +9,7 @@ arrayvec = { version = "0.7", default-features = false } bitflags = "2.4.1" either = "1.0" elsa = "1.11.0" -ena = "0.14.3" +ena = "0.14.4" indexmap = "2.14.0" jobserver_crate = { version = "0.1.28", package = "jobserver" } measureme = "12.0.1" diff --git a/compiler/rustc_infer/src/infer/context.rs b/compiler/rustc_infer/src/infer/context.rs index d1a56d5ab073c..2c99180a094e0 100644 --- a/compiler/rustc_infer/src/infer/context.rs +++ b/compiler/rustc_infer/src/infer/context.rs @@ -5,6 +5,7 @@ use rustc_middle::ty::relate::RelateResult; use rustc_middle::ty::relate::combine::PredicateEmittingRelation; use rustc_middle::ty::{self, Ty, TyCtxt, TypeFoldable}; use rustc_span::{DUMMY_SP, ErrorGuaranteed, Span}; +use rustc_type_ir::solve::TyOrConstInferVar; use super::{ BoundRegionConversionTime, InferCtxt, OpaqueTypeStorageEntries, RegionVariableOrigin, @@ -131,55 +132,8 @@ impl<'tcx> rustc_type_ir::InferCtxtLike for InferCtxt<'tcx> { self.inner.borrow_mut().unwrap_region_constraints().opportunistic_resolve_var(self.tcx, vid) } - fn is_changed_arg(&self, arg: ty::GenericArg<'tcx>) -> bool { - match arg.kind() { - ty::GenericArgKind::Lifetime(_) => { - // Lifetimes should not change affect trait selection. - false - } - ty::GenericArgKind::Type(ty) => { - if let ty::Infer(infer_ty) = *ty.kind() { - match infer_ty { - ty::InferTy::TyVar(vid) => { - !self.try_resolve_ty_var(vid).is_err_and(|_| self.root_var(vid) == vid) - } - ty::InferTy::IntVar(vid) => { - let mut inner = self.inner.borrow_mut(); - !matches!( - inner.int_unification_table().probe_value(vid), - ty::IntVarValue::Unknown - if inner.int_unification_table().find(vid) == vid - ) - } - ty::InferTy::FloatVar(vid) => { - let mut inner = self.inner.borrow_mut(); - !matches!( - inner.float_unification_table().probe_value(vid), - ty::FloatVarValue::Unknown - if inner.float_unification_table().find(vid) == vid - ) - } - ty::InferTy::FreshTy(_) - | ty::InferTy::FreshIntTy(_) - | ty::InferTy::FreshFloatTy(_) => true, - } - } else { - true - } - } - ty::GenericArgKind::Const(ct) => { - if let ty::ConstKind::Infer(infer_ct) = ct.kind() { - match infer_ct { - ty::InferConst::Var(vid) => !self - .try_resolve_const_var(vid) - .is_err_and(|_| self.root_const_var(vid) == vid), - ty::InferConst::Fresh(_) => true, - } - } else { - true - } - } - } + fn ty_or_const_infer_var_changed(&self, var: TyOrConstInferVar) -> bool { + self.ty_or_const_infer_var_changed(var) } fn next_region_infer(&self) -> ty::Region<'tcx> { diff --git a/compiler/rustc_infer/src/infer/mod.rs b/compiler/rustc_infer/src/infer/mod.rs index aa2fe2ee21dfa..be232395ed3c6 100644 --- a/compiler/rustc_infer/src/infer/mod.rs +++ b/compiler/rustc_infer/src/infer/mod.rs @@ -29,14 +29,15 @@ use rustc_middle::traits::solve::Goal; use rustc_middle::ty::error::{ExpectedFound, TypeError}; use rustc_middle::ty::{ self, BoundVarReplacerDelegate, ConstVid, FloatVid, GenericArg, GenericArgKind, GenericArgs, - GenericArgsRef, GenericParamDefKind, InferConst, IntVid, OpaqueTypeKey, ProvisionalHiddenType, - PseudoCanonicalInput, Term, TermKind, Ty, TyCtxt, TyVid, TypeFoldable, TypeFolder, - TypeSuperFoldable, TypeVisitable, TypeVisitableExt, TypingEnv, TypingMode, fold_regions, + GenericArgsRef, GenericParamDefKind, InferConst, OpaqueTypeKey, ProvisionalHiddenType, + PseudoCanonicalInput, Term, Ty, TyCtxt, TyVid, TypeFoldable, TypeFolder, TypeSuperFoldable, + TypeVisitable, TypeVisitableExt, TypingEnv, TypingMode, fold_regions, }; use rustc_span::{DUMMY_SP, Span, Symbol}; use rustc_type_ir::MayBeErased; use snapshot::undo_log::InferCtxtUndoLogs; use tracing::{debug, instrument}; +use ty::solve::TyOrConstInferVar; use type_variable::TypeVariableOrigin; use crate::infer::snapshot::undo_log::UndoLog; @@ -1208,6 +1209,21 @@ impl<'tcx> InferCtxt<'tcx> { } } + /// If `TyVar(vid)` resolves to a type, return that type. Else, return the + /// universe index of `TyVar(vid)`, along with the root vid, which can be + /// cheaply found at the same time. + pub fn try_resolve_ty_var_with_root_vid( + &self, + vid: TyVid, + ) -> Result, (TyVid, ty::UniverseIndex)> { + use self::type_variable::TypeVariableValue; + let (root_vid, res) = self.inner.borrow_mut().type_variables().probe_with_root_vid(vid); + match res { + TypeVariableValue::Known { value } => Ok(value), + TypeVariableValue::Unknown { universe } => Err((root_vid, universe)), + } + } + pub fn shallow_resolve(&self, ty: Ty<'tcx>) -> Ty<'tcx> { if let ty::Infer(v) = *ty.kind() { match v { @@ -1561,14 +1577,13 @@ impl<'tcx> InferCtxt<'tcx> { #[inline(always)] pub fn ty_or_const_infer_var_changed(&self, infer_var: TyOrConstInferVar) -> bool { match infer_var { - TyOrConstInferVar::Ty(v) => { - use self::type_variable::TypeVariableValue; - - // If `inlined_probe` returns a `Known` value, it never equals - // `ty::Infer(ty::TyVar(v))`. - match self.inner.borrow_mut().type_variables().inlined_probe(v) { - TypeVariableValue::Unknown { .. } => false, - TypeVariableValue::Known { .. } => true, + TyOrConstInferVar::Ty(vid) => { + // If `try_resolve_ty_var` returns `Err`, *and* the `root_vid` is unchanged, + // it's definitely unchanged. The check against `root_vid` matters, see + // https://github.com/rust-lang/rust/issues/158441 + match self.try_resolve_ty_var_with_root_vid(vid) { + Ok(_) => true, + Err((root_vid, _)) => root_vid != vid, } } @@ -1610,64 +1625,6 @@ impl<'tcx> InferCtxt<'tcx> { } } -/// Helper for [InferCtxt::ty_or_const_infer_var_changed] (see comment on that), currently -/// used only for `traits::fulfill`'s list of `stalled_on` inference variables. -#[derive(Copy, Clone, Debug)] -pub enum TyOrConstInferVar { - /// Equivalent to `ty::Infer(ty::TyVar(_))`. - Ty(TyVid), - /// Equivalent to `ty::Infer(ty::IntVar(_))`. - TyInt(IntVid), - /// Equivalent to `ty::Infer(ty::FloatVar(_))`. - TyFloat(FloatVid), - - /// Equivalent to `ty::ConstKind::Infer(ty::InferConst::Var(_))`. - Const(ConstVid), -} - -impl<'tcx> TyOrConstInferVar { - /// Tries to extract an inference variable from a type or a constant, returns `None` - /// for types other than `ty::Infer(_)` (or `InferTy::Fresh*`) and - /// for constants other than `ty::ConstKind::Infer(_)` (or `InferConst::Fresh`). - pub fn maybe_from_generic_arg(arg: GenericArg<'tcx>) -> Option { - match arg.kind() { - GenericArgKind::Type(ty) => Self::maybe_from_ty(ty), - GenericArgKind::Const(ct) => Self::maybe_from_const(ct), - GenericArgKind::Lifetime(_) => None, - } - } - - /// Tries to extract an inference variable from a type or a constant, returns `None` - /// for types other than `ty::Infer(_)` (or `InferTy::Fresh*`) and - /// for constants other than `ty::ConstKind::Infer(_)` (or `InferConst::Fresh`). - pub fn maybe_from_term(term: Term<'tcx>) -> Option { - match term.kind() { - TermKind::Ty(ty) => Self::maybe_from_ty(ty), - TermKind::Const(ct) => Self::maybe_from_const(ct), - } - } - - /// Tries to extract an inference variable from a type, returns `None` - /// for types other than `ty::Infer(_)` (or `InferTy::Fresh*`). - fn maybe_from_ty(ty: Ty<'tcx>) -> Option { - match *ty.kind() { - ty::Infer(ty::TyVar(v)) => Some(TyOrConstInferVar::Ty(v)), - ty::Infer(ty::IntVar(v)) => Some(TyOrConstInferVar::TyInt(v)), - ty::Infer(ty::FloatVar(v)) => Some(TyOrConstInferVar::TyFloat(v)), - _ => None, - } - } - - /// Tries to extract an inference variable from a constant, returns `None` - /// for constants other than `ty::ConstKind::Infer(_)` (or `InferConst::Fresh`). - fn maybe_from_const(ct: ty::Const<'tcx>) -> Option { - match ct.kind() { - ty::ConstKind::Infer(InferConst::Var(v)) => Some(TyOrConstInferVar::Const(v)), - _ => None, - } - } -} - /// Replace `{integer}` with `i32` and `{float}` with `f64`. /// Used only for diagnostics. struct InferenceLiteralEraser<'tcx> { diff --git a/compiler/rustc_infer/src/infer/type_variable.rs b/compiler/rustc_infer/src/infer/type_variable.rs index 21ba07428d9ec..8c061f96f019b 100644 --- a/compiler/rustc_infer/src/infer/type_variable.rs +++ b/compiler/rustc_infer/src/infer/type_variable.rs @@ -267,6 +267,25 @@ impl<'tcx> TypeVariableTable<'_, 'tcx> { self.eq_relations().inlined_probe_value(vid) } + /// Retrieves the type to which `vid` has been instantiated, if + /// any, along with the root `vid`. + pub(crate) fn probe_with_root_vid( + &mut self, + vid: ty::TyVid, + ) -> (ty::TyVid, TypeVariableValue<'tcx>) { + self.inlined_probe_with_root_vid(vid) + } + + /// An always-inlined variant of `probe_with_root_vid`, for very hot call sites. + #[inline(always)] + pub(crate) fn inlined_probe_with_root_vid( + &mut self, + vid: ty::TyVid, + ) -> (ty::TyVid, TypeVariableValue<'tcx>) { + let (id, value) = self.eq_relations().inlined_probe_key_value(vid); + (id.vid, value) + } + #[inline] fn eq_relations(&mut self) -> super::UnificationTable<'_, 'tcx, TyVidEqKey<'tcx>> { self.storage.eq_relations.with_log(self.undo_log) diff --git a/compiler/rustc_middle/src/traits/solve.rs b/compiler/rustc_middle/src/traits/solve.rs index ceecb26d242dd..c45ebb80255cc 100644 --- a/compiler/rustc_middle/src/traits/solve.rs +++ b/compiler/rustc_middle/src/traits/solve.rs @@ -16,6 +16,10 @@ pub type CanonicalInput<'tcx, P = ty::Predicate<'tcx>> = ir::solve::CanonicalInp pub type CanonicalResponse<'tcx> = ir::solve::CanonicalResponse>; pub type FetchEligibleAssocItemResponse<'tcx> = ir::solve::FetchEligibleAssocItemResponse>; +pub type ComputeGoalFastPathOutcome<'tcx> = ir::solve::ComputeGoalFastPathOutcome>; +pub type GoalStalledOn<'tcx> = ir::solve::GoalStalledOn>; +pub type GoalStalledOnReason<'tcx> = ir::solve::GoalStalledOnReason>; +pub type SucceededInErased<'tcx> = ir::solve::SucceededInErased>; pub type PredefinedOpaques<'tcx> = &'tcx ty::List<(ty::OpaqueTypeKey<'tcx>, Ty<'tcx>)>; diff --git a/compiler/rustc_next_trait_solver/src/delegate.rs b/compiler/rustc_next_trait_solver/src/delegate.rs index 2ebb0fb851a4c..7b66667486fc9 100644 --- a/compiler/rustc_next_trait_solver/src/delegate.rs +++ b/compiler/rustc_next_trait_solver/src/delegate.rs @@ -1,7 +1,8 @@ use std::ops::Deref; use rustc_type_ir::solve::{ - Certainty, FetchEligibleAssocItemResponse, Goal, NoSolution, VisibleForLeakCheck, + Certainty, ComputeGoalFastPathOutcome, FetchEligibleAssocItemResponse, Goal, NoSolution, + VisibleForLeakCheck, }; use rustc_type_ir::{self as ty, InferCtxtLike, Interner, TypeFoldable}; @@ -23,7 +24,7 @@ pub trait SolverDelegate: Deref + Sized { &self, goal: Goal::Predicate>, span: ::Span, - ) -> Option; + ) -> ComputeGoalFastPathOutcome; fn fresh_var_for_kind_with_span( &self, diff --git a/compiler/rustc_next_trait_solver/src/solve/eval_ctxt/fast_path.rs b/compiler/rustc_next_trait_solver/src/solve/eval_ctxt/fast_path.rs new file mode 100644 index 0000000000000..c45b631400ac8 --- /dev/null +++ b/compiler/rustc_next_trait_solver/src/solve/eval_ctxt/fast_path.rs @@ -0,0 +1,151 @@ +//! This file contains a number of standalone functions useful for taking _fast paths_ in the trait +//! solver. The exact place where we check for these fast paths changes, and matters a lot for +//! performance. Ideally we'd only check them in `evaluate_goal`, but when evaluating root goals +//! we can check them earlier and save some time creating an `EvalCtxt` in the first place. +//! +//! For debugging, fast paths can be disabled using `-Zdisable-fast-paths`. + +use rustc_type_ir::inherent::*; +use rustc_type_ir::solve::{ + Certainty, ComputeGoalFastPathOutcome, Goal, GoalStalledOn, GoalStalledOnReason, + SucceededInErased, +}; +use rustc_type_ir::{InferCtxtLike, Interner}; + +use crate::delegate::SolverDelegate; +use crate::solve::eval_ctxt::{RerunDecision, should_rerun_after_erased_canonicalization}; +use crate::solve::{GoalEvaluation, HasChanged}; + +#[derive(Debug, Clone, Copy)] +pub(super) enum RerunStalled { + WontMakeProgress(Certainty), + MayMakeProgress, +} + +/// If we have run a goal before, and it was stalled, check that any of the goal's +/// args have changed. This is a cheap way to determine that if we were to rerun this goal now, +/// it will remain stalled since it'll canonicalize the same way and evaluation is pure. +/// Therefore, we can skip this rerun +#[inline] +pub(super) fn rerunning_stalled_goal_may_make_progress( + delegate: &D, + stalled_on: Option<&GoalStalledOn>, +) -> RerunStalled +where + D: SolverDelegate, + I: Interner, +{ + use RerunStalled::*; + + // If fast paths are turned off, then we assume all goals can always make progress + if delegate.disable_trait_solver_fast_paths() { + return MayMakeProgress; + } + + // If the goal isn't stalled, we should definitely run it. + let Some(&GoalStalledOn { ref reason, ref stalled_vars, ref sub_roots, stalled_certainty }) = + stalled_on + else { + return MayMakeProgress; + }; + + // If any of the stalled goal's generic arguments changed, + // rerunning might make progress so we should rerun. + if stalled_vars.iter().any(|value| delegate.ty_or_const_infer_var_changed(*value)) { + return MayMakeProgress; + } + + // If some inference took place in any of the sub roots, + // rerunning might make progress so we should rerun. + if sub_roots.iter().any(|&vid| delegate.sub_unification_table_root_var(vid) != vid) { + return MayMakeProgress; + } + + match reason { + GoalStalledOnReason::FastPath => { + // fastpath is never because of opaques, we can skip this check + } + &GoalStalledOnReason::Other { num_opaques, ref previously_succeeded_in_erased } => { + // If any opaques changed in the opaque type storage, + // rerunning might make progress so we should rerun. + if delegate.opaque_types_storage_num_entries().needs_reevaluation(num_opaques) { + // Unless this goal previously succeeded in erased mode. + // If the stalled goal successfully evaluated while erasing opaque types, + // and the current state of the opaque type storage is not different in a way that is + // relevant, this stalled goal cannot make any progress and we set this variable to true. + let mut previous_erased_run_is_still_valid = false; + + if let &SucceededInErased::Yes { accessed_opaques } = previously_succeeded_in_erased + { + match should_rerun_after_erased_canonicalization( + accessed_opaques, + delegate.typing_mode_raw(), + &delegate.clone_opaque_types_lookup_table(), + ) { + RerunDecision::Yes => {} + RerunDecision::EagerlyPropagateToParent => { + unreachable!("we never retry stalled queries if the parent was erased") + } + RerunDecision::No => { + previous_erased_run_is_still_valid = true; + } + } + } + + if !previous_erased_run_is_still_valid { + return MayMakeProgress; + } + } + } + } + + // Otherwise, we can be sure that this stalled goal cannot make any progress + // and we can exit early. + WontMakeProgress(stalled_certainty) +} + +#[cold] +#[inline(never)] +pub(super) fn compute_goal_fast_path_cold( + delegate: &D, + goal: Goal, + origin_span: I::Span, +) -> Option> +where + D: SolverDelegate, + I: Interner, +{ + compute_goal_fast_path(delegate, goal, origin_span) +} + +/// This is a fast path optimization: +/// See the docs on [`ComputeGoalFastPathOutcome`] +pub fn compute_goal_fast_path( + delegate: &D, + goal: Goal, + origin_span: I::Span, +) -> Option> +where + D: SolverDelegate, + I: Interner, +{ + if delegate.disable_trait_solver_fast_paths() { + return None; + } + + match delegate.compute_goal_fast_path(goal, origin_span) { + ComputeGoalFastPathOutcome::NoFastPath => None, + ComputeGoalFastPathOutcome::TriviallyHolds => Some(GoalEvaluation { + goal, + certainty: Certainty::Yes, + has_changed: HasChanged::No, + stalled_on: None, + }), + ComputeGoalFastPathOutcome::TriviallyStalled { stalled_on } => Some(GoalEvaluation { + goal, + certainty: Certainty::AMBIGUOUS, + has_changed: HasChanged::No, + stalled_on: Some(stalled_on), + }), + } +} diff --git a/compiler/rustc_next_trait_solver/src/solve/eval_ctxt/mod.rs b/compiler/rustc_next_trait_solver/src/solve/eval_ctxt/mod.rs index d83320f5fe800..277cae9f239c7 100644 --- a/compiler/rustc_next_trait_solver/src/solve/eval_ctxt/mod.rs +++ b/compiler/rustc_next_trait_solver/src/solve/eval_ctxt/mod.rs @@ -12,7 +12,7 @@ use rustc_type_ir::search_graph::{CandidateHeadUsages, LowerAvailableDepth, Path use rustc_type_ir::solve::{ AccessedOpaques, ExternalRegionConstraints, FetchEligibleAssocItemResponse, MaybeInfo, NoSolutionOrRerunNonErased, OpaqueTypesJank, QueryResultOrRerunNonErased, RerunCondition, - RerunNonErased, RerunReason, RerunResultExt, SmallCopyList, + RerunNonErased, RerunReason, RerunResultExt, SmallCopyList, TyOrConstInferVar, }; use rustc_type_ir::{ self as ty, CanonicalVarValues, ClauseKind, InferCtxtLike, Interner, MayBeErased, @@ -31,15 +31,20 @@ use crate::delegate::SolverDelegate; use crate::normalize::{NormalizationFolder, NormalizationWasAmbiguous}; use crate::placeholder::BoundVarReplacer; use crate::resolve::eager_resolve_vars; +use crate::solve::eval_ctxt::fast_path::{ + RerunStalled, compute_goal_fast_path, rerunning_stalled_goal_may_make_progress, +}; +use crate::solve::fast_path::compute_goal_fast_path_cold; use crate::solve::search_graph::SearchGraph; use crate::solve::ty::may_use_unstable_feature; use crate::solve::{ CanonicalInput, CanonicalResponse, Certainty, ExternalConstraintsData, FIXPOINT_STEP_LIMIT, - Goal, GoalEvaluation, GoalSource, GoalStalledOn, HasChanged, MaybeCause, + Goal, GoalEvaluation, GoalSource, GoalStalledOn, GoalStalledOnReason, HasChanged, MaybeCause, NestedNormalizationGoals, NoSolution, QueryInput, QueryResult, Response, SucceededInErased, VisibleForLeakCheck, inspect, }; +pub mod fast_path; mod probe; mod solver_region_constraints; @@ -89,12 +94,6 @@ impl CurrentGoalKind { } } -#[derive(Debug)] -enum RerunDecision { - Yes, - No, - EagerlyPropagateToParent, -} pub struct EvalCtxt<'a, D, I = ::Interner> where D: SolverDelegate, @@ -226,8 +225,31 @@ where span: I::Span, stalled_on: Option>, ) -> Result, NoSolution> { + // Run fast paths *before* building an `EvalCtxt`, saving a little bit of time. + if let RerunStalled::WontMakeProgress(stalled_certainty) = + rerunning_stalled_goal_may_make_progress(self, stalled_on.as_ref()) + { + return Ok(GoalEvaluation { + goal, + certainty: stalled_certainty, + has_changed: HasChanged::No, + stalled_on, + }); + } + + if + // No need to try the fast path if stalled_on is `None`, since we already try the fast path + // immediately when adding new goals. If we didn't check `stalled_on` here we'd be trying + // the fast path twice for some goals. + stalled_on.is_some() + && let Some(res) = compute_goal_fast_path_cold(self, goal, span) + { + return Ok(res); + } + let result = EvalCtxt::enter_root(self, self.cx().recursion_limit(), span, |ecx| { - ecx.evaluate_goal(GoalSource::Misc, goal, stalled_on) + // Fast paths handled above + ecx.evaluate_goal_no_fast_paths(GoalSource::Misc, goal) }); match result { @@ -285,12 +307,6 @@ where } } -#[derive(Debug, Clone, Copy)] -enum RerunStalled { - WontMakeProgress(Certainty), - MayMakeProgress, -} - impl<'a, D, I> EvalCtxt<'a, D> where D: SolverDelegate, @@ -483,85 +499,42 @@ where goal: Goal, stalled_on: Option>, ) -> Result, NoSolutionOrRerunNonErased> { - let (normalization_nested_goals, goal_evaluation) = - self.evaluate_goal_raw(source, goal, stalled_on, LowerAvailableDepth::Yes)?; - assert!(normalization_nested_goals.is_empty()); - Ok(goal_evaluation) - } - - /// This is a fast path optimization: - /// If we have run this goal before, and it was stalled, check that any of the goal's - /// args have changed. This is a cheap way to determine that if we were to rerun this goal now, - /// it will remain stalled since it'll canonicalize the same way and evaluation is pure. - /// Therefore, we can skip this rerun - fn rerunning_stalled_goal_may_make_progress( - &self, - stalled_on: Option<&GoalStalledOn>, - ) -> RerunStalled { - use RerunStalled::*; - - // If fast paths are turned off, then we assume all goals can always make progress - if self.delegate.disable_trait_solver_fast_paths() { - return MayMakeProgress; - } - - // If the goal isn't stalled, we should definitely run it. - let Some(&GoalStalledOn { - num_opaques, - ref stalled_vars, - ref sub_roots, - stalled_certainty, - ref previously_succeeded_in_erased, - }) = stalled_on - else { - return MayMakeProgress; - }; - - // If any of the stalled goal's generic arguments changed, - // rerunning might make progress so we should rerun. - if stalled_vars.iter().any(|value| self.delegate.is_changed_arg(*value)) { - return MayMakeProgress; + if let RerunStalled::WontMakeProgress(stalled_certainty) = + rerunning_stalled_goal_may_make_progress(self.delegate, stalled_on.as_ref()) + { + return Ok(GoalEvaluation { + goal, + certainty: stalled_certainty, + has_changed: HasChanged::No, + stalled_on, + }); } - // If some inference took place in any of the sub roots, - // rerunning might make progress so we should rerun. - if sub_roots.iter().any(|&vid| self.delegate.sub_unification_table_root_var(vid) != vid) { - return MayMakeProgress; + if + // No need to try the fast path if stalled_on is `None`, since we already try the fast path + // immediately when adding new goals. If we didn't check `stalled_on` here we'd be trying + // the fast path twice for some goals. + stalled_on.is_some() + && let Some(res) = compute_goal_fast_path_cold(self.delegate, goal, self.origin_span) + { + return Ok(res); } - // If any opaques changed in the opaque type storage, - // rerunning might make progress so we should rerun. - if self.delegate.opaque_types_storage_num_entries().needs_reevaluation(num_opaques) { - // Unless this goal previously succeeded in erased mode. - // If the stalled goal successfully evaluated while erasing opaque types, - // and the current state of the opaque type storage is not different in a way that is - // relevant, this stalled goal cannot make any progress and we set this variable to true. - let mut previous_erased_run_is_still_valid = false; - - if let &SucceededInErased::Yes { accessed_opaques } = previously_succeeded_in_erased { - match self.should_rerun_after_erased_canonicalization( - accessed_opaques, - self.typing_mode(), - &self.delegate.clone_opaque_types_lookup_table(), - ) { - RerunDecision::Yes => {} - RerunDecision::EagerlyPropagateToParent => { - unreachable!("we never retry stalled queries if the parent was erased") - } - RerunDecision::No => { - previous_erased_run_is_still_valid = true; - } - } - } - - if !previous_erased_run_is_still_valid { - return MayMakeProgress; - } - } + self.evaluate_goal_no_fast_paths(source, goal) + } - // Otherwise, we can be sure that this stalled goal cannot make any progress - // and we can exit early. - WontMakeProgress(stalled_certainty) + // Outlining and `#[cold]` matter here because fast paths make it less likely to get here. + #[cold] + #[inline(never)] + fn evaluate_goal_no_fast_paths( + &mut self, + source: GoalSource, + goal: Goal, + ) -> Result, NoSolutionOrRerunNonErased> { + let (normalization_nested_goals, goal_evaluation) = + self.evaluate_goal_raw(source, goal, LowerAvailableDepth::Yes)?; + assert!(normalization_nested_goals.is_empty()); + Ok(goal_evaluation) } /// Recursively evaluates `goal`, returning the nested goals in case @@ -575,7 +548,6 @@ where &mut self, source: GoalSource, goal: Goal, - stalled_on: Option>, increase_depth_for_nested: LowerAvailableDepth, ) -> Result<(NestedNormalizationGoals, GoalEvaluation), NoSolutionOrRerunNonErased> { if let RerunStalled::WontMakeProgress(stalled_certainty) = @@ -669,7 +641,7 @@ where &mut inspect::ProofTreeBuilder::new_noop(), ); - let should_rerun = self.should_rerun_after_erased_canonicalization( + let should_rerun = should_rerun_after_erased_canonicalization( accessed_opaques, self.typing_mode(), &opaque_types, @@ -759,41 +731,12 @@ where // that is not resolved. Only when *these* have changed is it meaningful // to recompute this goal. HasChanged::Yes => None, - HasChanged::No => { - // Remove the canonicalized universal vars, since we only care about stalled existentials. - let mut sub_roots = Vec::new(); - let mut stalled_vars = orig_values; - stalled_vars.retain(|arg| match arg.kind() { - // Lifetimes can never stall goals. - ty::GenericArgKind::Lifetime(_) => false, - ty::GenericArgKind::Type(ty) => match ty.kind() { - ty::Infer(ty::TyVar(vid)) => { - sub_roots.push(self.delegate.sub_unification_table_root_var(vid)); - true - } - ty::Infer(_) => true, - ty::Param(_) | ty::Placeholder(_) => false, - _ => unreachable!("unexpected orig_value: {ty:?}"), - }, - ty::GenericArgKind::Const(ct) => match ct.kind() { - ty::ConstKind::Infer(_) => true, - ty::ConstKind::Param(_) | ty::ConstKind::Placeholder(_) => false, - _ => unreachable!("unexpected orig_value: {ct:?}"), - }, - }); - - Some(GoalStalledOn { - num_opaques: canonical_goal - .canonical - .value - .predefined_opaques_in_body - .len(), - stalled_vars, - sub_roots, - stalled_certainty: certainty, - previously_succeeded_in_erased: succeeded_in_erased, - }) - } + HasChanged::No => Some(self.build_stalled_on( + canonical_goal, + certainty, + orig_values, + succeeded_in_erased, + )), }, }; @@ -803,98 +746,49 @@ where )) } - fn should_rerun_after_erased_canonicalization( + fn build_stalled_on( &self, - AccessedOpaques { reason: _, rerun }: AccessedOpaques, - original_typing_mode: TypingMode, - parent_opaque_types: &[(OpaqueTypeKey, I::Ty)], - ) -> RerunDecision { - let parent_opaque_defids = parent_opaque_types.iter().map(|(key, _)| key.def_id.into()); - let opaque_in_storage = |opaques: I::LocalDefIds, defids: SmallCopyList<_>| { - if defids.as_ref().is_empty() { - RerunDecision::No - } else if opaques - .iter() - .chain(parent_opaque_defids) - .any(|opaque| defids.as_ref().contains(&opaque)) - { - RerunDecision::Yes - } else { - RerunDecision::No - } - }; - let any_opaque_has_infer_as_hidden = || { - if parent_opaque_types.iter().any(|(_, ty)| ty.is_ty_var()) { - RerunDecision::Yes - } else { - RerunDecision::No - } - }; - - let res = match (rerun, original_typing_mode) { - // ============================= - (RerunCondition::Never, _) => RerunDecision::No, - // ============================= - (_, TypingMode::ErasedNotCoherence(MayBeErased)) => { - RerunDecision::EagerlyPropagateToParent - } - // ============================= - // In coherence, we never switch to erased mode, so we will never register anything - // in the rerun state, so we should've taken the first branch of this match - (_, TypingMode::Coherence) => unreachable!(), - // ============================= - (RerunCondition::Always, _) => RerunDecision::Yes, - // ============================= - ( - RerunCondition::OpaqueInStorage(..), - TypingMode::PostAnalysis | TypingMode::Codegen, - ) => RerunDecision::Yes, - ( - RerunCondition::OpaqueInStorage(defids), - TypingMode::PostBorrowck { defined_opaque_types: opaques } - | TypingMode::Typeck { defining_opaque_types_and_generators: opaques } - | TypingMode::PostTypeckUntilBorrowck { defining_opaque_types: opaques }, - ) => opaque_in_storage(opaques, defids), - // ============================= - (RerunCondition::AnyOpaqueHasInferAsHidden, TypingMode::Typeck { .. }) => { - any_opaque_has_infer_as_hidden() - } - ( - RerunCondition::AnyOpaqueHasInferAsHidden, - TypingMode::PostBorrowck { .. } - | TypingMode::PostAnalysis - | TypingMode::Codegen - | TypingMode::PostTypeckUntilBorrowck { .. }, - ) => RerunDecision::No, - // ============================= - ( - RerunCondition::OpaqueInStorageOrAnyOpaqueHasInferAsHidden(_), - TypingMode::PostAnalysis | TypingMode::Codegen, - ) => RerunDecision::No, - ( - RerunCondition::OpaqueInStorageOrAnyOpaqueHasInferAsHidden(defids), - TypingMode::Typeck { defining_opaque_types_and_generators: opaques }, - ) => { - if let RerunDecision::Yes = any_opaque_has_infer_as_hidden() { - RerunDecision::Yes - } else if let RerunDecision::Yes = opaque_in_storage(opaques, defids) { - RerunDecision::Yes - } else { - RerunDecision::No - } - } - ( - RerunCondition::OpaqueInStorageOrAnyOpaqueHasInferAsHidden(defids), - TypingMode::PostBorrowck { defined_opaque_types: opaques } - | TypingMode::PostTypeckUntilBorrowck { defining_opaque_types: opaques }, - ) => opaque_in_storage(opaques, defids), - }; - - debug!( - "checking whether to rerun {rerun:?} in outer typing mode {original_typing_mode:?} and opaques {parent_opaque_types:?}: {res:?}" - ); - - res + canonical_goal: CanonicalInput, + certainty: Certainty, + orig_values: Vec, + previously_succeeded_in_erased: SucceededInErased, + ) -> GoalStalledOn { + // Remove the canonicalized universal vars, since we only care about stalled existentials. + let mut sub_roots = Vec::new(); + let stalled_vars = orig_values + .into_iter() + .filter_map(|arg| match arg.kind() { + // Lifetimes can never stall goals. + ty::GenericArgKind::Lifetime(_) => None, + ty::GenericArgKind::Type(ty) => match ty.kind() { + ty::Infer(ty::TyVar(vid)) => { + sub_roots.push(self.delegate.sub_unification_table_root_var(vid)); + Some(TyOrConstInferVar::Ty(vid)) + } + ty::Infer(ty::IntVar(vid)) => Some(TyOrConstInferVar::TyInt(vid)), + ty::Infer(ty::FloatVar(vid)) => Some(TyOrConstInferVar::TyFloat(vid)), + ty::Param(_) | ty::Placeholder(_) => None, + _ => unreachable!("unexpected orig_value: {ty:?}"), + }, + ty::GenericArgKind::Const(ct) => match ct.kind() { + ty::ConstKind::Infer(ty::InferConst::Var(v)) => { + Some(TyOrConstInferVar::Const(v)) + } + ty::ConstKind::Param(_) | ty::ConstKind::Placeholder(_) => None, + _ => unreachable!("unexpected orig_value: {ct:?}"), + }, + }) + .collect(); + + GoalStalledOn { + stalled_vars, + sub_roots, + stalled_certainty: certainty, + reason: GoalStalledOnReason::Other { + num_opaques: canonical_goal.canonical.value.predefined_opaques_in_body.len(), + previously_succeeded_in_erased, + }, + } } pub(super) fn compute_goal( @@ -983,6 +877,9 @@ where ) -> Result, NoSolutionOrRerunNonErased> { // If this loop did not result in any progress, what's our final certainty. let mut unchanged_certainty = Some(Certainty::Yes); + // This mem::take seems super inefficient, given that we push to it again later. + // Despite that, replacing it has no effect on performance. We tried. + // (https://github.com/rust-lang/rust/pull/158126) for (source, goal, stalled_on) in mem::take(&mut self.nested_goals) { // We never handle `NormalizesTo` as a nested goal debug_assert!(!matches!( @@ -990,20 +887,6 @@ where PredicateKind::NormalizesTo(_) )); - if !self.delegate.disable_trait_solver_fast_paths() - && let Some(certainty) = - self.delegate.compute_goal_fast_path(goal, self.origin_span) - { - match certainty { - Certainty::Yes => {} - Certainty::Maybe { .. } => { - self.nested_goals.push((source, goal, None)); - unchanged_certainty = unchanged_certainty.map(|c| c.and(certainty)); - } - } - continue; - } - let GoalEvaluation { goal, certainty, has_changed, stalled_on } = self.evaluate_goal(source, goal, stalled_on)?; if has_changed == HasChanged::Yes { @@ -1043,7 +926,20 @@ where ty::Unnormalized::new_wip(goal.predicate), )?; self.inspect.add_goal(self.delegate, self.max_input_universe, source, goal); - self.nested_goals.push((source, goal, None)); + + if let Some(GoalEvaluation { goal, certainty, has_changed: _, stalled_on }) = + compute_goal_fast_path(self.delegate, goal, self.origin_span) + { + match certainty { + // We're done here + Certainty::Yes => {} + Certainty::Maybe(_) => { + self.nested_goals.push((source, goal, stalled_on)); + } + } + } else { + self.nested_goals.push((source, goal, None)); + } Ok(()) } @@ -1735,6 +1631,103 @@ where } } +#[derive(Debug)] +enum RerunDecision { + Yes, + No, + EagerlyPropagateToParent, +} + +fn should_rerun_after_erased_canonicalization( + AccessedOpaques { reason: _, rerun }: AccessedOpaques, + original_typing_mode: TypingMode, + parent_opaque_types: &[(OpaqueTypeKey, I::Ty)], +) -> RerunDecision { + let parent_opaque_defids = parent_opaque_types.iter().map(|(key, _)| key.def_id.into()); + let opaque_in_storage = |opaques: I::LocalDefIds, defids: SmallCopyList<_>| { + if defids.as_ref().is_empty() { + RerunDecision::No + } else if opaques + .iter() + .chain(parent_opaque_defids) + .any(|opaque| defids.as_ref().contains(&opaque)) + { + RerunDecision::Yes + } else { + RerunDecision::No + } + }; + let any_opaque_has_infer_as_hidden = || { + if parent_opaque_types.iter().any(|(_, ty)| ty.is_ty_var()) { + RerunDecision::Yes + } else { + RerunDecision::No + } + }; + + let res = match (rerun, original_typing_mode) { + // ============================= + (RerunCondition::Never, _) => RerunDecision::No, + // ============================= + (_, TypingMode::ErasedNotCoherence(MayBeErased)) => RerunDecision::EagerlyPropagateToParent, + // ============================= + // In coherence, we never switch to erased mode, so we will never register anything + // in the rerun state, so we should've taken the first branch of this match + (_, TypingMode::Coherence) => unreachable!(), + // ============================= + (RerunCondition::Always, _) => RerunDecision::Yes, + // ============================= + (RerunCondition::OpaqueInStorage(..), TypingMode::PostAnalysis | TypingMode::Codegen) => { + RerunDecision::Yes + } + ( + RerunCondition::OpaqueInStorage(defids), + TypingMode::PostBorrowck { defined_opaque_types: opaques } + | TypingMode::Typeck { defining_opaque_types_and_generators: opaques } + | TypingMode::PostTypeckUntilBorrowck { defining_opaque_types: opaques }, + ) => opaque_in_storage(opaques, defids), + // ============================= + (RerunCondition::AnyOpaqueHasInferAsHidden, TypingMode::Typeck { .. }) => { + any_opaque_has_infer_as_hidden() + } + ( + RerunCondition::AnyOpaqueHasInferAsHidden, + TypingMode::PostBorrowck { .. } + | TypingMode::PostAnalysis + | TypingMode::Codegen + | TypingMode::PostTypeckUntilBorrowck { .. }, + ) => RerunDecision::No, + // ============================= + ( + RerunCondition::OpaqueInStorageOrAnyOpaqueHasInferAsHidden(_), + TypingMode::PostAnalysis | TypingMode::Codegen, + ) => RerunDecision::No, + ( + RerunCondition::OpaqueInStorageOrAnyOpaqueHasInferAsHidden(defids), + TypingMode::Typeck { defining_opaque_types_and_generators: opaques }, + ) => { + if let RerunDecision::Yes = any_opaque_has_infer_as_hidden() { + RerunDecision::Yes + } else if let RerunDecision::Yes = opaque_in_storage(opaques, defids) { + RerunDecision::Yes + } else { + RerunDecision::No + } + } + ( + RerunCondition::OpaqueInStorageOrAnyOpaqueHasInferAsHidden(defids), + TypingMode::PostBorrowck { defined_opaque_types: opaques } + | TypingMode::PostTypeckUntilBorrowck { defining_opaque_types: opaques }, + ) => opaque_in_storage(opaques, defids), + }; + + debug!( + "checking whether to rerun {rerun:?} in outer typing mode {original_typing_mode:?} and opaques {parent_opaque_types:?}: {res:?}" + ); + + res +} + /// Do not call this directly, use the `tcx` query instead. pub fn evaluate_root_goal_for_proof_tree_raw_provider< D: SolverDelegate, diff --git a/compiler/rustc_next_trait_solver/src/solve/mod.rs b/compiler/rustc_next_trait_solver/src/solve/mod.rs index fcf8ddf59fe3c..452a87a89a7f1 100644 --- a/compiler/rustc_next_trait_solver/src/solve/mod.rs +++ b/compiler/rustc_next_trait_solver/src/solve/mod.rs @@ -23,12 +23,12 @@ mod trait_goals; use derive_where::derive_where; use rustc_type_ir::inherent::*; pub use rustc_type_ir::solve::*; -use rustc_type_ir::{self as ty, Interner, TyVid}; +use rustc_type_ir::{self as ty, Interner}; use tracing::instrument; pub use self::eval_ctxt::{ EvalCtxt, GenerateProofTree, SolverDelegateEvalExt, - evaluate_root_goal_for_proof_tree_raw_provider, + evaluate_root_goal_for_proof_tree_raw_provider, fast_path, }; use crate::delegate::SolverDelegate; use crate::solve::assembly::Candidate; @@ -424,21 +424,3 @@ pub struct GoalEvaluation { /// before rerunning it. pub stalled_on: Option>, } - -/// The conditions that must change for a goal to warrant -#[derive_where(Clone, Debug; I: Interner)] -pub struct GoalStalledOn { - pub num_opaques: usize, - pub stalled_vars: Vec, - pub sub_roots: Vec, - /// The certainty that will be returned on subsequent evaluations if this - /// goal remains stalled. - pub stalled_certainty: Certainty, - pub previously_succeeded_in_erased: SucceededInErased, -} - -#[derive_where(Clone, Debug; I: Interner)] -pub enum SucceededInErased { - Yes { accessed_opaques: AccessedOpaques }, - No, -} diff --git a/compiler/rustc_next_trait_solver/src/solve/project_goals/mod.rs b/compiler/rustc_next_trait_solver/src/solve/project_goals/mod.rs index 52ef40daf40da..41fe4105cb745 100644 --- a/compiler/rustc_next_trait_solver/src/solve/project_goals/mod.rs +++ b/compiler/rustc_next_trait_solver/src/solve/project_goals/mod.rs @@ -71,7 +71,6 @@ where ) = self.evaluate_goal_raw( GoalSource::TypeRelating, normalizes_to, - None, // We don't lower thr available depth for this `NormalizesTo` goal, as evaluating // it is an extra step only exists in the new solver that behaves like a function // call rather than an independent nested goal evaluation. So, decreasing the diff --git a/compiler/rustc_trait_selection/src/error_reporting/infer/need_type_info.rs b/compiler/rustc_trait_selection/src/error_reporting/infer/need_type_info.rs index 624aef89911f7..d76369b068e6d 100644 --- a/compiler/rustc_trait_selection/src/error_reporting/infer/need_type_info.rs +++ b/compiler/rustc_trait_selection/src/error_reporting/infer/need_type_info.rs @@ -19,6 +19,7 @@ use rustc_middle::ty::{ IsSuggestable, Term, TermKind, Ty, TyCtxt, TypeFoldable, TypeFolder, TypeSuperFoldable, TypeVisitableExt, TypeckResults, }; +use rustc_next_trait_solver::solve::TyOrConstInferVar; use rustc_span::{BytePos, DUMMY_SP, Ident, Span, sym}; use tracing::{debug, instrument, warn}; @@ -28,7 +29,7 @@ use crate::diagnostics::{ SourceKindMultiSuggestion, SourceKindSubdiag, }; use crate::error_reporting::TypeErrCtxt; -use crate::infer::{InferCtxt, TyOrConstInferVar}; +use crate::infer::InferCtxt; pub enum TypeAnnotationNeeded { /// ```compile_fail,E0282 @@ -94,7 +95,7 @@ impl InferenceDiagnosticsData { } else { match displayed_ty .walk() - .filter_map(TyOrConstInferVar::maybe_from_generic_arg) + .filter_map(TyOrConstInferVar::maybe_from_generic_arg::>) .take(2) .count() { diff --git a/compiler/rustc_trait_selection/src/solve/delegate.rs b/compiler/rustc_trait_selection/src/solve/delegate.rs index e0877829b25f3..b994134beeffa 100644 --- a/compiler/rustc_trait_selection/src/solve/delegate.rs +++ b/compiler/rustc_trait_selection/src/solve/delegate.rs @@ -10,12 +10,15 @@ use rustc_infer::infer::canonical::{ QueryRegionConstraint, }; use rustc_infer::infer::{InferCtxt, RegionVariableOrigin, SubregionOrigin, TyCtxtInferExt}; -use rustc_infer::traits::solve::{FetchEligibleAssocItemResponse, Goal}; +use rustc_infer::traits::solve::{ + ComputeGoalFastPathOutcome, FetchEligibleAssocItemResponse, Goal, +}; use rustc_middle::traits::query::NoSolution; use rustc_middle::traits::solve::Certainty; use rustc_middle::ty::{ self, MayBeErased, Ty, TyCtxt, TypeFlags, TypeFoldable, TypeVisitableExt, TypingMode, }; +use rustc_next_trait_solver::solve::{GoalStalledOn, GoalStalledOnReason, TyOrConstInferVar}; use rustc_span::{DUMMY_SP, Span}; use crate::traits::{EvaluateConstErr, ObligationCause, sizedness_fast_path, specialization_graph}; @@ -73,10 +76,25 @@ impl<'tcx> rustc_next_trait_solver::delegate::SolverDelegate for SolverDelegate< &self, goal: Goal<'tcx, ty::Predicate<'tcx>>, span: Span, - ) -> Option { + ) -> ComputeGoalFastPathOutcome<'tcx> { + use ComputeGoalFastPathOutcome as Outcome; + + fn stalled_with_args<'tcx>( + stalled_vars: Vec, + ) -> ComputeGoalFastPathOutcome<'tcx> { + Outcome::TriviallyStalled { + stalled_on: GoalStalledOn { + stalled_vars, + sub_roots: Vec::new(), + stalled_certainty: Certainty::AMBIGUOUS, + reason: GoalStalledOnReason::FastPath, + }, + } + } + // FIXME(-Zassumptions-on-binders): actually handle fast path if self.tcx.assumptions_on_binders() { - return None; + return Outcome::NoFastPath; } let pred = goal.predicate.kind(); @@ -84,22 +102,23 @@ impl<'tcx> rustc_next_trait_solver::delegate::SolverDelegate for SolverDelegate< ty::PredicateKind::Clause(ty::ClauseKind::Trait(trait_pred)) => { let trait_pred = pred.rebind(trait_pred); - if self.shallow_resolve(trait_pred.self_ty().skip_binder()).is_ty_var() + let self_ty = self.shallow_resolve(trait_pred.self_ty().skip_binder()); + if let Some(vid) = self_ty.ty_vid() // We don't do this fast path when opaques are defined since we may // eventually use opaques to incompletely guide inference via ty var // self types. // FIXME: Properly consider opaques here. && self.known_no_opaque_types_in_storage() { - Some(Certainty::AMBIGUOUS) + stalled_with_args(vec![TyOrConstInferVar::Ty(vid)]) } else if trait_pred.polarity() == ty::PredicatePolarity::Positive { match self.0.tcx.as_lang_item(trait_pred.def_id()) { Some(LangItem::Sized) | Some(LangItem::MetaSized) => { let predicate = self.resolve_vars_if_possible(goal.predicate); if sizedness_fast_path(self.tcx, predicate, goal.param_env) { - return Some(Certainty::Yes); + return Outcome::TriviallyHolds; } else { - None + Outcome::NoFastPath } } Some(LangItem::Copy | LangItem::Clone) => { @@ -114,23 +133,23 @@ impl<'tcx> rustc_next_trait_solver::delegate::SolverDelegate for SolverDelegate< .has_type_flags(TypeFlags::HAS_FREE_REGIONS | TypeFlags::HAS_INFER) && self_ty.is_trivially_pure_clone_copy() { - return Some(Certainty::Yes); + return Outcome::TriviallyHolds; } else { - None + Outcome::NoFastPath } } - _ => None, + _ => Outcome::NoFastPath, } } else { - None + Outcome::NoFastPath } } ty::PredicateKind::DynCompatible(def_id) if self.0.tcx.is_dyn_compatible(def_id) => { - Some(Certainty::Yes) + Outcome::TriviallyHolds } ty::PredicateKind::Clause(ty::ClauseKind::RegionOutlives(outlives)) => { if outlives.has_escaping_bound_vars() { - return None; + return Outcome::NoFastPath; } self.0.sub_regions( @@ -139,11 +158,11 @@ impl<'tcx> rustc_next_trait_solver::delegate::SolverDelegate for SolverDelegate< outlives.0, ty::VisibleForLeakCheck::Yes, ); - Some(Certainty::Yes) + Outcome::TriviallyHolds } ty::PredicateKind::Clause(ty::ClauseKind::TypeOutlives(outlives)) => { if outlives.has_escaping_bound_vars() { - return None; + return Outcome::NoFastPath; } self.0.register_type_outlives_constraint( @@ -152,48 +171,55 @@ impl<'tcx> rustc_next_trait_solver::delegate::SolverDelegate for SolverDelegate< &ObligationCause::dummy_with_span(span), ); - Some(Certainty::Yes) + Outcome::TriviallyHolds } ty::PredicateKind::Subtype(ty::SubtypePredicate { a, b, .. }) | ty::PredicateKind::Coerce(ty::CoercePredicate { a, b }) => { if a.has_escaping_bound_vars() || b.has_escaping_bound_vars() { - return None; + return Outcome::NoFastPath; } match (self.shallow_resolve(a).kind(), self.shallow_resolve(b).kind()) { (&ty::Infer(ty::TyVar(a_vid)), &ty::Infer(ty::TyVar(b_vid))) => { self.sub_unify_ty_vids_raw(a_vid, b_vid); - Some(Certainty::AMBIGUOUS) + stalled_with_args(vec![ + TyOrConstInferVar::Ty(a_vid), + TyOrConstInferVar::Ty(b_vid), + ]) } - _ => None, + _ => Outcome::NoFastPath, } } ty::PredicateKind::Clause(ty::ClauseKind::ConstArgHasType(ct, _)) => { if ct.has_escaping_bound_vars() { - return None; + return Outcome::NoFastPath; } - if self.shallow_resolve_const(ct).is_ct_infer() { - Some(Certainty::AMBIGUOUS) + let arg = self.shallow_resolve_const(ct); + if let Some(vid) = arg.ct_vid() { + stalled_with_args(vec![TyOrConstInferVar::Const(vid)]) } else { - None + Outcome::NoFastPath } } ty::PredicateKind::Clause(ty::ClauseKind::WellFormed(arg)) => { if arg.has_escaping_bound_vars() { - return None; + return Outcome::NoFastPath; } let arg = self.shallow_resolve_term(arg); if arg.is_trivially_wf(self.tcx) { - Some(Certainty::Yes) + Outcome::TriviallyHolds } else if arg.is_infer() { - Some(Certainty::AMBIGUOUS) + stalled_with_args(vec![ + TyOrConstInferVar::maybe_from_term::>(arg) + .expect("its an infer var"), + ]) } else { - None + Outcome::NoFastPath } } - _ => None, + _ => Outcome::NoFastPath, } } diff --git a/compiler/rustc_trait_selection/src/solve/fulfill.rs b/compiler/rustc_trait_selection/src/solve/fulfill.rs index 4b80d22d6bb49..e142046041aeb 100644 --- a/compiler/rustc_trait_selection/src/solve/fulfill.rs +++ b/compiler/rustc_trait_selection/src/solve/fulfill.rs @@ -7,7 +7,7 @@ use rustc_infer::traits::{ FromSolverError, PredicateObligation, PredicateObligations, TraitEngine, }; use rustc_middle::ty::{self, TyCtxt, TyVid, TypeVisitableExt, TypingMode}; -use rustc_next_trait_solver::delegate::SolverDelegate as _; +use rustc_next_trait_solver::solve::fast_path::compute_goal_fast_path; use rustc_next_trait_solver::solve::{ GoalEvaluation, GoalStalledOn, HasChanged, MaybeInfo, SolverDelegateEvalExt as _, StalledOnCoroutines, @@ -97,12 +97,12 @@ impl<'tcx> ObligationStorage<'tcx> { // Conservative here: if a stalled var no longer resolves to an // infer var, some unification happened, so the goal is no longer // stalled. Include it to be re-evaluated downstream. - stalled_on.stalled_vars.iter().filter_map(|arg| arg.as_type()).any( - |ty| match *infcx.shallow_resolve(ty).kind() { + stalled_on.stalled_vars.iter().filter_map(|arg| arg.as_type(infcx.tcx)).any(|ty| { + match *infcx.shallow_resolve(ty).kind() { ty::Infer(ty::TyVar(tv)) => infcx.sub_unification_table_root_var(tv) == vid, _ => true, - }, - ) + } + }) }) .map(|(o, _)| o.clone()) .collect(); @@ -185,7 +185,22 @@ where obligation: PredicateObligation<'tcx>, ) { assert_eq!(self.usable_in_snapshot, infcx.num_open_snapshots()); - self.obligations.register(obligation, None); + + let delegate = <&SolverDelegate<'tcx>>::from(infcx); + if let Some(GoalEvaluation { goal: _, certainty, has_changed: _, stalled_on }) = + compute_goal_fast_path(delegate, obligation.as_goal(), obligation.cause.span) + { + // If we can take the fast path, don't even bother adding the goal to obligations, + // or if `Certainty::Maybe`, add it with precise stalled_on information. + match certainty { + Certainty::Yes => {} + Certainty::Maybe(_) => { + self.obligations.register(obligation, stalled_on); + } + } + } else { + self.obligations.register(obligation, None); + } } fn collect_remaining_errors(&mut self, infcx: &InferCtxt<'tcx>) -> Vec { @@ -209,32 +224,8 @@ where loop { let mut any_changed = false; for (mut obligation, stalled_on) in self.obligations.drain_pending(|_, _| true) { - if !infcx.tcx.recursion_limit().value_within_limit(obligation.recursion_depth) { - self.obligations.on_fulfillment_overflow(infcx); - // Only return true errors that we have accumulated while processing. - return errors; - } - let goal = obligation.as_goal(); let delegate = <&SolverDelegate<'tcx>>::from(infcx); - if !delegate.disable_trait_solver_fast_paths() - && let Some(certainty) = - delegate.compute_goal_fast_path(goal, obligation.cause.span) - { - match certainty { - // This fast path doesn't depend on region identity so it doesn't - // matter if the goal contains inference variables or not, so we - // don't need to call `push_hir_typeck_potentially_region_dependent_goal` - // here. - // - // Only goals proven via the trait solver should be region dependent. - Certainty::Yes => {} - Certainty::Maybe(_) => { - self.obligations.register(obligation, None); - } - } - continue; - } let result = delegate.evaluate_root_goal(goal, obligation.cause.span, stalled_on); self.inspect_evaluated_obligation(infcx, &obligation, &result); @@ -261,7 +252,14 @@ where // approximation and should only result in fulfillment overflow in // pathological cases. obligation.recursion_depth += 1; - any_changed = true; + + if !infcx.tcx.recursion_limit().value_within_limit(obligation.recursion_depth) { + self.obligations.on_fulfillment_overflow(infcx); + // Only return true errors that we have accumulated while processing. + return errors; + } else { + any_changed = true; + } } match certainty { diff --git a/compiler/rustc_trait_selection/src/traits/fulfill.rs b/compiler/rustc_trait_selection/src/traits/fulfill.rs index 00ad863f38346..944b1290b66e1 100644 --- a/compiler/rustc_trait_selection/src/traits/fulfill.rs +++ b/compiler/rustc_trait_selection/src/traits/fulfill.rs @@ -17,6 +17,7 @@ use rustc_middle::ty::{ self, Binder, Const, DelayedSet, GenericArgsRef, Ty, TyCtxt, TypeSuperVisitable, TypeVisitable, TypeVisitableExt, TypeVisitor, TypingMode, may_use_unstable_feature, }; +use rustc_next_trait_solver::solve::TyOrConstInferVar; use thin_vec::{ThinVec, thin_vec}; use tracing::{debug, debug_span, instrument}; @@ -28,7 +29,7 @@ use super::{ ScrubbedTraitError, const_evaluatable, wf, }; use crate::error_reporting::InferCtxtErrorExt; -use crate::infer::{InferCtxt, TyOrConstInferVar}; +use crate::infer::InferCtxt; use crate::traits::normalize::normalize_with_depth_to; use crate::traits::project::{PolyProjectionObligation, ProjectionCacheKeyExt as _}; use crate::traits::query::evaluate_obligation::InferCtxtExt; @@ -617,8 +618,9 @@ impl<'a, 'tcx> ObligationProcessor for FulfillProcessor<'a, 'tcx> { obligation.cause.span, ) { None => { - pending_obligation.stalled_on = - vec![TyOrConstInferVar::maybe_from_term(term).unwrap()]; + pending_obligation.stalled_on = vec![ + TyOrConstInferVar::maybe_from_term::>(term).unwrap(), + ]; ProcessResult::Unchanged } Some(os) => ProcessResult::Changed(mk_pending(obligation, os)), @@ -683,11 +685,9 @@ impl<'a, 'tcx> ObligationProcessor for FulfillProcessor<'a, 'tcx> { Ok(()) => ProcessResult::Changed(Default::default()), Err(NotConstEvaluatable::MentionsInfer) => { pending_obligation.stalled_on.clear(); - pending_obligation.stalled_on.extend( - alias_const - .walk() - .filter_map(TyOrConstInferVar::maybe_from_generic_arg), - ); + pending_obligation.stalled_on.extend(alias_const.walk().filter_map( + TyOrConstInferVar::maybe_from_generic_arg::>, + )); ProcessResult::Unchanged } Err( @@ -767,12 +767,9 @@ impl<'a, 'tcx> ObligationProcessor for FulfillProcessor<'a, 'tcx> { ) { Ok(val) => Ok(val), e @ Err(EvaluateConstErr::HasGenericsOrInfers) => { - stalled_on.extend( - alias_const - .args - .iter() - .filter_map(TyOrConstInferVar::maybe_from_generic_arg), - ); + stalled_on.extend(alias_const.args.iter().filter_map( + TyOrConstInferVar::maybe_from_generic_arg::>, + )); e } e @ Err( @@ -1016,7 +1013,7 @@ fn args_infer_vars<'tcx>( } walker.visited.into_iter() }) - .filter_map(TyOrConstInferVar::maybe_from_generic_arg) + .filter_map(TyOrConstInferVar::maybe_from_generic_arg::>) } #[derive(Debug)] diff --git a/compiler/rustc_type_ir/Cargo.toml b/compiler/rustc_type_ir/Cargo.toml index 97cddd857af18..5cc385766605b 100644 --- a/compiler/rustc_type_ir/Cargo.toml +++ b/compiler/rustc_type_ir/Cargo.toml @@ -8,7 +8,7 @@ edition = "2024" arrayvec = { version = "0.7", default-features = false } bitflags = "2.4.1" derive-where = "1.6.1" -ena = "0.14.3" +ena = "0.14.4" indexmap = "2.0.0" rustc-hash = "2.0.0" rustc_abi = { path = "../rustc_abi", default-features = false } diff --git a/compiler/rustc_type_ir/src/infer_ctxt.rs b/compiler/rustc_type_ir/src/infer_ctxt.rs index 6c8735eb4e5ef..84797fb55ed2e 100644 --- a/compiler/rustc_type_ir/src/infer_ctxt.rs +++ b/compiler/rustc_type_ir/src/infer_ctxt.rs @@ -9,7 +9,7 @@ use crate::fold::TypeFoldable; use crate::inherent::*; use crate::relate::RelateResult; use crate::relate::combine::PredicateEmittingRelation; -use crate::solve::VisibleForLeakCheck; +use crate::solve::{TyOrConstInferVar, VisibleForLeakCheck}; use crate::{self as ty, Interner, TyVid}; mod private { @@ -391,7 +391,7 @@ pub trait InferCtxtLike: Sized { vid: ty::RegionVid, ) -> ::Region; - fn is_changed_arg(&self, arg: ::GenericArg) -> bool; + fn ty_or_const_infer_var_changed(&self, var: TyOrConstInferVar) -> bool; fn next_region_infer(&self) -> ::Region; fn next_ty_infer(&self) -> ::Ty; diff --git a/compiler/rustc_type_ir/src/solve/mod.rs b/compiler/rustc_type_ir/src/solve/mod.rs index 8866c763927b3..6abb61585287b 100644 --- a/compiler/rustc_type_ir/src/solve/mod.rs +++ b/compiler/rustc_type_ir/src/solve/mod.rs @@ -12,11 +12,13 @@ use rustc_type_ir_macros::{ }; use tracing::debug; +use crate::inherent::*; use crate::lang_items::SolverTraitLangItem; use crate::region_constraint::RegionConstraint; use crate::search_graph::PathKind; use crate::{ - self as ty, Canonical, CanonicalVarValues, CantBeErased, Interner, TypingMode, Upcast, + self as ty, Canonical, CanonicalVarValues, CantBeErased, ConstVid, FloatVid, GenericArgKind, + InferConst, IntVid, Interner, TermKind, TyVid, TypingMode, Upcast, }; pub type CanonicalInput::Predicate> = @@ -942,3 +944,114 @@ impl SizedTraitKind { }) } } + +#[derive_where(Clone, Debug; I: Interner)] +pub enum SucceededInErased { + /// This goal previously succeeded in erased mode, which based on `accessed_opaques` + /// might make us take a fast path slightly more often. + Yes { + accessed_opaques: AccessedOpaques, + }, + No, +} + +#[derive_where(Clone, Debug; I: Interner)] +pub enum GoalStalledOnReason { + /// This goal got stalled in `compute_goal_fast_path`. Usually this means + /// the goal is stalled on not that much, only one or two variables, and + /// definitely nothing to do with opaque types. So we don't store that information. + FastPath, + Other { + num_opaques: usize, + previously_succeeded_in_erased: SucceededInErased, + }, +} + +/// The conditions that must change for a goal to warrant +#[derive_where(Clone, Debug; I: Interner)] +pub struct GoalStalledOn { + pub stalled_vars: Vec, + pub sub_roots: Vec, + /// The certainty that will be returned on subsequent evaluations if this + /// goal remains stalled. + pub stalled_certainty: Certainty, + pub reason: GoalStalledOnReason, +} + +/// For some goals we can trivially answer some questions without going through +/// canonicalization. There are three options: +#[derive(Clone, Debug)] +pub enum ComputeGoalFastPathOutcome { + /// Do not attempt the fast path. Compute as normal. + NoFastPath, + /// The goal trivially holds, immediately produce a result with [`Certainty::Yes`] + TriviallyHolds, + /// The goal is trivially stalled: we know for sure that it makes no sense to compute it right + /// now, but can return information about what its stalled on and when it can be computed for real. + TriviallyStalled { stalled_on: GoalStalledOn }, +} + +/// Helper for `InferCtxt::ty_or_const_infer_var_changed` (see comment on that), currently +/// used only for `traits::fulfill`'s list of `stalled_on` inference variables. +#[derive(Copy, Clone, Debug)] +pub enum TyOrConstInferVar { + /// Equivalent to `ty::Infer(ty::TyVar(_))`. + Ty(TyVid), + /// Equivalent to `ty::Infer(ty::IntVar(_))`. + TyInt(IntVid), + /// Equivalent to `ty::Infer(ty::FloatVar(_))`. + TyFloat(FloatVid), + + /// Equivalent to `ty::ConstKind::Infer(ty::InferConst::Var(_))`. + Const(ConstVid), +} + +impl TyOrConstInferVar { + pub fn as_type(&self, interner: I) -> Option { + match self { + Self::Ty(vid) => Some(I::Ty::new_var(interner, *vid)), + Self::TyInt(_) | Self::TyFloat(_) | Self::Const(_) => None, + } + } + + /// Tries to extract an inference variable from a type or a constant, returns `None` + /// for types other than `ty::Infer(_)` (or `InferTy::Fresh*`) and + /// for constants other than `ty::ConstKind::Infer(_)` (or `InferConst::Fresh`). + pub fn maybe_from_generic_arg(arg: I::GenericArg) -> Option { + match arg.kind() { + GenericArgKind::Type(ty) => Self::maybe_from_ty::(ty), + GenericArgKind::Const(ct) => Self::maybe_from_const::(ct), + GenericArgKind::Lifetime(_) => None, + } + } + + /// Tries to extract an inference variable from a type or a constant, returns `None` + /// for types other than `ty::Infer(_)` (or `InferTy::Fresh*`) and + /// for constants other than `ty::ConstKind::Infer(_)` (or `InferConst::Fresh`). + pub fn maybe_from_term(term: I::Term) -> Option { + match term.kind() { + TermKind::Ty(ty) => Self::maybe_from_ty::(ty), + TermKind::Const(ct) => Self::maybe_from_const::(ct), + } + } + + /// Tries to extract an inference variable from a type, returns `None` + /// for types other than `ty::Infer(_)` (or `InferTy::Fresh*`). + fn maybe_from_ty(ty: I::Ty) -> Option { + match ty.kind() { + ty::Infer(ty::TyVar(v)) => Some(TyOrConstInferVar::Ty(v)), + ty::Infer(ty::IntVar(v)) => Some(TyOrConstInferVar::TyInt(v)), + ty::Infer(ty::FloatVar(v)) => Some(TyOrConstInferVar::TyFloat(v)), + _ => None, + } + } + + /// Tries to extract an inference variable from a constant, returns `None` + /// for constants other than `ty::ConstKind::Infer(_)` (or `InferConst::Fresh`). + fn maybe_from_const(ct: I::Const) -> Option { + match ct.kind() { + ty::ConstKind::Infer(InferConst::Var(v)) => Some(TyOrConstInferVar::Const(v)), + _ => None, + } + } +}