From 0527256e5cbb5c1717b1372da1d98c082caae2e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jana=20D=C3=B6nszelmann?= Date: Thu, 18 Jun 2026 13:00:59 +0200 Subject: [PATCH 1/9] move overflow check --- .../rustc_trait_selection/src/solve/fulfill.rs | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/compiler/rustc_trait_selection/src/solve/fulfill.rs b/compiler/rustc_trait_selection/src/solve/fulfill.rs index 4b80d22d6bb49..4554bcf38ea8e 100644 --- a/compiler/rustc_trait_selection/src/solve/fulfill.rs +++ b/compiler/rustc_trait_selection/src/solve/fulfill.rs @@ -209,12 +209,6 @@ where loop { let mut any_changed = false; for (mut obligation, stalled_on) in self.obligations.drain_pending(|_, _| true) { - if !infcx.tcx.recursion_limit().value_within_limit(obligation.recursion_depth) { - self.obligations.on_fulfillment_overflow(infcx); - // Only return true errors that we have accumulated while processing. - return errors; - } - let goal = obligation.as_goal(); let delegate = <&SolverDelegate<'tcx>>::from(infcx); if !delegate.disable_trait_solver_fast_paths() @@ -261,7 +255,14 @@ where // approximation and should only result in fulfillment overflow in // pathological cases. obligation.recursion_depth += 1; - any_changed = true; + + if !infcx.tcx.recursion_limit().value_within_limit(obligation.recursion_depth) { + self.obligations.on_fulfillment_overflow(infcx); + // Only return true errors that we have accumulated while processing. + return errors; + } else { + any_changed = true; + } } match certainty { From 645ce2820eda95146fc72e1102c157277b5790fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jana=20D=C3=B6nszelmann?= Date: Thu, 18 Jun 2026 11:30:03 +0200 Subject: [PATCH 2/9] move fast path into evaluate_goal --- .../src/solve/eval_ctxt/mod.rs | 23 ++++++++----------- 1 file changed, 9 insertions(+), 14 deletions(-) diff --git a/compiler/rustc_next_trait_solver/src/solve/eval_ctxt/mod.rs b/compiler/rustc_next_trait_solver/src/solve/eval_ctxt/mod.rs index d83320f5fe800..b2941e577280d 100644 --- a/compiler/rustc_next_trait_solver/src/solve/eval_ctxt/mod.rs +++ b/compiler/rustc_next_trait_solver/src/solve/eval_ctxt/mod.rs @@ -592,6 +592,15 @@ where )); } + if !self.delegate.disable_trait_solver_fast_paths() + && let Some(certainty) = self.delegate.compute_goal_fast_path(goal, self.origin_span) + { + return Ok(( + NestedNormalizationGoals::empty(), + GoalEvaluation { goal, certainty, has_changed: HasChanged::No, stalled_on: None }, + )); + } + self.evaluate_goal_cold(source, goal, increase_depth_for_nested) } @@ -990,20 +999,6 @@ where PredicateKind::NormalizesTo(_) )); - if !self.delegate.disable_trait_solver_fast_paths() - && let Some(certainty) = - self.delegate.compute_goal_fast_path(goal, self.origin_span) - { - match certainty { - Certainty::Yes => {} - Certainty::Maybe { .. } => { - self.nested_goals.push((source, goal, None)); - unchanged_certainty = unchanged_certainty.map(|c| c.and(certainty)); - } - } - continue; - } - let GoalEvaluation { goal, certainty, has_changed, stalled_on } = self.evaluate_goal(source, goal, stalled_on)?; if has_changed == HasChanged::Yes { From bbddf9910f5d6df25eb27ad942d579be8ba055d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jana=20D=C3=B6nszelmann?= Date: Thu, 18 Jun 2026 13:16:33 +0200 Subject: [PATCH 3/9] remove fastpath from fullfilment loop --- .../src/solve/fulfill.rs | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/compiler/rustc_trait_selection/src/solve/fulfill.rs b/compiler/rustc_trait_selection/src/solve/fulfill.rs index 4554bcf38ea8e..0944da1f93dfe 100644 --- a/compiler/rustc_trait_selection/src/solve/fulfill.rs +++ b/compiler/rustc_trait_selection/src/solve/fulfill.rs @@ -7,7 +7,6 @@ use rustc_infer::traits::{ FromSolverError, PredicateObligation, PredicateObligations, TraitEngine, }; use rustc_middle::ty::{self, TyCtxt, TyVid, TypeVisitableExt, TypingMode}; -use rustc_next_trait_solver::delegate::SolverDelegate as _; use rustc_next_trait_solver::solve::{ GoalEvaluation, GoalStalledOn, HasChanged, MaybeInfo, SolverDelegateEvalExt as _, StalledOnCoroutines, @@ -211,24 +210,6 @@ where for (mut obligation, stalled_on) in self.obligations.drain_pending(|_, _| true) { let goal = obligation.as_goal(); let delegate = <&SolverDelegate<'tcx>>::from(infcx); - if !delegate.disable_trait_solver_fast_paths() - && let Some(certainty) = - delegate.compute_goal_fast_path(goal, obligation.cause.span) - { - match certainty { - // This fast path doesn't depend on region identity so it doesn't - // matter if the goal contains inference variables or not, so we - // don't need to call `push_hir_typeck_potentially_region_dependent_goal` - // here. - // - // Only goals proven via the trait solver should be region dependent. - Certainty::Yes => {} - Certainty::Maybe(_) => { - self.obligations.register(obligation, None); - } - } - continue; - } let result = delegate.evaluate_root_goal(goal, obligation.cause.span, stalled_on); self.inspect_evaluated_obligation(infcx, &obligation, &result); From 7a12d7d40c61d2e1205108ff6d045b28be83b81a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jana=20D=C3=B6nszelmann?= Date: Thu, 18 Jun 2026 14:12:54 +0200 Subject: [PATCH 4/9] experiment with better evaluate_goal_step --- compiler/rustc_next_trait_solver/src/solve/eval_ctxt/mod.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/compiler/rustc_next_trait_solver/src/solve/eval_ctxt/mod.rs b/compiler/rustc_next_trait_solver/src/solve/eval_ctxt/mod.rs index b2941e577280d..4ef0c73c9e8da 100644 --- a/compiler/rustc_next_trait_solver/src/solve/eval_ctxt/mod.rs +++ b/compiler/rustc_next_trait_solver/src/solve/eval_ctxt/mod.rs @@ -992,6 +992,9 @@ where ) -> Result, NoSolutionOrRerunNonErased> { // If this loop did not result in any progress, what's our final certainty. let mut unchanged_certainty = Some(Certainty::Yes); + // This mem::take seems super inefficient, given that we push to it again later. + // Despite that, replacing it has no effect on performance. We tried. + // (https://github.com/rust-lang/rust/pull/158126) for (source, goal, stalled_on) in mem::take(&mut self.nested_goals) { // We never handle `NormalizesTo` as a nested goal debug_assert!(!matches!( From 91070df0b0e544dd44f64991eef5860d50be0b6a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jana=20D=C3=B6nszelmann?= Date: Mon, 22 Jun 2026 10:13:06 +0200 Subject: [PATCH 5/9] return stalled info from fast path --- compiler/rustc_middle/src/traits/solve.rs | 4 + .../rustc_next_trait_solver/src/delegate.rs | 5 +- .../src/solve/eval_ctxt/mod.rs | 200 +++++++++++------- .../rustc_next_trait_solver/src/solve/mod.rs | 20 +- .../src/solve/delegate.rs | 76 ++++--- compiler/rustc_type_ir/src/solve/mod.rs | 49 ++++- 6 files changed, 227 insertions(+), 127 deletions(-) diff --git a/compiler/rustc_middle/src/traits/solve.rs b/compiler/rustc_middle/src/traits/solve.rs index ceecb26d242dd..c45ebb80255cc 100644 --- a/compiler/rustc_middle/src/traits/solve.rs +++ b/compiler/rustc_middle/src/traits/solve.rs @@ -16,6 +16,10 @@ pub type CanonicalInput<'tcx, P = ty::Predicate<'tcx>> = ir::solve::CanonicalInp pub type CanonicalResponse<'tcx> = ir::solve::CanonicalResponse>; pub type FetchEligibleAssocItemResponse<'tcx> = ir::solve::FetchEligibleAssocItemResponse>; +pub type ComputeGoalFastPathOutcome<'tcx> = ir::solve::ComputeGoalFastPathOutcome>; +pub type GoalStalledOn<'tcx> = ir::solve::GoalStalledOn>; +pub type GoalStalledOnReason<'tcx> = ir::solve::GoalStalledOnReason>; +pub type SucceededInErased<'tcx> = ir::solve::SucceededInErased>; pub type PredefinedOpaques<'tcx> = &'tcx ty::List<(ty::OpaqueTypeKey<'tcx>, Ty<'tcx>)>; diff --git a/compiler/rustc_next_trait_solver/src/delegate.rs b/compiler/rustc_next_trait_solver/src/delegate.rs index 2ebb0fb851a4c..7b66667486fc9 100644 --- a/compiler/rustc_next_trait_solver/src/delegate.rs +++ b/compiler/rustc_next_trait_solver/src/delegate.rs @@ -1,7 +1,8 @@ use std::ops::Deref; use rustc_type_ir::solve::{ - Certainty, FetchEligibleAssocItemResponse, Goal, NoSolution, VisibleForLeakCheck, + Certainty, ComputeGoalFastPathOutcome, FetchEligibleAssocItemResponse, Goal, NoSolution, + VisibleForLeakCheck, }; use rustc_type_ir::{self as ty, InferCtxtLike, Interner, TypeFoldable}; @@ -23,7 +24,7 @@ pub trait SolverDelegate: Deref + Sized { &self, goal: Goal::Predicate>, span: ::Span, - ) -> Option; + ) -> ComputeGoalFastPathOutcome; fn fresh_var_for_kind_with_span( &self, diff --git a/compiler/rustc_next_trait_solver/src/solve/eval_ctxt/mod.rs b/compiler/rustc_next_trait_solver/src/solve/eval_ctxt/mod.rs index 4ef0c73c9e8da..835720fd6dc70 100644 --- a/compiler/rustc_next_trait_solver/src/solve/eval_ctxt/mod.rs +++ b/compiler/rustc_next_trait_solver/src/solve/eval_ctxt/mod.rs @@ -10,9 +10,10 @@ use rustc_type_ir::relate::Relate; use rustc_type_ir::relate::solver_relating::RelateExt; use rustc_type_ir::search_graph::{CandidateHeadUsages, LowerAvailableDepth, PathKind}; use rustc_type_ir::solve::{ - AccessedOpaques, ExternalRegionConstraints, FetchEligibleAssocItemResponse, MaybeInfo, - NoSolutionOrRerunNonErased, OpaqueTypesJank, QueryResultOrRerunNonErased, RerunCondition, - RerunNonErased, RerunReason, RerunResultExt, SmallCopyList, + AccessedOpaques, ComputeGoalFastPathOutcome, ExternalRegionConstraints, + FetchEligibleAssocItemResponse, MaybeInfo, NoSolutionOrRerunNonErased, OpaqueTypesJank, + QueryResultOrRerunNonErased, RerunCondition, RerunNonErased, RerunReason, RerunResultExt, + SmallCopyList, }; use rustc_type_ir::{ self as ty, CanonicalVarValues, ClauseKind, InferCtxtLike, Interner, MayBeErased, @@ -35,7 +36,7 @@ use crate::solve::search_graph::SearchGraph; use crate::solve::ty::may_use_unstable_feature; use crate::solve::{ CanonicalInput, CanonicalResponse, Certainty, ExternalConstraintsData, FIXPOINT_STEP_LIMIT, - Goal, GoalEvaluation, GoalSource, GoalStalledOn, HasChanged, MaybeCause, + Goal, GoalEvaluation, GoalSource, GoalStalledOn, GoalStalledOnReason, HasChanged, MaybeCause, NestedNormalizationGoals, NoSolution, QueryInput, QueryResult, Response, SucceededInErased, VisibleForLeakCheck, inspect, }; @@ -506,13 +507,8 @@ where } // If the goal isn't stalled, we should definitely run it. - let Some(&GoalStalledOn { - num_opaques, - ref stalled_vars, - ref sub_roots, - stalled_certainty, - ref previously_succeeded_in_erased, - }) = stalled_on + let Some(&GoalStalledOn { ref reason, ref stalled_vars, ref sub_roots, stalled_certainty }) = + stalled_on else { return MayMakeProgress; }; @@ -529,34 +525,46 @@ where return MayMakeProgress; } - // If any opaques changed in the opaque type storage, - // rerunning might make progress so we should rerun. - if self.delegate.opaque_types_storage_num_entries().needs_reevaluation(num_opaques) { - // Unless this goal previously succeeded in erased mode. - // If the stalled goal successfully evaluated while erasing opaque types, - // and the current state of the opaque type storage is not different in a way that is - // relevant, this stalled goal cannot make any progress and we set this variable to true. - let mut previous_erased_run_is_still_valid = false; - - if let &SucceededInErased::Yes { accessed_opaques } = previously_succeeded_in_erased { - match self.should_rerun_after_erased_canonicalization( - accessed_opaques, - self.typing_mode(), - &self.delegate.clone_opaque_types_lookup_table(), - ) { - RerunDecision::Yes => {} - RerunDecision::EagerlyPropagateToParent => { - unreachable!("we never retry stalled queries if the parent was erased") + match reason { + GoalStalledOnReason::FastPath => { + // fastpath is never because of opaques, we can skip this check + } + &GoalStalledOnReason::Other { num_opaques, ref previously_succeeded_in_erased } => { + // If any opaques changed in the opaque type storage, + // rerunning might make progress so we should rerun. + if self.delegate.opaque_types_storage_num_entries().needs_reevaluation(num_opaques) + { + // Unless this goal previously succeeded in erased mode. + // If the stalled goal successfully evaluated while erasing opaque types, + // and the current state of the opaque type storage is not different in a way that is + // relevant, this stalled goal cannot make any progress and we set this variable to true. + let mut previous_erased_run_is_still_valid = false; + + if let &SucceededInErased::Yes { accessed_opaques } = + previously_succeeded_in_erased + { + match self.should_rerun_after_erased_canonicalization( + accessed_opaques, + self.typing_mode(), + &self.delegate.clone_opaque_types_lookup_table(), + ) { + RerunDecision::Yes => {} + RerunDecision::EagerlyPropagateToParent => { + unreachable!( + "we never retry stalled queries if the parent was erased" + ) + } + RerunDecision::No => { + previous_erased_run_is_still_valid = true; + } + } } - RerunDecision::No => { - previous_erased_run_is_still_valid = true; + + if !previous_erased_run_is_still_valid { + return MayMakeProgress; } } } - - if !previous_erased_run_is_still_valid { - return MayMakeProgress; - } } // Otherwise, we can be sure that this stalled goal cannot make any progress @@ -564,6 +572,39 @@ where WontMakeProgress(stalled_certainty) } + /// This is a fast path optimization: + /// See the docs on [`ComputeGoalFastPathOutcome`] + pub fn compute_goal_fast_path( + &self, + goal: Goal, + ) -> Option<(NestedNormalizationGoals, GoalEvaluation)> { + if self.delegate.disable_trait_solver_fast_paths() { + return None; + } + + match self.delegate.compute_goal_fast_path(goal, self.origin_span) { + ComputeGoalFastPathOutcome::NoFastPath => None, + ComputeGoalFastPathOutcome::TriviallyHolds => Some(( + NestedNormalizationGoals::empty(), + GoalEvaluation { + goal, + certainty: Certainty::Yes, + has_changed: HasChanged::No, + stalled_on: None, + }, + )), + ComputeGoalFastPathOutcome::TriviallyStalled { stalled_on } => Some(( + NestedNormalizationGoals::empty(), + GoalEvaluation { + goal, + certainty: Certainty::AMBIGUOUS, + has_changed: HasChanged::No, + stalled_on: Some(stalled_on), + }, + )), + } + } + /// Recursively evaluates `goal`, returning the nested goals in case /// the nested goal is a `NormalizesTo` goal. /// @@ -592,13 +633,8 @@ where )); } - if !self.delegate.disable_trait_solver_fast_paths() - && let Some(certainty) = self.delegate.compute_goal_fast_path(goal, self.origin_span) - { - return Ok(( - NestedNormalizationGoals::empty(), - GoalEvaluation { goal, certainty, has_changed: HasChanged::No, stalled_on: None }, - )); + if let Some(res) = self.compute_goal_fast_path(goal) { + return Ok(res); } self.evaluate_goal_cold(source, goal, increase_depth_for_nested) @@ -768,41 +804,12 @@ where // that is not resolved. Only when *these* have changed is it meaningful // to recompute this goal. HasChanged::Yes => None, - HasChanged::No => { - // Remove the canonicalized universal vars, since we only care about stalled existentials. - let mut sub_roots = Vec::new(); - let mut stalled_vars = orig_values; - stalled_vars.retain(|arg| match arg.kind() { - // Lifetimes can never stall goals. - ty::GenericArgKind::Lifetime(_) => false, - ty::GenericArgKind::Type(ty) => match ty.kind() { - ty::Infer(ty::TyVar(vid)) => { - sub_roots.push(self.delegate.sub_unification_table_root_var(vid)); - true - } - ty::Infer(_) => true, - ty::Param(_) | ty::Placeholder(_) => false, - _ => unreachable!("unexpected orig_value: {ty:?}"), - }, - ty::GenericArgKind::Const(ct) => match ct.kind() { - ty::ConstKind::Infer(_) => true, - ty::ConstKind::Param(_) | ty::ConstKind::Placeholder(_) => false, - _ => unreachable!("unexpected orig_value: {ct:?}"), - }, - }); - - Some(GoalStalledOn { - num_opaques: canonical_goal - .canonical - .value - .predefined_opaques_in_body - .len(), - stalled_vars, - sub_roots, - stalled_certainty: certainty, - previously_succeeded_in_erased: succeeded_in_erased, - }) - } + HasChanged::No => Some(self.build_stalled_on( + canonical_goal, + certainty, + orig_values, + succeeded_in_erased, + )), }, }; @@ -812,6 +819,45 @@ where )) } + fn build_stalled_on( + &self, + canonical_goal: CanonicalInput, + certainty: Certainty, + mut stalled_vars: Vec, + previously_succeeded_in_erased: SucceededInErased, + ) -> GoalStalledOn { + // Remove the canonicalized universal vars, since we only care about stalled existentials. + let mut sub_roots = Vec::new(); + stalled_vars.retain(|arg| match arg.kind() { + // Lifetimes can never stall goals. + ty::GenericArgKind::Lifetime(_) => false, + ty::GenericArgKind::Type(ty) => match ty.kind() { + ty::Infer(ty::TyVar(vid)) => { + sub_roots.push(self.delegate.sub_unification_table_root_var(vid)); + true + } + ty::Infer(_) => true, + ty::Param(_) | ty::Placeholder(_) => false, + _ => unreachable!("unexpected orig_value: {ty:?}"), + }, + ty::GenericArgKind::Const(ct) => match ct.kind() { + ty::ConstKind::Infer(_) => true, + ty::ConstKind::Param(_) | ty::ConstKind::Placeholder(_) => false, + _ => unreachable!("unexpected orig_value: {ct:?}"), + }, + }); + + GoalStalledOn { + stalled_vars, + sub_roots, + stalled_certainty: certainty, + reason: GoalStalledOnReason::Other { + num_opaques: canonical_goal.canonical.value.predefined_opaques_in_body.len(), + previously_succeeded_in_erased, + }, + } + } + fn should_rerun_after_erased_canonicalization( &self, AccessedOpaques { reason: _, rerun }: AccessedOpaques, diff --git a/compiler/rustc_next_trait_solver/src/solve/mod.rs b/compiler/rustc_next_trait_solver/src/solve/mod.rs index fcf8ddf59fe3c..100ef083ac727 100644 --- a/compiler/rustc_next_trait_solver/src/solve/mod.rs +++ b/compiler/rustc_next_trait_solver/src/solve/mod.rs @@ -23,7 +23,7 @@ mod trait_goals; use derive_where::derive_where; use rustc_type_ir::inherent::*; pub use rustc_type_ir::solve::*; -use rustc_type_ir::{self as ty, Interner, TyVid}; +use rustc_type_ir::{self as ty, Interner}; use tracing::instrument; pub use self::eval_ctxt::{ @@ -424,21 +424,3 @@ pub struct GoalEvaluation { /// before rerunning it. pub stalled_on: Option>, } - -/// The conditions that must change for a goal to warrant -#[derive_where(Clone, Debug; I: Interner)] -pub struct GoalStalledOn { - pub num_opaques: usize, - pub stalled_vars: Vec, - pub sub_roots: Vec, - /// The certainty that will be returned on subsequent evaluations if this - /// goal remains stalled. - pub stalled_certainty: Certainty, - pub previously_succeeded_in_erased: SucceededInErased, -} - -#[derive_where(Clone, Debug; I: Interner)] -pub enum SucceededInErased { - Yes { accessed_opaques: AccessedOpaques }, - No, -} diff --git a/compiler/rustc_trait_selection/src/solve/delegate.rs b/compiler/rustc_trait_selection/src/solve/delegate.rs index e0877829b25f3..913cf28cd2426 100644 --- a/compiler/rustc_trait_selection/src/solve/delegate.rs +++ b/compiler/rustc_trait_selection/src/solve/delegate.rs @@ -10,12 +10,15 @@ use rustc_infer::infer::canonical::{ QueryRegionConstraint, }; use rustc_infer::infer::{InferCtxt, RegionVariableOrigin, SubregionOrigin, TyCtxtInferExt}; -use rustc_infer::traits::solve::{FetchEligibleAssocItemResponse, Goal}; +use rustc_infer::traits::solve::{ + ComputeGoalFastPathOutcome, FetchEligibleAssocItemResponse, Goal, +}; use rustc_middle::traits::query::NoSolution; use rustc_middle::traits::solve::Certainty; use rustc_middle::ty::{ self, MayBeErased, Ty, TyCtxt, TypeFlags, TypeFoldable, TypeVisitableExt, TypingMode, }; +use rustc_next_trait_solver::solve::{GoalStalledOn, GoalStalledOnReason}; use rustc_span::{DUMMY_SP, Span}; use crate::traits::{EvaluateConstErr, ObligationCause, sizedness_fast_path, specialization_graph}; @@ -73,10 +76,25 @@ impl<'tcx> rustc_next_trait_solver::delegate::SolverDelegate for SolverDelegate< &self, goal: Goal<'tcx, ty::Predicate<'tcx>>, span: Span, - ) -> Option { + ) -> ComputeGoalFastPathOutcome<'tcx> { + use ComputeGoalFastPathOutcome as Outcome; + + fn stalled_with_args<'tcx>( + stalled_vars: Vec>, + ) -> ComputeGoalFastPathOutcome<'tcx> { + Outcome::TriviallyStalled { + stalled_on: GoalStalledOn { + stalled_vars, + sub_roots: Vec::new(), + stalled_certainty: Certainty::AMBIGUOUS, + reason: GoalStalledOnReason::FastPath, + }, + } + } + // FIXME(-Zassumptions-on-binders): actually handle fast path if self.tcx.assumptions_on_binders() { - return None; + return Outcome::NoFastPath; } let pred = goal.predicate.kind(); @@ -84,22 +102,23 @@ impl<'tcx> rustc_next_trait_solver::delegate::SolverDelegate for SolverDelegate< ty::PredicateKind::Clause(ty::ClauseKind::Trait(trait_pred)) => { let trait_pred = pred.rebind(trait_pred); - if self.shallow_resolve(trait_pred.self_ty().skip_binder()).is_ty_var() + let self_ty = self.shallow_resolve(trait_pred.self_ty().skip_binder()); + if self_ty.is_ty_var() // We don't do this fast path when opaques are defined since we may // eventually use opaques to incompletely guide inference via ty var // self types. // FIXME: Properly consider opaques here. && self.known_no_opaque_types_in_storage() { - Some(Certainty::AMBIGUOUS) + stalled_with_args(vec![self_ty.into()]) } else if trait_pred.polarity() == ty::PredicatePolarity::Positive { match self.0.tcx.as_lang_item(trait_pred.def_id()) { Some(LangItem::Sized) | Some(LangItem::MetaSized) => { let predicate = self.resolve_vars_if_possible(goal.predicate); if sizedness_fast_path(self.tcx, predicate, goal.param_env) { - return Some(Certainty::Yes); + return Outcome::TriviallyHolds; } else { - None + Outcome::NoFastPath } } Some(LangItem::Copy | LangItem::Clone) => { @@ -114,23 +133,23 @@ impl<'tcx> rustc_next_trait_solver::delegate::SolverDelegate for SolverDelegate< .has_type_flags(TypeFlags::HAS_FREE_REGIONS | TypeFlags::HAS_INFER) && self_ty.is_trivially_pure_clone_copy() { - return Some(Certainty::Yes); + return Outcome::TriviallyHolds; } else { - None + Outcome::NoFastPath } } - _ => None, + _ => Outcome::NoFastPath, } } else { - None + Outcome::NoFastPath } } ty::PredicateKind::DynCompatible(def_id) if self.0.tcx.is_dyn_compatible(def_id) => { - Some(Certainty::Yes) + Outcome::TriviallyHolds } ty::PredicateKind::Clause(ty::ClauseKind::RegionOutlives(outlives)) => { if outlives.has_escaping_bound_vars() { - return None; + return Outcome::NoFastPath; } self.0.sub_regions( @@ -139,11 +158,11 @@ impl<'tcx> rustc_next_trait_solver::delegate::SolverDelegate for SolverDelegate< outlives.0, ty::VisibleForLeakCheck::Yes, ); - Some(Certainty::Yes) + Outcome::TriviallyHolds } ty::PredicateKind::Clause(ty::ClauseKind::TypeOutlives(outlives)) => { if outlives.has_escaping_bound_vars() { - return None; + return Outcome::NoFastPath; } self.0.register_type_outlives_constraint( @@ -152,48 +171,49 @@ impl<'tcx> rustc_next_trait_solver::delegate::SolverDelegate for SolverDelegate< &ObligationCause::dummy_with_span(span), ); - Some(Certainty::Yes) + Outcome::TriviallyHolds } ty::PredicateKind::Subtype(ty::SubtypePredicate { a, b, .. }) | ty::PredicateKind::Coerce(ty::CoercePredicate { a, b }) => { if a.has_escaping_bound_vars() || b.has_escaping_bound_vars() { - return None; + return Outcome::NoFastPath; } match (self.shallow_resolve(a).kind(), self.shallow_resolve(b).kind()) { (&ty::Infer(ty::TyVar(a_vid)), &ty::Infer(ty::TyVar(b_vid))) => { self.sub_unify_ty_vids_raw(a_vid, b_vid); - Some(Certainty::AMBIGUOUS) + stalled_with_args(vec![a.into(), b.into()]) } - _ => None, + _ => Outcome::NoFastPath, } } ty::PredicateKind::Clause(ty::ClauseKind::ConstArgHasType(ct, _)) => { if ct.has_escaping_bound_vars() { - return None; + return Outcome::NoFastPath; } - if self.shallow_resolve_const(ct).is_ct_infer() { - Some(Certainty::AMBIGUOUS) + let arg = self.shallow_resolve_const(ct); + if arg.is_ct_infer() { + stalled_with_args(vec![arg.into()]) } else { - None + Outcome::NoFastPath } } ty::PredicateKind::Clause(ty::ClauseKind::WellFormed(arg)) => { if arg.has_escaping_bound_vars() { - return None; + return Outcome::NoFastPath; } let arg = self.shallow_resolve_term(arg); if arg.is_trivially_wf(self.tcx) { - Some(Certainty::Yes) + Outcome::TriviallyHolds } else if arg.is_infer() { - Some(Certainty::AMBIGUOUS) + stalled_with_args(vec![arg.into_arg()]) } else { - None + Outcome::NoFastPath } } - _ => None, + _ => Outcome::NoFastPath, } } diff --git a/compiler/rustc_type_ir/src/solve/mod.rs b/compiler/rustc_type_ir/src/solve/mod.rs index 8866c763927b3..5b07a23387ecb 100644 --- a/compiler/rustc_type_ir/src/solve/mod.rs +++ b/compiler/rustc_type_ir/src/solve/mod.rs @@ -16,7 +16,7 @@ use crate::lang_items::SolverTraitLangItem; use crate::region_constraint::RegionConstraint; use crate::search_graph::PathKind; use crate::{ - self as ty, Canonical, CanonicalVarValues, CantBeErased, Interner, TypingMode, Upcast, + self as ty, Canonical, CanonicalVarValues, CantBeErased, Interner, TyVid, TypingMode, Upcast, }; pub type CanonicalInput::Predicate> = @@ -942,3 +942,50 @@ impl SizedTraitKind { }) } } + +#[derive_where(Clone, Debug; I: Interner)] +pub enum SucceededInErased { + /// This goal previously succeeded in erased mode, which based on `accessed_opaques` + /// might make us take a fast path slightly more often. + Yes { + accessed_opaques: AccessedOpaques, + }, + No, +} + +#[derive_where(Clone, Debug; I: Interner)] +pub enum GoalStalledOnReason { + /// This goal got stalled in `compute_goal_fast_path`. Usually this means + /// the goal is stalled on not that much, only one or two variables, and + /// definitely nothing to do with opaque types. So we don't store that information. + FastPath, + Other { + num_opaques: usize, + previously_succeeded_in_erased: SucceededInErased, + }, +} + +/// The conditions that must change for a goal to warrant +#[derive_where(Clone, Debug; I: Interner)] +pub struct GoalStalledOn { + pub stalled_vars: Vec, + pub sub_roots: Vec, + /// The certainty that will be returned on subsequent evaluations if this + /// goal remains stalled. + pub stalled_certainty: Certainty, + pub reason: GoalStalledOnReason, +} + +/// For some goals we can trivially answer some questions without going through +/// canonicalization. There are three options: + +#[derive(Clone, Debug)] +pub enum ComputeGoalFastPathOutcome { + /// Do not attempt the fast path. Compute as normal. + NoFastPath, + /// The goal trivially holds, immediately produce a result with [`Certainty::Yes`] + TriviallyHolds, + /// The goal is trivially stalled: we know for sure that it makes no sense to compute it right + /// now, but can return information about what its stalled on and when it can be computed for real. + TriviallyStalled { stalled_on: GoalStalledOn }, +} From aa10dde3ed4fca2e2e37a9c91df46f44f3d831cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jana=20D=C3=B6nszelmann?= Date: Mon, 22 Jun 2026 14:19:47 +0200 Subject: [PATCH 6/9] move goal fast path into evaluate_goal_cold --- .../rustc_next_trait_solver/src/solve/eval_ctxt/mod.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/compiler/rustc_next_trait_solver/src/solve/eval_ctxt/mod.rs b/compiler/rustc_next_trait_solver/src/solve/eval_ctxt/mod.rs index 835720fd6dc70..25e3dd49ee438 100644 --- a/compiler/rustc_next_trait_solver/src/solve/eval_ctxt/mod.rs +++ b/compiler/rustc_next_trait_solver/src/solve/eval_ctxt/mod.rs @@ -633,10 +633,6 @@ where )); } - if let Some(res) = self.compute_goal_fast_path(goal) { - return Ok(res); - } - self.evaluate_goal_cold(source, goal, increase_depth_for_nested) } @@ -648,6 +644,10 @@ where goal: Goal, increase_depth_for_nested: LowerAvailableDepth, ) -> Result<(NestedNormalizationGoals, GoalEvaluation), NoSolutionOrRerunNonErased> { + if let Some(res) = self.compute_goal_fast_path(goal) { + return Ok(res); + } + // We only care about one entry per `OpaqueTypeKey` here, // so we only canonicalize the lookup table and ignore // duplicate entries. From 78f0efff3aeaedefde8cea68efd3cc3dff8a017a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jana=20D=C3=B6nszelmann?= Date: Thu, 25 Jun 2026 10:25:23 +0200 Subject: [PATCH 7/9] shuffle fastpaths around to before creating an evalctxt --- .../src/solve/eval_ctxt/fast_path.rs | 139 ++++++ .../src/solve/eval_ctxt/mod.rs | 431 ++++++++---------- .../src/solve/project_goals/mod.rs | 1 - 3 files changed, 317 insertions(+), 254 deletions(-) create mode 100644 compiler/rustc_next_trait_solver/src/solve/eval_ctxt/fast_path.rs diff --git a/compiler/rustc_next_trait_solver/src/solve/eval_ctxt/fast_path.rs b/compiler/rustc_next_trait_solver/src/solve/eval_ctxt/fast_path.rs new file mode 100644 index 0000000000000..978d97f184797 --- /dev/null +++ b/compiler/rustc_next_trait_solver/src/solve/eval_ctxt/fast_path.rs @@ -0,0 +1,139 @@ +//! This file contains a number of standalone functions useful for taking _fast paths_ in the trait +//! solver. The exact place where we check for these fast paths changes, and matters a lot for +//! performance. Ideally we'd only check them in `evaluate_goal`, but when evaluating root goals +//! we can check them earlier and save some time creating an `EvalCtxt` in the first place. +//! +//! For debugging, fast paths can be disabled using `-Zdisable-fast-paths`. + +use rustc_type_ir::InferCtxtLike; +use rustc_type_ir::Interner; +use rustc_type_ir::inherent::*; +use rustc_type_ir::solve::Goal; +use rustc_type_ir::solve::{ + Certainty, ComputeGoalFastPathOutcome, GoalStalledOn, GoalStalledOnReason, SucceededInErased, +}; + +use crate::delegate::SolverDelegate; +use crate::solve::GoalEvaluation; +use crate::solve::HasChanged; +use crate::solve::eval_ctxt::RerunDecision; +use crate::solve::eval_ctxt::should_rerun_after_erased_canonicalization; + +#[derive(Debug, Clone, Copy)] +pub(super) enum RerunStalled { + WontMakeProgress(Certainty), + MayMakeProgress, +} + +/// If we have run a goal before, and it was stalled, check that any of the goal's +/// args have changed. This is a cheap way to determine that if we were to rerun this goal now, +/// it will remain stalled since it'll canonicalize the same way and evaluation is pure. +/// Therefore, we can skip this rerun +pub(super) fn rerunning_stalled_goal_may_make_progress( + delegate: &D, + stalled_on: Option<&GoalStalledOn>, +) -> RerunStalled +where + D: SolverDelegate, + I: Interner, +{ + use RerunStalled::*; + + // If fast paths are turned off, then we assume all goals can always make progress + if delegate.disable_trait_solver_fast_paths() { + return MayMakeProgress; + } + + // If the goal isn't stalled, we should definitely run it. + let Some(&GoalStalledOn { ref reason, ref stalled_vars, ref sub_roots, stalled_certainty }) = + stalled_on + else { + return MayMakeProgress; + }; + + // If any of the stalled goal's generic arguments changed, + // rerunning might make progress so we should rerun. + if stalled_vars.iter().any(|value| delegate.is_changed_arg(*value)) { + return MayMakeProgress; + } + + // If some inference took place in any of the sub roots, + // rerunning might make progress so we should rerun. + if sub_roots.iter().any(|&vid| delegate.sub_unification_table_root_var(vid) != vid) { + return MayMakeProgress; + } + + match reason { + GoalStalledOnReason::FastPath => { + // fastpath is never because of opaques, we can skip this check + } + &GoalStalledOnReason::Other { num_opaques, ref previously_succeeded_in_erased } => { + // If any opaques changed in the opaque type storage, + // rerunning might make progress so we should rerun. + if delegate.opaque_types_storage_num_entries().needs_reevaluation(num_opaques) { + // Unless this goal previously succeeded in erased mode. + // If the stalled goal successfully evaluated while erasing opaque types, + // and the current state of the opaque type storage is not different in a way that is + // relevant, this stalled goal cannot make any progress and we set this variable to true. + let mut previous_erased_run_is_still_valid = false; + + if let &SucceededInErased::Yes { accessed_opaques } = previously_succeeded_in_erased + { + match should_rerun_after_erased_canonicalization( + accessed_opaques, + delegate.typing_mode_raw(), + &delegate.clone_opaque_types_lookup_table(), + ) { + RerunDecision::Yes => {} + RerunDecision::EagerlyPropagateToParent => { + unreachable!("we never retry stalled queries if the parent was erased") + } + RerunDecision::No => { + previous_erased_run_is_still_valid = true; + } + } + } + + if !previous_erased_run_is_still_valid { + return MayMakeProgress; + } + } + } + } + + // Otherwise, we can be sure that this stalled goal cannot make any progress + // and we can exit early. + WontMakeProgress(stalled_certainty) +} + +/// This is a fast path optimization: +/// See the docs on [`ComputeGoalFastPathOutcome`] +pub(super) fn compute_goal_fast_path( + delegate: &D, + goal: Goal, + origin_span: I::Span, +) -> Option> +where + D: SolverDelegate, + I: Interner, +{ + if delegate.disable_trait_solver_fast_paths() { + return None; + } + + match delegate.compute_goal_fast_path(goal, origin_span) { + ComputeGoalFastPathOutcome::NoFastPath => None, + ComputeGoalFastPathOutcome::TriviallyHolds => Some(GoalEvaluation { + goal, + certainty: Certainty::Yes, + has_changed: HasChanged::No, + stalled_on: None, + }), + ComputeGoalFastPathOutcome::TriviallyStalled { stalled_on } => Some(GoalEvaluation { + goal, + certainty: Certainty::AMBIGUOUS, + has_changed: HasChanged::No, + stalled_on: Some(stalled_on), + }), + } +} diff --git a/compiler/rustc_next_trait_solver/src/solve/eval_ctxt/mod.rs b/compiler/rustc_next_trait_solver/src/solve/eval_ctxt/mod.rs index 25e3dd49ee438..ffcd388208fdc 100644 --- a/compiler/rustc_next_trait_solver/src/solve/eval_ctxt/mod.rs +++ b/compiler/rustc_next_trait_solver/src/solve/eval_ctxt/mod.rs @@ -10,10 +10,9 @@ use rustc_type_ir::relate::Relate; use rustc_type_ir::relate::solver_relating::RelateExt; use rustc_type_ir::search_graph::{CandidateHeadUsages, LowerAvailableDepth, PathKind}; use rustc_type_ir::solve::{ - AccessedOpaques, ComputeGoalFastPathOutcome, ExternalRegionConstraints, - FetchEligibleAssocItemResponse, MaybeInfo, NoSolutionOrRerunNonErased, OpaqueTypesJank, - QueryResultOrRerunNonErased, RerunCondition, RerunNonErased, RerunReason, RerunResultExt, - SmallCopyList, + AccessedOpaques, ExternalRegionConstraints, FetchEligibleAssocItemResponse, MaybeInfo, + NoSolutionOrRerunNonErased, OpaqueTypesJank, QueryResultOrRerunNonErased, RerunCondition, + RerunNonErased, RerunReason, RerunResultExt, SmallCopyList, }; use rustc_type_ir::{ self as ty, CanonicalVarValues, ClauseKind, InferCtxtLike, Interner, MayBeErased, @@ -32,6 +31,9 @@ use crate::delegate::SolverDelegate; use crate::normalize::{NormalizationFolder, NormalizationWasAmbiguous}; use crate::placeholder::BoundVarReplacer; use crate::resolve::eager_resolve_vars; +use crate::solve::eval_ctxt::fast_path::{ + RerunStalled, compute_goal_fast_path, rerunning_stalled_goal_may_make_progress, +}; use crate::solve::search_graph::SearchGraph; use crate::solve::ty::may_use_unstable_feature; use crate::solve::{ @@ -41,6 +43,7 @@ use crate::solve::{ VisibleForLeakCheck, inspect, }; +mod fast_path; mod probe; mod solver_region_constraints; @@ -90,12 +93,6 @@ impl CurrentGoalKind { } } -#[derive(Debug)] -enum RerunDecision { - Yes, - No, - EagerlyPropagateToParent, -} pub struct EvalCtxt<'a, D, I = ::Interner> where D: SolverDelegate, @@ -227,8 +224,25 @@ where span: I::Span, stalled_on: Option>, ) -> Result, NoSolution> { + // Run fast paths *before* building an `EvalCtxt`, saving a little bit of time. + if let RerunStalled::WontMakeProgress(stalled_certainty) = + rerunning_stalled_goal_may_make_progress(self, stalled_on.as_ref()) + { + return Ok(GoalEvaluation { + goal, + certainty: stalled_certainty, + has_changed: HasChanged::No, + stalled_on, + }); + } + + if let Some(res) = compute_goal_fast_path(self, goal, span) { + return Ok(res); + } + let result = EvalCtxt::enter_root(self, self.cx().recursion_limit(), span, |ecx| { - ecx.evaluate_goal(GoalSource::Misc, goal, stalled_on) + // Fast paths handled above + ecx.evaluate_goal_no_fast_paths(GoalSource::Misc, goal) }); match result { @@ -286,12 +300,6 @@ where } } -#[derive(Debug, Clone, Copy)] -enum RerunStalled { - WontMakeProgress(Certainty), - MayMakeProgress, -} - impl<'a, D, I> EvalCtxt<'a, D> where D: SolverDelegate, @@ -484,125 +492,35 @@ where goal: Goal, stalled_on: Option>, ) -> Result, NoSolutionOrRerunNonErased> { - let (normalization_nested_goals, goal_evaluation) = - self.evaluate_goal_raw(source, goal, stalled_on, LowerAvailableDepth::Yes)?; - assert!(normalization_nested_goals.is_empty()); - Ok(goal_evaluation) - } - - /// This is a fast path optimization: - /// If we have run this goal before, and it was stalled, check that any of the goal's - /// args have changed. This is a cheap way to determine that if we were to rerun this goal now, - /// it will remain stalled since it'll canonicalize the same way and evaluation is pure. - /// Therefore, we can skip this rerun - fn rerunning_stalled_goal_may_make_progress( - &self, - stalled_on: Option<&GoalStalledOn>, - ) -> RerunStalled { - use RerunStalled::*; - - // If fast paths are turned off, then we assume all goals can always make progress - if self.delegate.disable_trait_solver_fast_paths() { - return MayMakeProgress; - } - - // If the goal isn't stalled, we should definitely run it. - let Some(&GoalStalledOn { ref reason, ref stalled_vars, ref sub_roots, stalled_certainty }) = - stalled_on - else { - return MayMakeProgress; - }; - - // If any of the stalled goal's generic arguments changed, - // rerunning might make progress so we should rerun. - if stalled_vars.iter().any(|value| self.delegate.is_changed_arg(*value)) { - return MayMakeProgress; - } - - // If some inference took place in any of the sub roots, - // rerunning might make progress so we should rerun. - if sub_roots.iter().any(|&vid| self.delegate.sub_unification_table_root_var(vid) != vid) { - return MayMakeProgress; + if let RerunStalled::WontMakeProgress(stalled_certainty) = + rerunning_stalled_goal_may_make_progress(self.delegate, stalled_on.as_ref()) + { + return Ok(GoalEvaluation { + goal, + certainty: stalled_certainty, + has_changed: HasChanged::No, + stalled_on, + }); } - match reason { - GoalStalledOnReason::FastPath => { - // fastpath is never because of opaques, we can skip this check - } - &GoalStalledOnReason::Other { num_opaques, ref previously_succeeded_in_erased } => { - // If any opaques changed in the opaque type storage, - // rerunning might make progress so we should rerun. - if self.delegate.opaque_types_storage_num_entries().needs_reevaluation(num_opaques) - { - // Unless this goal previously succeeded in erased mode. - // If the stalled goal successfully evaluated while erasing opaque types, - // and the current state of the opaque type storage is not different in a way that is - // relevant, this stalled goal cannot make any progress and we set this variable to true. - let mut previous_erased_run_is_still_valid = false; - - if let &SucceededInErased::Yes { accessed_opaques } = - previously_succeeded_in_erased - { - match self.should_rerun_after_erased_canonicalization( - accessed_opaques, - self.typing_mode(), - &self.delegate.clone_opaque_types_lookup_table(), - ) { - RerunDecision::Yes => {} - RerunDecision::EagerlyPropagateToParent => { - unreachable!( - "we never retry stalled queries if the parent was erased" - ) - } - RerunDecision::No => { - previous_erased_run_is_still_valid = true; - } - } - } - - if !previous_erased_run_is_still_valid { - return MayMakeProgress; - } - } - } + if let Some(res) = compute_goal_fast_path(self.delegate, goal, self.origin_span) { + return Ok(res); } - // Otherwise, we can be sure that this stalled goal cannot make any progress - // and we can exit early. - WontMakeProgress(stalled_certainty) + self.evaluate_goal_no_fast_paths(source, goal) } - /// This is a fast path optimization: - /// See the docs on [`ComputeGoalFastPathOutcome`] - pub fn compute_goal_fast_path( - &self, + // Outlining and `#[cold]` matter here because fast paths make it less likely to get here. + #[cold] + #[inline(never)] + fn evaluate_goal_no_fast_paths( + &mut self, + source: GoalSource, goal: Goal, - ) -> Option<(NestedNormalizationGoals, GoalEvaluation)> { - if self.delegate.disable_trait_solver_fast_paths() { - return None; - } - - match self.delegate.compute_goal_fast_path(goal, self.origin_span) { - ComputeGoalFastPathOutcome::NoFastPath => None, - ComputeGoalFastPathOutcome::TriviallyHolds => Some(( - NestedNormalizationGoals::empty(), - GoalEvaluation { - goal, - certainty: Certainty::Yes, - has_changed: HasChanged::No, - stalled_on: None, - }, - )), - ComputeGoalFastPathOutcome::TriviallyStalled { stalled_on } => Some(( - NestedNormalizationGoals::empty(), - GoalEvaluation { - goal, - certainty: Certainty::AMBIGUOUS, - has_changed: HasChanged::No, - stalled_on: Some(stalled_on), - }, - )), - } + ) -> Result, NoSolutionOrRerunNonErased> { + let (normalization_nested_goals, goal_evaluation) = + self.evaluate_goal_raw(source, goal, LowerAvailableDepth::Yes)?; + Ok(goal_evaluation) } /// Recursively evaluates `goal`, returning the nested goals in case @@ -616,38 +534,42 @@ where &mut self, source: GoalSource, goal: Goal, - stalled_on: Option>, increase_depth_for_nested: LowerAvailableDepth, ) -> Result<(NestedNormalizationGoals, GoalEvaluation), NoSolutionOrRerunNonErased> { - if let RerunStalled::WontMakeProgress(stalled_certainty) = - self.rerunning_stalled_goal_may_make_progress(stalled_on.as_ref()) - { - return Ok(( - NestedNormalizationGoals::empty(), - GoalEvaluation { - goal, - certainty: stalled_certainty, - has_changed: HasChanged::No, - stalled_on, - }, - )); - } - - self.evaluate_goal_cold(source, goal, increase_depth_for_nested) - } - - #[cold] - #[inline(never)] - pub(super) fn evaluate_goal_cold( - &mut self, - source: GoalSource, - goal: Goal, - increase_depth_for_nested: LowerAvailableDepth, + if let RerunStalled::WontMakeProgress(stalled_certainty) = + self.rerunning_stalled_goal_may_make_progress(stalled_on.as_ref()) + { + return Ok(( + NestedNormalizationGoals::empty(), + GoalEvaluation { + goal, + certainty: stalled_certainty, + has_changed: HasChanged::No, + stalled_on, + }, + )); + } + + self.evaluate_goal_cold(source, goal, increase_depth_for_nested) + } + + #[cold] + #[inline(never)] + pub(super) fn evaluate_goal_cold( + &mut self, + source: GoalSource, + goal: Goal, +- increase_depth_for_nested: IncreaseDepthForNested, ++ increase_depth_for_nested: LowerAvailableDepth, + ) -> Result<(NestedNormalizationGoals, GoalEvaluation), NoSolutionOrRerunNonErased> { + if let Some(res) = self.compute_goal_fast_path(goal) { + return Ok(res); + } + ++++++++ ktlxysvk c626102e "shuffle fastpaths around to before creating an evalctxt" (rebased revision) + increase_depth_for_nested: IncreaseDepthForNested, ) -> Result<(NestedNormalizationGoals, GoalEvaluation), NoSolutionOrRerunNonErased> { - if let Some(res) = self.compute_goal_fast_path(goal) { - return Ok(res); - } - +>>>>>>> conflict 2 of 2 ends // We only care about one entry per `OpaqueTypeKey` here, // so we only canonicalize the lookup table and ignore // duplicate entries. @@ -714,7 +636,7 @@ where &mut inspect::ProofTreeBuilder::new_noop(), ); - let should_rerun = self.should_rerun_after_erased_canonicalization( + let should_rerun = should_rerun_after_erased_canonicalization( accessed_opaques, self.typing_mode(), &opaque_types, @@ -858,100 +780,6 @@ where } } - fn should_rerun_after_erased_canonicalization( - &self, - AccessedOpaques { reason: _, rerun }: AccessedOpaques, - original_typing_mode: TypingMode, - parent_opaque_types: &[(OpaqueTypeKey, I::Ty)], - ) -> RerunDecision { - let parent_opaque_defids = parent_opaque_types.iter().map(|(key, _)| key.def_id.into()); - let opaque_in_storage = |opaques: I::LocalDefIds, defids: SmallCopyList<_>| { - if defids.as_ref().is_empty() { - RerunDecision::No - } else if opaques - .iter() - .chain(parent_opaque_defids) - .any(|opaque| defids.as_ref().contains(&opaque)) - { - RerunDecision::Yes - } else { - RerunDecision::No - } - }; - let any_opaque_has_infer_as_hidden = || { - if parent_opaque_types.iter().any(|(_, ty)| ty.is_ty_var()) { - RerunDecision::Yes - } else { - RerunDecision::No - } - }; - - let res = match (rerun, original_typing_mode) { - // ============================= - (RerunCondition::Never, _) => RerunDecision::No, - // ============================= - (_, TypingMode::ErasedNotCoherence(MayBeErased)) => { - RerunDecision::EagerlyPropagateToParent - } - // ============================= - // In coherence, we never switch to erased mode, so we will never register anything - // in the rerun state, so we should've taken the first branch of this match - (_, TypingMode::Coherence) => unreachable!(), - // ============================= - (RerunCondition::Always, _) => RerunDecision::Yes, - // ============================= - ( - RerunCondition::OpaqueInStorage(..), - TypingMode::PostAnalysis | TypingMode::Codegen, - ) => RerunDecision::Yes, - ( - RerunCondition::OpaqueInStorage(defids), - TypingMode::PostBorrowck { defined_opaque_types: opaques } - | TypingMode::Typeck { defining_opaque_types_and_generators: opaques } - | TypingMode::PostTypeckUntilBorrowck { defining_opaque_types: opaques }, - ) => opaque_in_storage(opaques, defids), - // ============================= - (RerunCondition::AnyOpaqueHasInferAsHidden, TypingMode::Typeck { .. }) => { - any_opaque_has_infer_as_hidden() - } - ( - RerunCondition::AnyOpaqueHasInferAsHidden, - TypingMode::PostBorrowck { .. } - | TypingMode::PostAnalysis - | TypingMode::Codegen - | TypingMode::PostTypeckUntilBorrowck { .. }, - ) => RerunDecision::No, - // ============================= - ( - RerunCondition::OpaqueInStorageOrAnyOpaqueHasInferAsHidden(_), - TypingMode::PostAnalysis | TypingMode::Codegen, - ) => RerunDecision::No, - ( - RerunCondition::OpaqueInStorageOrAnyOpaqueHasInferAsHidden(defids), - TypingMode::Typeck { defining_opaque_types_and_generators: opaques }, - ) => { - if let RerunDecision::Yes = any_opaque_has_infer_as_hidden() { - RerunDecision::Yes - } else if let RerunDecision::Yes = opaque_in_storage(opaques, defids) { - RerunDecision::Yes - } else { - RerunDecision::No - } - } - ( - RerunCondition::OpaqueInStorageOrAnyOpaqueHasInferAsHidden(defids), - TypingMode::PostBorrowck { defined_opaque_types: opaques } - | TypingMode::PostTypeckUntilBorrowck { defining_opaque_types: opaques }, - ) => opaque_in_storage(opaques, defids), - }; - - debug!( - "checking whether to rerun {rerun:?} in outer typing mode {original_typing_mode:?} and opaques {parent_opaque_types:?}: {res:?}" - ); - - res - } - pub(super) fn compute_goal( &mut self, goal: Goal, @@ -1779,6 +1607,103 @@ where } } +#[derive(Debug)] +enum RerunDecision { + Yes, + No, + EagerlyPropagateToParent, +} + +fn should_rerun_after_erased_canonicalization( + AccessedOpaques { reason: _, rerun }: AccessedOpaques, + original_typing_mode: TypingMode, + parent_opaque_types: &[(OpaqueTypeKey, I::Ty)], +) -> RerunDecision { + let parent_opaque_defids = parent_opaque_types.iter().map(|(key, _)| key.def_id.into()); + let opaque_in_storage = |opaques: I::LocalDefIds, defids: SmallCopyList<_>| { + if defids.as_ref().is_empty() { + RerunDecision::No + } else if opaques + .iter() + .chain(parent_opaque_defids) + .any(|opaque| defids.as_ref().contains(&opaque)) + { + RerunDecision::Yes + } else { + RerunDecision::No + } + }; + let any_opaque_has_infer_as_hidden = || { + if parent_opaque_types.iter().any(|(_, ty)| ty.is_ty_var()) { + RerunDecision::Yes + } else { + RerunDecision::No + } + }; + + let res = match (rerun, original_typing_mode) { + // ============================= + (RerunCondition::Never, _) => RerunDecision::No, + // ============================= + (_, TypingMode::ErasedNotCoherence(MayBeErased)) => RerunDecision::EagerlyPropagateToParent, + // ============================= + // In coherence, we never switch to erased mode, so we will never register anything + // in the rerun state, so we should've taken the first branch of this match + (_, TypingMode::Coherence) => unreachable!(), + // ============================= + (RerunCondition::Always, _) => RerunDecision::Yes, + // ============================= + (RerunCondition::OpaqueInStorage(..), TypingMode::PostAnalysis | TypingMode::Codegen) => { + RerunDecision::Yes + } + ( + RerunCondition::OpaqueInStorage(defids), + TypingMode::PostBorrowck { defined_opaque_types: opaques } + | TypingMode::Typeck { defining_opaque_types_and_generators: opaques } + | TypingMode::PostTypeckUntilBorrowck { defining_opaque_types: opaques }, + ) => opaque_in_storage(opaques, defids), + // ============================= + (RerunCondition::AnyOpaqueHasInferAsHidden, TypingMode::Typeck { .. }) => { + any_opaque_has_infer_as_hidden() + } + ( + RerunCondition::AnyOpaqueHasInferAsHidden, + TypingMode::PostBorrowck { .. } + | TypingMode::PostAnalysis + | TypingMode::Codegen + | TypingMode::PostTypeckUntilBorrowck { .. }, + ) => RerunDecision::No, + // ============================= + ( + RerunCondition::OpaqueInStorageOrAnyOpaqueHasInferAsHidden(_), + TypingMode::PostAnalysis | TypingMode::Codegen, + ) => RerunDecision::No, + ( + RerunCondition::OpaqueInStorageOrAnyOpaqueHasInferAsHidden(defids), + TypingMode::Typeck { defining_opaque_types_and_generators: opaques }, + ) => { + if let RerunDecision::Yes = any_opaque_has_infer_as_hidden() { + RerunDecision::Yes + } else if let RerunDecision::Yes = opaque_in_storage(opaques, defids) { + RerunDecision::Yes + } else { + RerunDecision::No + } + } + ( + RerunCondition::OpaqueInStorageOrAnyOpaqueHasInferAsHidden(defids), + TypingMode::PostBorrowck { defined_opaque_types: opaques } + | TypingMode::PostTypeckUntilBorrowck { defining_opaque_types: opaques }, + ) => opaque_in_storage(opaques, defids), + }; + + debug!( + "checking whether to rerun {rerun:?} in outer typing mode {original_typing_mode:?} and opaques {parent_opaque_types:?}: {res:?}" + ); + + res +} + /// Do not call this directly, use the `tcx` query instead. pub fn evaluate_root_goal_for_proof_tree_raw_provider< D: SolverDelegate, diff --git a/compiler/rustc_next_trait_solver/src/solve/project_goals/mod.rs b/compiler/rustc_next_trait_solver/src/solve/project_goals/mod.rs index 52ef40daf40da..41fe4105cb745 100644 --- a/compiler/rustc_next_trait_solver/src/solve/project_goals/mod.rs +++ b/compiler/rustc_next_trait_solver/src/solve/project_goals/mod.rs @@ -71,7 +71,6 @@ where ) = self.evaluate_goal_raw( GoalSource::TypeRelating, normalizes_to, - None, // We don't lower thr available depth for this `NormalizesTo` goal, as evaluating // it is an extra step only exists in the new solver that behaves like a function // call rather than an independent nested goal evaluation. So, decreasing the From 65b9e21f3535cd05af87b30ddd6350c6aeed989a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jana=20D=C3=B6nszelmann?= Date: Thu, 25 Jun 2026 13:26:20 +0200 Subject: [PATCH 8/9] fast path when adding goals --- .../src/solve/eval_ctxt/fast_path.rs | 30 +++++--- .../src/solve/eval_ctxt/mod.rs | 71 +++++++++---------- .../rustc_next_trait_solver/src/solve/mod.rs | 2 +- .../src/solve/fulfill.rs | 18 ++++- 4 files changed, 71 insertions(+), 50 deletions(-) diff --git a/compiler/rustc_next_trait_solver/src/solve/eval_ctxt/fast_path.rs b/compiler/rustc_next_trait_solver/src/solve/eval_ctxt/fast_path.rs index 978d97f184797..6a7d49b36b64f 100644 --- a/compiler/rustc_next_trait_solver/src/solve/eval_ctxt/fast_path.rs +++ b/compiler/rustc_next_trait_solver/src/solve/eval_ctxt/fast_path.rs @@ -5,19 +5,16 @@ //! //! For debugging, fast paths can be disabled using `-Zdisable-fast-paths`. -use rustc_type_ir::InferCtxtLike; -use rustc_type_ir::Interner; use rustc_type_ir::inherent::*; -use rustc_type_ir::solve::Goal; use rustc_type_ir::solve::{ - Certainty, ComputeGoalFastPathOutcome, GoalStalledOn, GoalStalledOnReason, SucceededInErased, + Certainty, ComputeGoalFastPathOutcome, Goal, GoalStalledOn, GoalStalledOnReason, + SucceededInErased, }; +use rustc_type_ir::{InferCtxtLike, Interner}; use crate::delegate::SolverDelegate; -use crate::solve::GoalEvaluation; -use crate::solve::HasChanged; -use crate::solve::eval_ctxt::RerunDecision; -use crate::solve::eval_ctxt::should_rerun_after_erased_canonicalization; +use crate::solve::eval_ctxt::{RerunDecision, should_rerun_after_erased_canonicalization}; +use crate::solve::{GoalEvaluation, HasChanged}; #[derive(Debug, Clone, Copy)] pub(super) enum RerunStalled { @@ -29,6 +26,7 @@ pub(super) enum RerunStalled { /// args have changed. This is a cheap way to determine that if we were to rerun this goal now, /// it will remain stalled since it'll canonicalize the same way and evaluation is pure. /// Therefore, we can skip this rerun +#[inline] pub(super) fn rerunning_stalled_goal_may_make_progress( delegate: &D, stalled_on: Option<&GoalStalledOn>, @@ -106,9 +104,23 @@ where WontMakeProgress(stalled_certainty) } +#[cold] +#[inline(never)] +pub(super) fn compute_goal_fast_path_cold( + delegate: &D, + goal: Goal, + origin_span: I::Span, +) -> Option> +where + D: SolverDelegate, + I: Interner, +{ + compute_goal_fast_path(delegate, goal, origin_span) +} + /// This is a fast path optimization: /// See the docs on [`ComputeGoalFastPathOutcome`] -pub(super) fn compute_goal_fast_path( +pub fn compute_goal_fast_path( delegate: &D, goal: Goal, origin_span: I::Span, diff --git a/compiler/rustc_next_trait_solver/src/solve/eval_ctxt/mod.rs b/compiler/rustc_next_trait_solver/src/solve/eval_ctxt/mod.rs index ffcd388208fdc..949cf3b4a8086 100644 --- a/compiler/rustc_next_trait_solver/src/solve/eval_ctxt/mod.rs +++ b/compiler/rustc_next_trait_solver/src/solve/eval_ctxt/mod.rs @@ -34,6 +34,7 @@ use crate::resolve::eager_resolve_vars; use crate::solve::eval_ctxt::fast_path::{ RerunStalled, compute_goal_fast_path, rerunning_stalled_goal_may_make_progress, }; +use crate::solve::fast_path::compute_goal_fast_path_cold; use crate::solve::search_graph::SearchGraph; use crate::solve::ty::may_use_unstable_feature; use crate::solve::{ @@ -43,7 +44,7 @@ use crate::solve::{ VisibleForLeakCheck, inspect, }; -mod fast_path; +pub mod fast_path; mod probe; mod solver_region_constraints; @@ -236,7 +237,13 @@ where }); } - if let Some(res) = compute_goal_fast_path(self, goal, span) { + if + // No need to try the fast path if stalled_on is `None`, since we already try the fast path + // immediately when adding new goals. If we didn't check `stalled_on` here we'd be trying + // the fast path twice for some goals. + stalled_on.is_some() + && let Some(res) = compute_goal_fast_path_cold(self, goal, span) + { return Ok(res); } @@ -503,7 +510,13 @@ where }); } - if let Some(res) = compute_goal_fast_path(self.delegate, goal, self.origin_span) { + if + // No need to try the fast path if stalled_on is `None`, since we already try the fast path + // immediately when adding new goals. If we didn't check `stalled_on` here we'd be trying + // the fast path twice for some goals. + stalled_on.is_some() + && let Some(res) = compute_goal_fast_path_cold(self.delegate, goal, self.origin_span) + { return Ok(res); } @@ -519,7 +532,8 @@ where goal: Goal, ) -> Result, NoSolutionOrRerunNonErased> { let (normalization_nested_goals, goal_evaluation) = - self.evaluate_goal_raw(source, goal, LowerAvailableDepth::Yes)?; + self.evaluate_goal_raw(source, goal, LowerAvailableDepth::Yes)?; + assert!(normalization_nested_goals.is_empty()); Ok(goal_evaluation) } @@ -536,40 +550,6 @@ where goal: Goal, increase_depth_for_nested: LowerAvailableDepth, ) -> Result<(NestedNormalizationGoals, GoalEvaluation), NoSolutionOrRerunNonErased> { - if let RerunStalled::WontMakeProgress(stalled_certainty) = - self.rerunning_stalled_goal_may_make_progress(stalled_on.as_ref()) - { - return Ok(( - NestedNormalizationGoals::empty(), - GoalEvaluation { - goal, - certainty: stalled_certainty, - has_changed: HasChanged::No, - stalled_on, - }, - )); - } - - self.evaluate_goal_cold(source, goal, increase_depth_for_nested) - } - - #[cold] - #[inline(never)] - pub(super) fn evaluate_goal_cold( - &mut self, - source: GoalSource, - goal: Goal, -- increase_depth_for_nested: IncreaseDepthForNested, -+ increase_depth_for_nested: LowerAvailableDepth, - ) -> Result<(NestedNormalizationGoals, GoalEvaluation), NoSolutionOrRerunNonErased> { - if let Some(res) = self.compute_goal_fast_path(goal) { - return Ok(res); - } - -+++++++ ktlxysvk c626102e "shuffle fastpaths around to before creating an evalctxt" (rebased revision) - increase_depth_for_nested: IncreaseDepthForNested, - ) -> Result<(NestedNormalizationGoals, GoalEvaluation), NoSolutionOrRerunNonErased> { ->>>>>>> conflict 2 of 2 ends // We only care about one entry per `OpaqueTypeKey` here, // so we only canonicalize the lookup table and ignore // duplicate entries. @@ -915,7 +895,20 @@ where ty::Unnormalized::new_wip(goal.predicate), )?; self.inspect.add_goal(self.delegate, self.max_input_universe, source, goal); - self.nested_goals.push((source, goal, None)); + + if let Some(GoalEvaluation { goal, certainty, has_changed: _, stalled_on }) = + compute_goal_fast_path(self.delegate, goal, self.origin_span) + { + match certainty { + // We're done here + Certainty::Yes => {} + Certainty::Maybe(_) => { + self.nested_goals.push((source, goal, stalled_on)); + } + } + } else { + self.nested_goals.push((source, goal, None)); + } Ok(()) } diff --git a/compiler/rustc_next_trait_solver/src/solve/mod.rs b/compiler/rustc_next_trait_solver/src/solve/mod.rs index 100ef083ac727..452a87a89a7f1 100644 --- a/compiler/rustc_next_trait_solver/src/solve/mod.rs +++ b/compiler/rustc_next_trait_solver/src/solve/mod.rs @@ -28,7 +28,7 @@ use tracing::instrument; pub use self::eval_ctxt::{ EvalCtxt, GenerateProofTree, SolverDelegateEvalExt, - evaluate_root_goal_for_proof_tree_raw_provider, + evaluate_root_goal_for_proof_tree_raw_provider, fast_path, }; use crate::delegate::SolverDelegate; use crate::solve::assembly::Candidate; diff --git a/compiler/rustc_trait_selection/src/solve/fulfill.rs b/compiler/rustc_trait_selection/src/solve/fulfill.rs index 0944da1f93dfe..4c2c92ebc5072 100644 --- a/compiler/rustc_trait_selection/src/solve/fulfill.rs +++ b/compiler/rustc_trait_selection/src/solve/fulfill.rs @@ -7,6 +7,7 @@ use rustc_infer::traits::{ FromSolverError, PredicateObligation, PredicateObligations, TraitEngine, }; use rustc_middle::ty::{self, TyCtxt, TyVid, TypeVisitableExt, TypingMode}; +use rustc_next_trait_solver::solve::fast_path::compute_goal_fast_path; use rustc_next_trait_solver::solve::{ GoalEvaluation, GoalStalledOn, HasChanged, MaybeInfo, SolverDelegateEvalExt as _, StalledOnCoroutines, @@ -184,7 +185,22 @@ where obligation: PredicateObligation<'tcx>, ) { assert_eq!(self.usable_in_snapshot, infcx.num_open_snapshots()); - self.obligations.register(obligation, None); + + let delegate = <&SolverDelegate<'tcx>>::from(infcx); + if let Some(GoalEvaluation { goal: _, certainty, has_changed: _, stalled_on }) = + compute_goal_fast_path(delegate, obligation.as_goal(), obligation.cause.span) + { + // If we can take the fast path, don't even bother adding the goal to obligations, + // or if `Certainty::Maybe`, add it with precise stalled_on information. + match certainty { + Certainty::Yes => {} + Certainty::Maybe(_) => { + self.obligations.register(obligation, stalled_on); + } + } + } else { + self.obligations.register(obligation, None); + } } fn collect_remaining_errors(&mut self, infcx: &InferCtxt<'tcx>) -> Vec { From 23b11abe829f0a1525cb2ed7441fe3c0751d6b58 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jana=20D=C3=B6nszelmann?= Date: Thu, 9 Jul 2026 10:18:22 +0200 Subject: [PATCH 9/9] fix bug when fast paths aren't rerun when opaques change --- compiler/rustc_middle/src/traits/solve.rs | 2 +- .../src/solve/eval_ctxt/fast_path.rs | 20 +++--- .../src/solve/eval_ctxt/mod.rs | 10 ++- .../src/solve/delegate.rs | 66 +++++++++++++------ compiler/rustc_type_ir/src/solve/mod.rs | 10 +-- 5 files changed, 70 insertions(+), 38 deletions(-) diff --git a/compiler/rustc_middle/src/traits/solve.rs b/compiler/rustc_middle/src/traits/solve.rs index c45ebb80255cc..c424ca71c0801 100644 --- a/compiler/rustc_middle/src/traits/solve.rs +++ b/compiler/rustc_middle/src/traits/solve.rs @@ -18,7 +18,7 @@ pub type FetchEligibleAssocItemResponse<'tcx> = ir::solve::FetchEligibleAssocItemResponse>; pub type ComputeGoalFastPathOutcome<'tcx> = ir::solve::ComputeGoalFastPathOutcome>; pub type GoalStalledOn<'tcx> = ir::solve::GoalStalledOn>; -pub type GoalStalledOnReason<'tcx> = ir::solve::GoalStalledOnReason>; +pub type GoalStalledOnOpaques<'tcx> = ir::solve::GoalStalledOnOpaques>; pub type SucceededInErased<'tcx> = ir::solve::SucceededInErased>; pub type PredefinedOpaques<'tcx> = &'tcx ty::List<(ty::OpaqueTypeKey<'tcx>, Ty<'tcx>)>; diff --git a/compiler/rustc_next_trait_solver/src/solve/eval_ctxt/fast_path.rs b/compiler/rustc_next_trait_solver/src/solve/eval_ctxt/fast_path.rs index 6a7d49b36b64f..a6c2bce88b031 100644 --- a/compiler/rustc_next_trait_solver/src/solve/eval_ctxt/fast_path.rs +++ b/compiler/rustc_next_trait_solver/src/solve/eval_ctxt/fast_path.rs @@ -7,7 +7,7 @@ use rustc_type_ir::inherent::*; use rustc_type_ir::solve::{ - Certainty, ComputeGoalFastPathOutcome, Goal, GoalStalledOn, GoalStalledOnReason, + Certainty, ComputeGoalFastPathOutcome, Goal, GoalStalledOn, GoalStalledOnOpaques, SucceededInErased, }; use rustc_type_ir::{InferCtxtLike, Interner}; @@ -43,7 +43,7 @@ where } // If the goal isn't stalled, we should definitely run it. - let Some(&GoalStalledOn { ref reason, ref stalled_vars, ref sub_roots, stalled_certainty }) = + let Some(&GoalStalledOn { ref opaques, ref stalled_vars, ref sub_roots, stalled_certainty }) = stalled_on else { return MayMakeProgress; @@ -61,14 +61,18 @@ where return MayMakeProgress; } - match reason { - GoalStalledOnReason::FastPath => { - // fastpath is never because of opaques, we can skip this check - } - &GoalStalledOnReason::Other { num_opaques, ref previously_succeeded_in_erased } => { + match opaques { + GoalStalledOnOpaques::No => {} + &GoalStalledOnOpaques::Yes { + num_opaques_in_storage, + ref previously_succeeded_in_erased, + } => { // If any opaques changed in the opaque type storage, // rerunning might make progress so we should rerun. - if delegate.opaque_types_storage_num_entries().needs_reevaluation(num_opaques) { + if delegate + .opaque_types_storage_num_entries() + .needs_reevaluation(num_opaques_in_storage) + { // Unless this goal previously succeeded in erased mode. // If the stalled goal successfully evaluated while erasing opaque types, // and the current state of the opaque type storage is not different in a way that is diff --git a/compiler/rustc_next_trait_solver/src/solve/eval_ctxt/mod.rs b/compiler/rustc_next_trait_solver/src/solve/eval_ctxt/mod.rs index 949cf3b4a8086..1fc95fca1f38f 100644 --- a/compiler/rustc_next_trait_solver/src/solve/eval_ctxt/mod.rs +++ b/compiler/rustc_next_trait_solver/src/solve/eval_ctxt/mod.rs @@ -39,7 +39,7 @@ use crate::solve::search_graph::SearchGraph; use crate::solve::ty::may_use_unstable_feature; use crate::solve::{ CanonicalInput, CanonicalResponse, Certainty, ExternalConstraintsData, FIXPOINT_STEP_LIMIT, - Goal, GoalEvaluation, GoalSource, GoalStalledOn, GoalStalledOnReason, HasChanged, MaybeCause, + Goal, GoalEvaluation, GoalSource, GoalStalledOn, GoalStalledOnOpaques, HasChanged, MaybeCause, NestedNormalizationGoals, NoSolution, QueryInput, QueryResult, Response, SucceededInErased, VisibleForLeakCheck, inspect, }; @@ -753,8 +753,12 @@ where stalled_vars, sub_roots, stalled_certainty: certainty, - reason: GoalStalledOnReason::Other { - num_opaques: canonical_goal.canonical.value.predefined_opaques_in_body.len(), + opaques: GoalStalledOnOpaques::Yes { + num_opaques_in_storage: canonical_goal + .canonical + .value + .predefined_opaques_in_body + .len(), previously_succeeded_in_erased, }, } diff --git a/compiler/rustc_trait_selection/src/solve/delegate.rs b/compiler/rustc_trait_selection/src/solve/delegate.rs index 913cf28cd2426..32aaa03de646e 100644 --- a/compiler/rustc_trait_selection/src/solve/delegate.rs +++ b/compiler/rustc_trait_selection/src/solve/delegate.rs @@ -11,14 +11,14 @@ use rustc_infer::infer::canonical::{ }; use rustc_infer::infer::{InferCtxt, RegionVariableOrigin, SubregionOrigin, TyCtxtInferExt}; use rustc_infer::traits::solve::{ - ComputeGoalFastPathOutcome, FetchEligibleAssocItemResponse, Goal, + ComputeGoalFastPathOutcome, FetchEligibleAssocItemResponse, Goal, SucceededInErased, }; use rustc_middle::traits::query::NoSolution; use rustc_middle::traits::solve::Certainty; use rustc_middle::ty::{ self, MayBeErased, Ty, TyCtxt, TypeFlags, TypeFoldable, TypeVisitableExt, TypingMode, }; -use rustc_next_trait_solver::solve::{GoalStalledOn, GoalStalledOnReason}; +use rustc_next_trait_solver::solve::{GoalStalledOn, GoalStalledOnOpaques}; use rustc_span::{DUMMY_SP, Span}; use crate::traits::{EvaluateConstErr, ObligationCause, sizedness_fast_path, specialization_graph}; @@ -50,6 +50,43 @@ impl<'tcx> SolverDelegate<'tcx> { } } +/// Create a [`ComputeGoalFastPathOutcome`] signalling the goal is stalled +/// on a list of [`ty::GenericArg`] +fn goal_stalled_on_args<'tcx>( + stalled_vars: Vec>, +) -> ComputeGoalFastPathOutcome<'tcx> { + ComputeGoalFastPathOutcome::TriviallyStalled { + stalled_on: GoalStalledOn { + stalled_vars, + sub_roots: Vec::new(), + stalled_certainty: Certainty::AMBIGUOUS, + opaques: GoalStalledOnOpaques::No, + }, + } +} + +/// Create a [`ComputeGoalFastPathOutcome`] signalling the goal is stalled +/// on a list of [`ty::GenericArg`] *or* the opaque type storage being nonempty. +/// +fn goal_stalled_on_args_or_nonempty_opaques<'tcx>( + stalled_vars: Vec>, +) -> ComputeGoalFastPathOutcome<'tcx> { + ComputeGoalFastPathOutcome::TriviallyStalled { + stalled_on: GoalStalledOn { + stalled_vars, + sub_roots: Vec::new(), + stalled_certainty: Certainty::AMBIGUOUS, + opaques: GoalStalledOnOpaques::Yes { + num_opaques_in_storage: 0, + // This function should only be called when not in erased mode, + // otherwise this is wrong. The `compute_goal_fast_path` does this + // through `known_no_opaque_types_in_storage` + previously_succeeded_in_erased: SucceededInErased::No, + }, + }, + } +} + impl<'tcx> rustc_next_trait_solver::delegate::SolverDelegate for SolverDelegate<'tcx> { type Infcx = InferCtxt<'tcx>; type Interner = TyCtxt<'tcx>; @@ -79,19 +116,6 @@ impl<'tcx> rustc_next_trait_solver::delegate::SolverDelegate for SolverDelegate< ) -> ComputeGoalFastPathOutcome<'tcx> { use ComputeGoalFastPathOutcome as Outcome; - fn stalled_with_args<'tcx>( - stalled_vars: Vec>, - ) -> ComputeGoalFastPathOutcome<'tcx> { - Outcome::TriviallyStalled { - stalled_on: GoalStalledOn { - stalled_vars, - sub_roots: Vec::new(), - stalled_certainty: Certainty::AMBIGUOUS, - reason: GoalStalledOnReason::FastPath, - }, - } - } - // FIXME(-Zassumptions-on-binders): actually handle fast path if self.tcx.assumptions_on_binders() { return Outcome::NoFastPath; @@ -110,13 +134,13 @@ impl<'tcx> rustc_next_trait_solver::delegate::SolverDelegate for SolverDelegate< // FIXME: Properly consider opaques here. && self.known_no_opaque_types_in_storage() { - stalled_with_args(vec![self_ty.into()]) + goal_stalled_on_args_or_nonempty_opaques(vec![self_ty.into()]) } else if trait_pred.polarity() == ty::PredicatePolarity::Positive { match self.0.tcx.as_lang_item(trait_pred.def_id()) { Some(LangItem::Sized) | Some(LangItem::MetaSized) => { let predicate = self.resolve_vars_if_possible(goal.predicate); if sizedness_fast_path(self.tcx, predicate, goal.param_env) { - return Outcome::TriviallyHolds; + Outcome::TriviallyHolds } else { Outcome::NoFastPath } @@ -133,7 +157,7 @@ impl<'tcx> rustc_next_trait_solver::delegate::SolverDelegate for SolverDelegate< .has_type_flags(TypeFlags::HAS_FREE_REGIONS | TypeFlags::HAS_INFER) && self_ty.is_trivially_pure_clone_copy() { - return Outcome::TriviallyHolds; + Outcome::TriviallyHolds } else { Outcome::NoFastPath } @@ -182,7 +206,7 @@ impl<'tcx> rustc_next_trait_solver::delegate::SolverDelegate for SolverDelegate< match (self.shallow_resolve(a).kind(), self.shallow_resolve(b).kind()) { (&ty::Infer(ty::TyVar(a_vid)), &ty::Infer(ty::TyVar(b_vid))) => { self.sub_unify_ty_vids_raw(a_vid, b_vid); - stalled_with_args(vec![a.into(), b.into()]) + goal_stalled_on_args(vec![a.into(), b.into()]) } _ => Outcome::NoFastPath, } @@ -194,7 +218,7 @@ impl<'tcx> rustc_next_trait_solver::delegate::SolverDelegate for SolverDelegate< let arg = self.shallow_resolve_const(ct); if arg.is_ct_infer() { - stalled_with_args(vec![arg.into()]) + goal_stalled_on_args(vec![arg.into()]) } else { Outcome::NoFastPath } @@ -208,7 +232,7 @@ impl<'tcx> rustc_next_trait_solver::delegate::SolverDelegate for SolverDelegate< if arg.is_trivially_wf(self.tcx) { Outcome::TriviallyHolds } else if arg.is_infer() { - stalled_with_args(vec![arg.into_arg()]) + goal_stalled_on_args(vec![arg.into_arg()]) } else { Outcome::NoFastPath } diff --git a/compiler/rustc_type_ir/src/solve/mod.rs b/compiler/rustc_type_ir/src/solve/mod.rs index 5b07a23387ecb..5c8f1bab46b94 100644 --- a/compiler/rustc_type_ir/src/solve/mod.rs +++ b/compiler/rustc_type_ir/src/solve/mod.rs @@ -954,13 +954,13 @@ pub enum SucceededInErased { } #[derive_where(Clone, Debug; I: Interner)] -pub enum GoalStalledOnReason { +pub enum GoalStalledOnOpaques { /// This goal got stalled in `compute_goal_fast_path`. Usually this means /// the goal is stalled on not that much, only one or two variables, and /// definitely nothing to do with opaque types. So we don't store that information. - FastPath, - Other { - num_opaques: usize, + No, + Yes { + num_opaques_in_storage: usize, previously_succeeded_in_erased: SucceededInErased, }, } @@ -973,7 +973,7 @@ pub struct GoalStalledOn { /// The certainty that will be returned on subsequent evaluations if this /// goal remains stalled. pub stalled_certainty: Certainty, - pub reason: GoalStalledOnReason, + pub opaques: GoalStalledOnOpaques, } /// For some goals we can trivially answer some questions without going through