Avoid redundant next-solver fulfillment scans

compiler/rustc_infer/src/infer/canonical/query_response.rs

@@ -155,13 +155,7 @@ impl<'tcx> InferCtxt<'tcx> {
         });
         debug!(?region_constraints);
 
-        let opaque_types = self
-            .inner
-            .borrow_mut()
-            .opaque_type_storage
-            .take_opaque_types()
-            .map(|(k, v)| (k, v.ty))
-            .collect();
+        let opaque_types = self.take_opaque_types().into_iter().map(|(k, v)| (k, v.ty)).collect();
 
         Ok(QueryResponse {
             var_values: inference_vars,

compiler/rustc_infer/src/infer/context.rs

@@ -195,38 +195,35 @@ impl<'tcx> rustc_type_ir::InferCtxtLike for InferCtxt<'tcx> {
     }
 
     fn equate_ty_vids_raw(&self, a: ty::TyVid, b: ty::TyVid) {
-        self.inner.borrow_mut().type_variables().equate(a, b);
+        self.inner.borrow_mut().equate_ty_vids(a, b);
     }
 
     fn sub_unify_ty_vids_raw(&self, a: ty::TyVid, b: ty::TyVid) {
         self.sub_unify_ty_vids_raw(a, b);
     }
 
     fn equate_int_vids_raw(&self, a: ty::IntVid, b: ty::IntVid) {
-        self.inner.borrow_mut().int_unification_table().union(a, b);
+        self.inner.borrow_mut().equate_int_vids(a, b);
     }
 
     fn equate_float_vids_raw(&self, a: ty::FloatVid, b: ty::FloatVid) {
-        self.inner.borrow_mut().float_unification_table().union(a, b);
+        self.inner.borrow_mut().equate_float_vids(a, b);
     }
 
     fn equate_const_vids_raw(&self, a: ty::ConstVid, b: ty::ConstVid) {
-        self.inner.borrow_mut().const_unification_table().union(a, b);
+        self.inner.borrow_mut().equate_const_vids(a, b);
     }
 
     fn instantiate_ty_var_raw(&self, vid: ty::TyVid, ty: Ty<'tcx>) {
         let ty = lower_universe(self, self.try_resolve_ty_var(vid).unwrap_err(), ty);
 
-        self.inner.borrow_mut().type_variables().instantiate(vid, ty);
+        self.inner.borrow_mut().instantiate_ty_var(vid, ty);
     }
 
     fn instantiate_const_var_raw(&self, vid: ty::ConstVid, ct: ty::Const<'tcx>) {
         let ct = lower_universe(self, self.try_resolve_const_var(vid).unwrap_err(), ct);
 
-        self.inner
-            .borrow_mut()
-            .const_unification_table()
-            .union_value(vid, ConstVariableValue::Known { value: ct });
+        self.inner.borrow_mut().instantiate_const_var(vid, ct);
     }
 
     fn instantiate_ty_var<R: PredicateEmittingRelation<Self>>(
@@ -247,11 +244,11 @@ impl<'tcx> rustc_type_ir::InferCtxtLike for InferCtxt<'tcx> {
     }
 
     fn instantiate_int_var_raw(&self, vid: ty::IntVid, value: ty::IntVarValue) {
-        self.inner.borrow_mut().int_unification_table().union_value(vid, value);
+        self.inner.borrow_mut().instantiate_int_var(vid, value);
     }
 
     fn instantiate_float_var_raw(&self, vid: ty::FloatVid, value: ty::FloatVarValue) {
-        self.inner.borrow_mut().float_unification_table().union_value(vid, value);
+        self.inner.borrow_mut().instantiate_float_var(vid, value);
     }
 
     fn instantiate_const_var<R: PredicateEmittingRelation<Self>>(
@@ -380,10 +377,10 @@ impl<'tcx> rustc_type_ir::InferCtxtLike for InferCtxt<'tcx> {
         hidden_ty: Ty<'tcx>,
         span: Span,
     ) {
-        self.inner
-            .borrow_mut()
-            .opaque_types()
-            .add_duplicate(opaque_type_key, ty::ProvisionalHiddenType { span, ty: hidden_ty })
+        self.inner.borrow_mut().add_duplicate_opaque_type(
+            opaque_type_key,
+            ty::ProvisionalHiddenType { span, ty: hidden_ty },
+        )
     }
 
     fn reset_opaque_types(&self) {
@@ -461,7 +458,7 @@ impl<'a, 'tcx> ty::TypeFolder<TyCtxt<'tcx>> for LowerUniverseFolder<'a, 'tcx> {
                             let origin = inner.type_variables().var_origin(vid);
                             let new_var_id =
                                 inner.type_variables().new_var(self.for_universe, origin);
-                            inner.type_variables().equate(vid, new_var_id);
+                            inner.equate_ty_vids(vid, new_var_id);
                             Ty::new_var(self.cx(), new_var_id)
                         }
                     }
@@ -501,7 +498,7 @@ impl<'a, 'tcx> ty::TypeFolder<TyCtxt<'tcx>> for LowerUniverseFolder<'a, 'tcx> {
                         })
                         .vid;
 
-                    self.infcx.inner.borrow_mut().const_unification_table().union(vid, new_var_id);
+                    self.infcx.inner.borrow_mut().equate_const_vids(vid, new_var_id);
 
                     ty::Const::new_var(self.cx(), new_var_id)
                 }

compiler/rustc_infer/src/infer/mod.rs

@@ -97,6 +97,11 @@ pub(crate) type UnificationTable<'a, 'tcx, T> = ut::UnificationTable<
 pub struct InferCtxtInner<'tcx> {
     undo_log: InferCtxtUndoLogs<'tcx>,
 
+    /// Bumped whenever an inference change may let a stalled fulfillment goal
+    /// make progress. Snapshots save and restore the value, but individual bumps
+    /// are not undo-log entries.
+    stalled_goal_generation: Option<u64>,
+
     /// Cache for projections.
     ///
     /// This cache is snapshotted along with the infcx.
@@ -171,9 +176,10 @@ pub struct InferCtxtInner<'tcx> {
 }
 
 impl<'tcx> InferCtxtInner<'tcx> {
-    fn new() -> InferCtxtInner<'tcx> {
+    fn new(next_trait_solver: bool) -> InferCtxtInner<'tcx> {
         InferCtxtInner {
             undo_log: InferCtxtUndoLogs::default(),
+            stalled_goal_generation: next_trait_solver.then_some(0),
 
             projection_cache: Default::default(),
             type_variable_storage: Default::default(),
@@ -222,6 +228,26 @@ impl<'tcx> InferCtxtInner<'tcx> {
         self.opaque_type_storage.with_log(&mut self.undo_log)
     }
 
+    #[inline]
+    fn register_opaque_type(
+        &mut self,
+        key: OpaqueTypeKey<'tcx>,
+        hidden_type: ProvisionalHiddenType<'tcx>,
+    ) -> Option<Ty<'tcx>> {
+        self.bump_stalled_goal_generation();
+        self.opaque_types().register(key, hidden_type)
+    }
+
+    #[inline]
+    fn add_duplicate_opaque_type(
+        &mut self,
+        key: OpaqueTypeKey<'tcx>,
+        hidden_type: ProvisionalHiddenType<'tcx>,
+    ) {
+        self.bump_stalled_goal_generation();
+        self.opaque_types().add_duplicate(key, hidden_type);
+    }
+
     #[inline]
     fn int_unification_table(&mut self) -> UnificationTable<'_, 'tcx, ty::IntVid> {
         self.int_unification_storage.with_log(&mut self.undo_log)
@@ -237,6 +263,79 @@ impl<'tcx> InferCtxtInner<'tcx> {
         self.const_unification_storage.with_log(&mut self.undo_log)
     }
 
+    #[inline]
+    pub(crate) fn start_snapshot(&mut self) -> snapshot::undo_log::Snapshot<'tcx> {
+        self.undo_log.start_snapshot(self.stalled_goal_generation)
+    }
+
+    #[inline]
+    fn stalled_goal_generation(&self) -> Option<u64> {
+        self.stalled_goal_generation
+    }
+
+    #[inline]
+    fn bump_stalled_goal_generation(&mut self) {
+        if let Some(generation) = &mut self.stalled_goal_generation {
+            *generation = generation.wrapping_add(1);
+        }
+    }
+
+    #[inline]
+    fn equate_ty_vids(&mut self, a: ty::TyVid, b: ty::TyVid) {
+        self.bump_stalled_goal_generation();
+        self.type_variables().equate(a, b);
+    }
+
+    #[inline]
+    fn sub_unify_ty_vids(&mut self, a: ty::TyVid, b: ty::TyVid) {
+        self.bump_stalled_goal_generation();
+        self.type_variables().sub_unify(a, b);
+    }
+
+    #[inline]
+    fn instantiate_ty_var(&mut self, vid: ty::TyVid, ty: Ty<'tcx>) {
+        self.bump_stalled_goal_generation();
+        self.type_variables().instantiate(vid, ty);
+    }
+
+    // These mutations can unblock stalled goals too, so route them through the
+    // same generation bump.
+    #[inline]
+    fn equate_int_vids(&mut self, a: ty::IntVid, b: ty::IntVid) {
+        self.bump_stalled_goal_generation();
+        self.int_unification_table().union(a, b);
+    }
+
+    #[inline]
+    fn equate_float_vids(&mut self, a: ty::FloatVid, b: ty::FloatVid) {
+        self.bump_stalled_goal_generation();
+        self.float_unification_table().union(a, b);
+    }
+
+    #[inline]
+    fn equate_const_vids(&mut self, a: ty::ConstVid, b: ty::ConstVid) {
+        self.bump_stalled_goal_generation();
+        self.const_unification_table().union(a, b);
+    }
+
+    #[inline]
+    fn instantiate_int_var(&mut self, vid: ty::IntVid, value: ty::IntVarValue) {
+        self.bump_stalled_goal_generation();
+        self.int_unification_table().union_value(vid, value);
+    }
+
+    #[inline]
+    fn instantiate_float_var(&mut self, vid: ty::FloatVid, value: ty::FloatVarValue) {
+        self.bump_stalled_goal_generation();
+        self.float_unification_table().union_value(vid, value);
+    }
+
+    #[inline]
+    fn instantiate_const_var(&mut self, vid: ty::ConstVid, value: ty::Const<'tcx>) {
+        self.bump_stalled_goal_generation();
+        self.const_unification_table().union_value(vid, ConstVariableValue::Known { value });
+    }
+
     #[inline]
     pub fn unwrap_region_constraints(&mut self) -> RegionConstraintCollector<'_, 'tcx> {
         self.region_constraint_storage
@@ -684,7 +783,7 @@ impl<'tcx> InferCtxtBuilder<'tcx> {
             considering_regions,
             in_hir_typeck,
             skip_leak_check,
-            inner: RefCell::new(InferCtxtInner::new()),
+            inner: RefCell::new(InferCtxtInner::new(next_trait_solver)),
             lexical_region_resolutions: RefCell::new(None),
             selection_cache: Default::default(),
             evaluation_cache: Default::default(),
@@ -1121,7 +1220,13 @@ impl<'tcx> InferCtxt<'tcx> {
 
     #[instrument(level = "debug", skip(self), ret)]
     pub fn take_opaque_types(&self) -> Vec<(OpaqueTypeKey<'tcx>, ProvisionalHiddenType<'tcx>)> {
-        self.inner.borrow_mut().opaque_type_storage.take_opaque_types().collect()
+        let inner = &mut *self.inner.borrow_mut();
+
+        if !inner.opaque_type_storage.is_empty() {
+            inner.bump_stalled_goal_generation();
+        }
+
+        inner.opaque_type_storage.take_opaque_types().collect()
     }
 
     #[instrument(level = "debug", skip(self), ret)]
@@ -1379,7 +1484,7 @@ impl<'tcx> InferCtxt<'tcx> {
     }
 
     pub fn sub_unify_ty_vids_raw(&self, a: ty::TyVid, b: ty::TyVid) {
-        self.inner.borrow_mut().type_variables().sub_unify(a, b);
+        self.inner.borrow_mut().sub_unify_ty_vids(a, b);
     }
 
     pub fn sub_unification_table_root_var(&self, var: ty::TyVid) -> ty::TyVid {
@@ -1713,6 +1818,14 @@ impl<'tcx> InferCtxt<'tcx> {
         self.typing_env(param_env).as_query_input(value)
     }
 
+    #[inline]
+    pub fn stalled_goal_generation(&self) -> u64 {
+        self.inner
+            .borrow()
+            .stalled_goal_generation()
+            .expect("stalled-goal generation requires the next trait solver")
+    }
+
     /// The returned function is used in a fast path. If it returns `true` the variable is
     /// unchanged, `false` indicates that the status is unknown.
     #[inline]

compiler/rustc_infer/src/infer/opaque_types/mod.rs

@@ -204,7 +204,7 @@ impl<'tcx> InferCtxt<'tcx> {
         opaque_type_key: OpaqueTypeKey<'tcx>,
         hidden_ty: ProvisionalHiddenType<'tcx>,
     ) -> Option<Ty<'tcx>> {
-        self.inner.borrow_mut().opaque_types().register(opaque_type_key, hidden_ty)
+        self.inner.borrow_mut().register_opaque_type(opaque_type_key, hidden_ty)
     }
 
     /// Insert a hidden type into the opaque type storage, equating it
@@ -238,11 +238,10 @@ impl<'tcx> InferCtxt<'tcx> {
                 goals.push(Goal::new(tcx, param_env, ty::PredicateKind::Ambiguous));
             }
             ty::TypingMode::Typeck { .. } => {
-                let prev = self
-                    .inner
-                    .borrow_mut()
-                    .opaque_types()
-                    .register(opaque_type_key, ProvisionalHiddenType { ty: hidden_ty, span });
+                let prev = self.inner.borrow_mut().register_opaque_type(
+                    opaque_type_key,
+                    ProvisionalHiddenType { ty: hidden_ty, span },
+                );
                 if let Some(prev) = prev {
                     goals.extend(
                         self.at(&ObligationCause::dummy_with_span(span), param_env)
@@ -254,11 +253,10 @@ impl<'tcx> InferCtxt<'tcx> {
                 }
             }
             ty::TypingMode::PostTypeckUntilBorrowck { .. } => {
-                let prev = self
-                    .inner
-                    .borrow_mut()
-                    .opaque_types()
-                    .register(opaque_type_key, ProvisionalHiddenType { ty: hidden_ty, span });
+                let prev = self.inner.borrow_mut().register_opaque_type(
+                    opaque_type_key,
+                    ProvisionalHiddenType { ty: hidden_ty, span },
+                );
 
                 // We either equate the new hidden type with the previous entry or with the type
                 // inferred by HIR typeck.