From 1ee381331c2e1324a377a8797fd5c94bb2c41643 Mon Sep 17 00:00:00 2001 From: Miguel Marques Date: Fri, 29 May 2026 14:07:48 +0100 Subject: [PATCH 1/5] feat: Improves enum niche-filling variant packing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This feature only works with "-Zunstable-options --edition future" as it changes the rustc_abi. When a variant cannot fit as a whole before or after the reserved niche, try repacking its fields around the niche instead. This makes niche-filling less conservative while preserving the existing layout behavior when whole-variant placement already succeeds. Signed-off-by: Miguel Marques Co-authored-by: Vicente Gusmão --- compiler/rustc_abi/src/layout.rs | 308 +++++++++++++++++- compiler/rustc_abi/src/lib.rs | 10 +- compiler/rustc_middle/src/ty/mod.rs | 4 + .../enum-niche-fill-around-tag.rs | 140 ++++++++ 4 files changed, 460 insertions(+), 2 deletions(-) create mode 100644 tests/ui/structs-enums/enum-niche-fill-around-tag.rs diff --git a/compiler/rustc_abi/src/layout.rs b/compiler/rustc_abi/src/layout.rs index 733a1956231b1..9d1ee4120524d 100644 --- a/compiler/rustc_abi/src/layout.rs +++ b/compiler/rustc_abi/src/layout.rs @@ -742,7 +742,198 @@ impl LayoutCalculator { Some(layout) }; - let niche_filling_layout = calculate_niche_filling_layout(); + let calculate_niche_filling_layout_repacked = + || -> Option> { + struct VariantLayoutInfo { + align_abi: Align, + } + + if repr.inhibit_enum_layout_opt() { + return None; + } + + if variants.len() < 2 { + return None; + } + + let mut align = dl.aggregate_align; + let mut max_repr_align = repr.align; + let mut unadjusted_abi_align = align; + let mut combined_seed = repr.field_shuffle_seed; + + let mut variants_info = IndexVec::::with_capacity(variants.len()); + let variant_layouts = variants + .iter() + .map(|v| { + let st = self.univariant(v, repr, StructKind::AlwaysSized).ok()?; + + variants_info.push(VariantLayoutInfo { align_abi: st.align.abi }); + + align = align.max(st.align.abi); + max_repr_align = max_repr_align.max(st.max_repr_align); + unadjusted_abi_align = unadjusted_abi_align.max(st.unadjusted_abi_align); + combined_seed = combined_seed.wrapping_add(st.randomization_seed); + + Some(VariantLayout::from_layout(st)) + }) + .collect::>>()?; + + let max_variant_size = variant_layouts.iter().map(|layout| layout.size).max()?; + + // Chooses the first max-sized niche-providing variant whose layout succeeds. + for (largest_variant_index, _) in + variant_layouts.iter_enumerated().filter(|&(_, layout)| { + layout.size == max_variant_size && layout.largest_niche.is_some() + }) + { + let mut variant_layouts = variant_layouts.clone(); + + let all_indices = variants.indices(); + let needs_disc = |index: VariantIdx| { + index != largest_variant_index && !absent(&variants[index]) + }; + let Some(niche_variants_start) = all_indices.clone().find(|v| needs_disc(*v)) + else { + continue; + }; + let Some(niche_variants_end) = all_indices.rev().find(|v| needs_disc(*v)) + else { + continue; + }; + let niche_variants = niche_variants_start..=niche_variants_end; + + let count = (niche_variants.end().index() as u128 + - niche_variants.start().index() as u128) + + 1; + + // Use the largest niche in the largest variant. + let Some(niche) = variant_layouts[largest_variant_index].largest_niche else { + continue; + }; + let Some((niche_start, niche_scalar)) = niche.reserve(dl, count) else { + continue; + }; + let niche_offset = niche.offset; + let niche_size = niche.value.size(dl); + let size = variant_layouts[largest_variant_index].size.align_to(align); + + let all_variants_fit = + variant_layouts.iter_enumerated_mut().all(|(i, layout)| { + if i == largest_variant_index { + return true; + } + + layout.largest_niche = None; + + if layout.size <= niche_offset { + // This variant will fit before the niche. + return true; + } + + // Determine if it'll fit after the niche. + let this_align = variants_info[i].align_abi; + let this_offset = (niche_offset + niche_size).align_to(this_align); + + if this_offset + layout.size > size { + // The ordinary niche-filling path can only move a non-largest variant as a + // whole before or after the chosen niche. If that fails, try placing the + // variant's fields individually while treating the niche bytes as reserved. + if let Some(repacked) = self.try_layout_variant_around_niche( + &variants[i], + repr, + i, + size, + align, + niche_offset, + niche_size, + ) { + *layout = VariantLayout::from_layout(repacked); + return true; + } + return false; + } + + // It'll fit, but we need to make some adjustments. + for offset in layout.field_offsets.iter_mut() { + *offset += this_offset; + } + + // It can't be a Scalar or ScalarPair because the offset isn't 0. + if !layout.is_uninhabited() { + layout.backend_repr = BackendRepr::Memory { sized: true }; + } + layout.size += this_offset; + + true + }); + + if !all_variants_fit { + continue; + } + + let largest_niche = Niche::from_scalar(dl, niche_offset, niche_scalar); + + let others_zst = variant_layouts + .iter_enumerated() + .all(|(i, layout)| i == largest_variant_index || layout.size == Size::ZERO); + let same_size = size == variant_layouts[largest_variant_index].size; + let same_align = align == variants_info[largest_variant_index].align_abi; + + let uninhabited = variant_layouts.iter().all(|v| v.is_uninhabited()); + let abi = if same_size && same_align && others_zst { + match variant_layouts[largest_variant_index].backend_repr { + // When the total alignment and size match, we can use the + // same ABI as the scalar variant with the reserved niche. + BackendRepr::Scalar(_) => BackendRepr::Scalar(niche_scalar), + BackendRepr::ScalarPair(first, second) => { + // Only the niche is guaranteed to be initialised, + // so use union layouts for the other primitive. + if niche_offset == Size::ZERO { + BackendRepr::ScalarPair(niche_scalar, second.to_union()) + } else { + BackendRepr::ScalarPair(first.to_union(), niche_scalar) + } + } + _ => BackendRepr::Memory { sized: true }, + } + } else { + BackendRepr::Memory { sized: true } + }; + + return Some(LayoutData { + variants: Variants::Multiple { + tag: niche_scalar, + tag_encoding: TagEncoding::Niche { + untagged_variant: largest_variant_index, + niche_variants, + niche_start, + }, + tag_field: FieldIdx::new(0), + variants: variant_layouts, + }, + fields: FieldsShape::Arbitrary { + offsets: [niche_offset].into(), + in_memory_order: [FieldIdx::new(0)].into(), + }, + backend_repr: abi, + largest_niche, + uninhabited, + size, + align: AbiAlign::new(align), + max_repr_align, + unadjusted_abi_align, + randomization_seed: combined_seed, + }); + } + + None + }; + + let niche_filling_layout = if repr.can_repack_variant_around_niche() { + calculate_niche_filling_layout_repacked() + } else { + calculate_niche_filling_layout() + }; let discr_type = repr.discr_type(); let discr_int = Integer::from_attr(dl, discr_type); @@ -1091,6 +1282,121 @@ impl LayoutCalculator { Ok(best_layout) } + fn try_layout_variant_around_niche< + 'a, + FieldIdx: Idx, + VariantIdx: Idx, + F: Deref> + fmt::Debug + Copy, + >( + &self, + fields: &IndexSlice, + repr: &ReprOptions, + variant_index: VariantIdx, + total_size: Size, + total_align: Align, + niche_offset: Size, + niche_size: Size, + ) -> Option> { + let dl = self.cx.data_layout(); + + if repr.inhibit_struct_field_reordering() { + return None; + } + if fields.iter().any(|f| f.is_unsized()) { + return None; + } + + // This is only a necessary condition: alignment can still make placement fail below. + let min_used_size = fields.iter().try_fold(niche_size, |size, field| { + if field.is_zst() { Some(size) } else { size.checked_add(field.size, dl) } + })?; + if min_used_size > total_size { + return None; + } + + let mut offsets = IndexVec::from_elem(Size::ZERO, fields); + let mut in_memory_order: IndexVec = fields.indices().collect(); + + in_memory_order.raw.sort_by_key(|&i| { + let field = &fields[i]; + + let field_align = if let Some(pack) = repr.pack { + field.align.min(AbiAlign::new(pack)) + } else { + field.align + }; + + (cmp::Reverse(field_align.abi.bytes()), cmp::Reverse(field.size.bytes())) + }); + + let niche_end = niche_offset.checked_add(niche_size, dl)?; + let mut occupied = vec![(niche_offset, niche_end)]; + + let mut max_repr_align = repr.align; + let mut unadjusted_abi_align = + if repr.pack.is_some() { dl.i8_align } else { dl.aggregate_align }; + + for &i in &in_memory_order { + let field = &fields[i]; + + let field_align = if let Some(pack) = repr.pack { + field.align.min(AbiAlign::new(pack)) + } else { + field.align + }; + + max_repr_align = max_repr_align.max(field.max_repr_align); + unadjusted_abi_align = unadjusted_abi_align.max(field_align.abi); + + let mut candidate = Size::ZERO; + 'search: loop { + candidate = candidate.align_to(field_align.abi); + + let end = candidate.checked_add(field.size, dl)?; + if end > total_size { + return None; + } + + for &(occupied_start, occupied_end) in &occupied { + if end <= occupied_start { + break; + } + if candidate < occupied_end { + candidate = occupied_end; + continue 'search; + } + } + + offsets[i] = candidate; + + if !field.is_zst() { + occupied.push((candidate, end)); + occupied.sort_by_key(|&(start, _end)| start); + } + + break; + } + } + + in_memory_order.raw.sort_by_key(|&i| offsets[i]); + + Some(LayoutData { + variants: Variants::Single { index: variant_index }, + fields: FieldsShape::Arbitrary { offsets, in_memory_order }, + backend_repr: BackendRepr::Memory { sized: true }, + largest_niche: None, + uninhabited: fields.iter().any(|f| f.is_uninhabited()), + align: AbiAlign::new(total_align), + size: total_size, + max_repr_align, + unadjusted_abi_align, + randomization_seed: fields + .iter() + .fold(Hash64::ZERO, |acc, f| acc.wrapping_add(f.randomization_seed)) + .wrapping_add(repr.field_shuffle_seed), + }) + } + fn univariant_biased< 'a, FieldIdx: Idx, diff --git a/compiler/rustc_abi/src/lib.rs b/compiler/rustc_abi/src/lib.rs index 166c8bea6f354..16e76062ff5bc 100644 --- a/compiler/rustc_abi/src/lib.rs +++ b/compiler/rustc_abi/src/lib.rs @@ -94,7 +94,9 @@ bitflags! { /// See [`TyAndLayout::pass_indirectly_in_non_rustic_abis`] for details. const PASS_INDIRECTLY_IN_NON_RUSTIC_ABIS = 1 << 5; const IS_SCALABLE = 1 << 6; - // Any of these flags being set prevent field reordering optimisation. + /// If true, enum niche-filling may repack variant fields around the reserved niche. + const CAN_REPACK_VARIANT_AROUND_NICHE = 1 << 7; + // Any of these flags being set prevent field reordering optimisation. const FIELD_ORDER_UNOPTIMIZABLE = ReprFlags::IS_C.bits() | ReprFlags::IS_SIMD.bits() | ReprFlags::IS_SCALABLE.bits() @@ -226,6 +228,12 @@ impl ReprOptions { !self.inhibit_struct_field_reordering() && self.flags.contains(ReprFlags::RANDOMIZE_LAYOUT) } + /// Returns `true` if enum niche-filling can try to pack variant fields around + /// the reserved niche when whole-variant placement fails. + pub fn can_repack_variant_around_niche(&self) -> bool { + self.flags.contains(ReprFlags::CAN_REPACK_VARIANT_AROUND_NICHE) + } + /// Returns `true` if this `#[repr()]` should inhibit union ABI optimisations. pub fn inhibits_union_abi_opt(&self) -> bool { self.c() diff --git a/compiler/rustc_middle/src/ty/mod.rs b/compiler/rustc_middle/src/ty/mod.rs index ac82cf1981032..79620944fb2bd 100644 --- a/compiler/rustc_middle/src/ty/mod.rs +++ b/compiler/rustc_middle/src/ty/mod.rs @@ -1603,6 +1603,10 @@ impl<'tcx> TyCtxt<'tcx> { flags.insert(ReprFlags::RANDOMIZE_LAYOUT); } + if self.def_span(did).edition().at_least_edition_future() { + flags.insert(ReprFlags::CAN_REPACK_VARIANT_AROUND_NICHE); + } + // box is special, on the one hand the compiler assumes an ordered layout, with the pointer // always at offset zero. On the other hand we want scalar abi optimizations. let is_box = self.is_lang_item(did.to_def_id(), LangItem::OwnedBox); diff --git a/tests/ui/structs-enums/enum-niche-fill-around-tag.rs b/tests/ui/structs-enums/enum-niche-fill-around-tag.rs new file mode 100644 index 0000000000000..85965762b8fda --- /dev/null +++ b/tests/ui/structs-enums/enum-niche-fill-around-tag.rs @@ -0,0 +1,140 @@ +//@ revisions: old future +//@ run-pass +//@ [old] edition:2024 +//@ [future] edition:future +//@ [future] compile-flags: -Z unstable-options + +#![allow(dead_code)] +#![cfg_attr(future, feature(offset_of_enum))] + +#[cfg(future)] +use std::mem::{align_of, offset_of}; +use std::mem::size_of; +use std::num::NonZero; + +enum Inner { + A(u8, u32), + B(u32), +} + +enum Outer { + A(Inner), + B(u32, u8), +} + +struct WithPadding { + a: bool, + b: bool, + c: u32, +} + +enum RepackPadding { + A(u32, u8), + B(WithPadding), +} + +struct RepackedBase { + int32: u32, + boolean: bool, +} + +enum RepackedAroundNiche { + A(RepackedBase), + B(u16, u32, u8), +} + +enum NestedRepackedAroundNiche { + A(RepackedAroundNiche), + B(u16, u32, u8), +} + +struct RepackedPair { + field2: u16, + field3: u8, +} + +enum RepackedNestedAggregate { + A(u16, RepackedPair), + B(NestedRepackedAroundNiche), +} + +struct FittingNichePayload { + word: u32, + flag: bool, +} + +// The first and last max-sized niche candidates cannot encode all variants. +// The middle candidate can, so the enum layout has to keep looking. +enum ChooseFittingNiche { + SmallNicheFront(u16, u32, NonZero), + WideNiche(FittingNichePayload), + SmallNicheBack(u16, u32, NonZero), + V0, + V1, + V2, +} + +fn main() { + assert_eq!(size_of::(), 8); + + #[cfg(old)] + { + assert_eq!(size_of::(), 12); + assert_eq!(size_of::(), 12); + } + + #[cfg(future)] + { + assert_eq!(size_of::(), 8); + assert_eq!(size_of::(), 8); + + assert_eq!(size_of::(), 8); + assert_eq!(align_of::(), 4); + assert_eq!(offset_of!(RepackedBase, int32), 0); + assert_eq!(offset_of!(RepackedBase, boolean), 4); + + assert_eq!(size_of::(), 8); + assert_eq!(align_of::(), 4); + assert_eq!(size_of::>(), 8); + assert_eq!(offset_of!(RepackedAroundNiche, A.0), 0); + assert_eq!(offset_of!(RepackedAroundNiche, B.1), 0); + assert_eq!(offset_of!(RepackedAroundNiche, B.2), 5); + assert_eq!(offset_of!(RepackedAroundNiche, B.0), 6); + + assert_eq!(size_of::(), 8); + assert_eq!(align_of::(), 4); + assert_eq!(size_of::>(), 8); + assert_eq!(offset_of!(NestedRepackedAroundNiche, A.0), 0); + assert_eq!(offset_of!(NestedRepackedAroundNiche, B.1), 0); + assert_eq!(offset_of!(NestedRepackedAroundNiche, B.2), 5); + assert_eq!(offset_of!(NestedRepackedAroundNiche, B.0), 6); + + assert_eq!(size_of::(), 4); + assert_eq!(align_of::(), 2); + assert_eq!(offset_of!(RepackedPair, field2), 0); + assert_eq!(offset_of!(RepackedPair, field3), 2); + + assert_eq!(size_of::(), 8); + assert_eq!(align_of::(), 4); + assert_eq!(size_of::>(), 8); + assert_eq!(offset_of!(RepackedNestedAggregate, B.0), 0); + assert_eq!(offset_of!(RepackedNestedAggregate, A.1), 0); + assert_eq!(offset_of!(RepackedNestedAggregate, A.0), 6); + + assert_eq!(size_of::(), 8); + assert_eq!(align_of::(), 4); + assert_eq!(offset_of!(FittingNichePayload, word), 0); + assert_eq!(offset_of!(FittingNichePayload, flag), 4); + + assert_eq!(size_of::(), 8); + assert_eq!(align_of::(), 4); + assert_eq!(size_of::>(), 8); + assert_eq!(offset_of!(ChooseFittingNiche, WideNiche.0), 0); + assert_eq!(offset_of!(ChooseFittingNiche, SmallNicheFront.1), 0); + assert_eq!(offset_of!(ChooseFittingNiche, SmallNicheFront.2), 5); + assert_eq!(offset_of!(ChooseFittingNiche, SmallNicheFront.0), 6); + assert_eq!(offset_of!(ChooseFittingNiche, SmallNicheBack.1), 0); + assert_eq!(offset_of!(ChooseFittingNiche, SmallNicheBack.2), 5); + assert_eq!(offset_of!(ChooseFittingNiche, SmallNicheBack.0), 6); + } +} From ae0498897b3c04e5e1b2a7e452cb64b2d8d2d79c Mon Sep 17 00:00:00 2001 From: Miguel Marques Date: Wed, 10 Jun 2026 14:12:16 +0100 Subject: [PATCH 2/5] fix: Organises the enum niche-filling variant packing implementation. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Organises the niche-repacking solution structure for better mantainability. Minor bug fix cause by type mismatch. Signed-off-by: Miguel Marques Co-authored-by: Vicente Gusmão --- compiler/rustc_abi/src/layout.rs | 282 ++++++++----------------------- 1 file changed, 68 insertions(+), 214 deletions(-) diff --git a/compiler/rustc_abi/src/layout.rs b/compiler/rustc_abi/src/layout.rs index 9d1ee4120524d..5e20dcb474e5d 100644 --- a/compiler/rustc_abi/src/layout.rs +++ b/compiler/rustc_abi/src/layout.rs @@ -608,7 +608,7 @@ impl LayoutCalculator { let mut combined_seed = repr.field_shuffle_seed; let mut variants_info = IndexVec::::with_capacity(variants.len()); - let mut variant_layouts = variants + let variant_layouts = variants .iter() .map(|v| { let st = self.univariant(v, repr, StructKind::AlwaysSized).ok()?; @@ -624,195 +624,27 @@ impl LayoutCalculator { }) .collect::>>()?; - let largest_variant_index = variant_layouts - .iter_enumerated() - .max_by_key(|(_i, layout)| layout.size.bytes()) - .map(|(i, _layout)| i)?; - - let all_indices = variants.indices(); - let needs_disc = - |index: VariantIdx| index != largest_variant_index && !absent(&variants[index]); - let niche_variants = RangeInclusive { - start: all_indices.clone().find(|v| needs_disc(*v)).unwrap(), - last: all_indices.rev().find(|v| needs_disc(*v)).unwrap(), - }; - - let count = - (niche_variants.last.index() as u128 - niche_variants.start.index() as u128) + 1; - - // Use the largest niche in the largest variant. - let niche = variant_layouts[largest_variant_index].largest_niche?; - let (niche_start, niche_scalar) = niche.reserve(dl, count)?; - let niche_offset = niche.offset; - let niche_size = niche.value.size(dl); - let size = variant_layouts[largest_variant_index].size.align_to(align); - - let all_variants_fit = variant_layouts.iter_enumerated_mut().all(|(i, layout)| { - if i == largest_variant_index { - return true; - } - - layout.largest_niche = None; - - if layout.size <= niche_offset { - // This variant will fit before the niche. - return true; - } - - // Determine if it'll fit after the niche. - let this_align = variants_info[i].align_abi; - let this_offset = (niche_offset + niche_size).align_to(this_align); - - if this_offset + layout.size > size { - return false; - } - - // It'll fit, but we need to make some adjustments. - for offset in layout.field_offsets.iter_mut() { - *offset += this_offset; - } - - // It can't be a Scalar or ScalarPair because the offset isn't 0. - if !layout.is_uninhabited() { - layout.backend_repr = BackendRepr::Memory { sized: true }; - } - layout.size += this_offset; - - true - }); - - if !all_variants_fit { - return None; - } - - let largest_niche = Niche::from_scalar(dl, niche_offset, niche_scalar); - - let others_zst = variant_layouts - .iter_enumerated() - .all(|(i, layout)| i == largest_variant_index || layout.size == Size::ZERO); - let same_size = size == variant_layouts[largest_variant_index].size; - let same_align = align == variants_info[largest_variant_index].align_abi; - - let uninhabited = variant_layouts.iter().all(|v| v.is_uninhabited()); - let abi = if same_size && same_align && others_zst { - match variant_layouts[largest_variant_index].backend_repr { - // When the total alignment and size match, we can use the - // same ABI as the scalar variant with the reserved niche. - BackendRepr::Scalar(_) => BackendRepr::Scalar(niche_scalar), - BackendRepr::ScalarPair(first, second) => { - // Only the niche is guaranteed to be initialised, - // so use union layouts for the other primitive. - if niche_offset == Size::ZERO { - BackendRepr::ScalarPair(niche_scalar, second.to_union()) - } else { - BackendRepr::ScalarPair(first.to_union(), niche_scalar) - } - } - _ => BackendRepr::Memory { sized: true }, - } - } else { - BackendRepr::Memory { sized: true } - }; - - let layout = LayoutData { - variants: Variants::Multiple { - tag: niche_scalar, - tag_encoding: TagEncoding::Niche { - untagged_variant: largest_variant_index, - niche_variants, - niche_start, - }, - tag_field: FieldIdx::new(0), - variants: variant_layouts, - }, - fields: FieldsShape::Arbitrary { - offsets: [niche_offset].into(), - in_memory_order: [FieldIdx::new(0)].into(), - }, - backend_repr: abi, - largest_niche, - uninhabited, - size, - align: AbiAlign::new(align), - max_repr_align, - unadjusted_abi_align, - randomization_seed: combined_seed, - }; - - Some(layout) - }; - - let calculate_niche_filling_layout_repacked = - || -> Option> { - struct VariantLayoutInfo { - align_abi: Align, - } - - if repr.inhibit_enum_layout_opt() { - return None; - } - - if variants.len() < 2 { - return None; - } - - let mut align = dl.aggregate_align; - let mut max_repr_align = repr.align; - let mut unadjusted_abi_align = align; - let mut combined_seed = repr.field_shuffle_seed; - - let mut variants_info = IndexVec::::with_capacity(variants.len()); - let variant_layouts = variants - .iter() - .map(|v| { - let st = self.univariant(v, repr, StructKind::AlwaysSized).ok()?; - - variants_info.push(VariantLayoutInfo { align_abi: st.align.abi }); - - align = align.max(st.align.abi); - max_repr_align = max_repr_align.max(st.max_repr_align); - unadjusted_abi_align = unadjusted_abi_align.max(st.unadjusted_abi_align); - combined_seed = combined_seed.wrapping_add(st.randomization_seed); - - Some(VariantLayout::from_layout(st)) - }) - .collect::>>()?; - - let max_variant_size = variant_layouts.iter().map(|layout| layout.size).max()?; - - // Chooses the first max-sized niche-providing variant whose layout succeeds. - for (largest_variant_index, _) in - variant_layouts.iter_enumerated().filter(|&(_, layout)| { - layout.size == max_variant_size && layout.largest_niche.is_some() - }) - { + let try_niche_variant = + |largest_variant_index: VariantIdx| -> Option> { + //so niche_variant candidates don't contaminate each other if one fails let mut variant_layouts = variant_layouts.clone(); let all_indices = variants.indices(); let needs_disc = |index: VariantIdx| { index != largest_variant_index && !absent(&variants[index]) }; - let Some(niche_variants_start) = all_indices.clone().find(|v| needs_disc(*v)) - else { - continue; - }; - let Some(niche_variants_end) = all_indices.rev().find(|v| needs_disc(*v)) - else { - continue; + let niche_variants = RangeInclusive { + start: all_indices.clone().find(|v| needs_disc(*v)).unwrap(), + last: all_indices.rev().find(|v| needs_disc(*v)).unwrap(), }; - let niche_variants = niche_variants_start..=niche_variants_end; - let count = (niche_variants.end().index() as u128 - - niche_variants.start().index() as u128) + let count = (niche_variants.last.index() as u128 + - niche_variants.start.index() as u128) + 1; // Use the largest niche in the largest variant. - let Some(niche) = variant_layouts[largest_variant_index].largest_niche else { - continue; - }; - let Some((niche_start, niche_scalar)) = niche.reserve(dl, count) else { - continue; - }; + let niche = variant_layouts[largest_variant_index].largest_niche?; + let (niche_start, niche_scalar) = niche.reserve(dl, count)?; let niche_offset = niche.offset; let niche_size = niche.value.size(dl); let size = variant_layouts[largest_variant_index].size.align_to(align); @@ -834,41 +666,46 @@ impl LayoutCalculator { let this_align = variants_info[i].align_abi; let this_offset = (niche_offset + niche_size).align_to(this_align); - if this_offset + layout.size > size { - // The ordinary niche-filling path can only move a non-largest variant as a - // whole before or after the chosen niche. If that fails, try placing the - // variant's fields individually while treating the niche bytes as reserved. - if let Some(repacked) = self.try_layout_variant_around_niche( - &variants[i], - repr, - i, - size, - align, - niche_offset, - niche_size, - ) { - *layout = VariantLayout::from_layout(repacked); - return true; + if this_offset + layout.size <= size { + // It'll fit, but we need to make some adjustments. + for offset in layout.field_offsets.iter_mut() { + *offset += this_offset; } - return false; - } - // It'll fit, but we need to make some adjustments. - for offset in layout.field_offsets.iter_mut() { - *offset += this_offset; - } + // It can't be a Scalar or ScalarPair because the offset isn't 0. + if !layout.is_uninhabited() { + layout.backend_repr = BackendRepr::Memory { sized: true }; + } + layout.size += this_offset; - // It can't be a Scalar or ScalarPair because the offset isn't 0. - if !layout.is_uninhabited() { - layout.backend_repr = BackendRepr::Memory { sized: true }; + return true; + } + // Repacking is currently only on future edition. For editions where it is disabled, + // fall back to the original niche-filling behaviour. + if !repr.can_repack_variant_around_niche() { + return false; + } + // The ordinary niche-filling path can only move a non-largest variant as a + // whole before or after the chosen niche. If that fails, try placing the + // variant's fields individually while treating the niche bytes as reserved. + if let Some(repacked) = self.try_layout_variant_around_niche( + &variants[i], + repr, + i, + size, + align, + niche_offset, + niche_size, + ) { + *layout = VariantLayout::from_layout(repacked); + return true; } - layout.size += this_offset; - true + false }); if !all_variants_fit { - continue; + return None; } let largest_niche = Niche::from_scalar(dl, niche_offset, niche_scalar); @@ -900,7 +737,7 @@ impl LayoutCalculator { BackendRepr::Memory { sized: true } }; - return Some(LayoutData { + let layout = LayoutData { variants: Variants::Multiple { tag: niche_scalar, tag_encoding: TagEncoding::Niche { @@ -923,18 +760,35 @@ impl LayoutCalculator { max_repr_align, unadjusted_abi_align, randomization_seed: combined_seed, - }); + }; + + Some(layout) + }; + if repr.can_repack_variant_around_niche() { + let max_variant_size = variant_layouts.iter().map(|layout| layout.size).max()?; + // Chooses the first max-sized niche-providing variant whose layout succeeds. + for (largest_variant_index, _) in + variant_layouts.iter_enumerated().filter(|&(_, layout)| { + layout.size == max_variant_size && layout.largest_niche.is_some() + }) + { + if let Some(layout) = try_niche_variant(largest_variant_index) { + return Some(layout); + } } None - }; - - let niche_filling_layout = if repr.can_repack_variant_around_niche() { - calculate_niche_filling_layout_repacked() - } else { - calculate_niche_filling_layout() + } else { + let largest_variant_index = variant_layouts + .iter_enumerated() + .max_by_key(|(_i, layout)| layout.size.bytes()) + .map(|(i, _layout)| i)?; + try_niche_variant(largest_variant_index) + } }; + let niche_filling_layout = calculate_niche_filling_layout(); + let discr_type = repr.discr_type(); let discr_int = Integer::from_attr(dl, discr_type); // Because we can only represent one range of valid values, we'll look for the From 5431efd93a0a0af303fa330d9d491ce6eb626260 Mon Sep 17 00:00:00 2001 From: Miguel Marques Date: Wed, 10 Jun 2026 19:01:26 +0100 Subject: [PATCH 3/5] Trigger CI From 1098028d66e2b3a77314201b9549cb3f2c1e1633 Mon Sep 17 00:00:00 2001 From: Miguel Marques Date: Sat, 27 Jun 2026 23:36:25 +0100 Subject: [PATCH 4/5] fix: Adapts new enum layout to work in all editions. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The new implementation was only working for future edition, but since currently enum layout is not guaranteed, it adds the changes to all editions. Signed-off-by: Miguel Marques Co-authored-by: Vicente Gusmão --- compiler/rustc_abi/src/layout.rs | 37 ++---- compiler/rustc_abi/src/lib.rs | 9 -- compiler/rustc_middle/src/ty/mod.rs | 4 - .../enum-niche-fill-around-tag.rs | 118 ++++++++---------- 4 files changed, 65 insertions(+), 103 deletions(-) diff --git a/compiler/rustc_abi/src/layout.rs b/compiler/rustc_abi/src/layout.rs index 5e20dcb474e5d..8e36e92c3413d 100644 --- a/compiler/rustc_abi/src/layout.rs +++ b/compiler/rustc_abi/src/layout.rs @@ -680,11 +680,7 @@ impl LayoutCalculator { return true; } - // Repacking is currently only on future edition. For editions where it is disabled, - // fall back to the original niche-filling behaviour. - if !repr.can_repack_variant_around_niche() { - return false; - } + // The ordinary niche-filling path can only move a non-largest variant as a // whole before or after the chosen niche. If that fails, try placing the // variant's fields individually while treating the niche bytes as reserved. @@ -764,27 +760,20 @@ impl LayoutCalculator { Some(layout) }; - if repr.can_repack_variant_around_niche() { - let max_variant_size = variant_layouts.iter().map(|layout| layout.size).max()?; - // Chooses the first max-sized niche-providing variant whose layout succeeds. - for (largest_variant_index, _) in - variant_layouts.iter_enumerated().filter(|&(_, layout)| { - layout.size == max_variant_size && layout.largest_niche.is_some() - }) - { - if let Some(layout) = try_niche_variant(largest_variant_index) { - return Some(layout); - } - } - None - } else { - let largest_variant_index = variant_layouts - .iter_enumerated() - .max_by_key(|(_i, layout)| layout.size.bytes()) - .map(|(i, _layout)| i)?; - try_niche_variant(largest_variant_index) + let max_variant_size = variant_layouts.iter().map(|layout| layout.size).max()?; + // Chooses the first max-sized niche-providing variant whose layout succeeds. + for (largest_variant_index, _) in + variant_layouts.iter_enumerated().filter(|&(_, layout)| { + layout.size == max_variant_size && layout.largest_niche.is_some() + }) + { + if let Some(layout) = try_niche_variant(largest_variant_index) { + return Some(layout); + } } + + None }; let niche_filling_layout = calculate_niche_filling_layout(); diff --git a/compiler/rustc_abi/src/lib.rs b/compiler/rustc_abi/src/lib.rs index 16e76062ff5bc..5aaa0a7e3eeb4 100644 --- a/compiler/rustc_abi/src/lib.rs +++ b/compiler/rustc_abi/src/lib.rs @@ -94,9 +94,6 @@ bitflags! { /// See [`TyAndLayout::pass_indirectly_in_non_rustic_abis`] for details. const PASS_INDIRECTLY_IN_NON_RUSTIC_ABIS = 1 << 5; const IS_SCALABLE = 1 << 6; - /// If true, enum niche-filling may repack variant fields around the reserved niche. - const CAN_REPACK_VARIANT_AROUND_NICHE = 1 << 7; - // Any of these flags being set prevent field reordering optimisation. const FIELD_ORDER_UNOPTIMIZABLE = ReprFlags::IS_C.bits() | ReprFlags::IS_SIMD.bits() | ReprFlags::IS_SCALABLE.bits() @@ -228,12 +225,6 @@ impl ReprOptions { !self.inhibit_struct_field_reordering() && self.flags.contains(ReprFlags::RANDOMIZE_LAYOUT) } - /// Returns `true` if enum niche-filling can try to pack variant fields around - /// the reserved niche when whole-variant placement fails. - pub fn can_repack_variant_around_niche(&self) -> bool { - self.flags.contains(ReprFlags::CAN_REPACK_VARIANT_AROUND_NICHE) - } - /// Returns `true` if this `#[repr()]` should inhibit union ABI optimisations. pub fn inhibits_union_abi_opt(&self) -> bool { self.c() diff --git a/compiler/rustc_middle/src/ty/mod.rs b/compiler/rustc_middle/src/ty/mod.rs index 79620944fb2bd..ac82cf1981032 100644 --- a/compiler/rustc_middle/src/ty/mod.rs +++ b/compiler/rustc_middle/src/ty/mod.rs @@ -1603,10 +1603,6 @@ impl<'tcx> TyCtxt<'tcx> { flags.insert(ReprFlags::RANDOMIZE_LAYOUT); } - if self.def_span(did).edition().at_least_edition_future() { - flags.insert(ReprFlags::CAN_REPACK_VARIANT_AROUND_NICHE); - } - // box is special, on the one hand the compiler assumes an ordered layout, with the pointer // always at offset zero. On the other hand we want scalar abi optimizations. let is_box = self.is_lang_item(did.to_def_id(), LangItem::OwnedBox); diff --git a/tests/ui/structs-enums/enum-niche-fill-around-tag.rs b/tests/ui/structs-enums/enum-niche-fill-around-tag.rs index 85965762b8fda..31f137e4057d1 100644 --- a/tests/ui/structs-enums/enum-niche-fill-around-tag.rs +++ b/tests/ui/structs-enums/enum-niche-fill-around-tag.rs @@ -1,13 +1,8 @@ -//@ revisions: old future //@ run-pass -//@ [old] edition:2024 -//@ [future] edition:future -//@ [future] compile-flags: -Z unstable-options #![allow(dead_code)] -#![cfg_attr(future, feature(offset_of_enum))] +#![feature(offset_of_enum)] -#[cfg(future)] use std::mem::{align_of, offset_of}; use std::mem::size_of; use std::num::NonZero; @@ -77,64 +72,55 @@ enum ChooseFittingNiche { fn main() { assert_eq!(size_of::(), 8); - #[cfg(old)] - { - assert_eq!(size_of::(), 12); - assert_eq!(size_of::(), 12); - } - - #[cfg(future)] - { - assert_eq!(size_of::(), 8); - assert_eq!(size_of::(), 8); - - assert_eq!(size_of::(), 8); - assert_eq!(align_of::(), 4); - assert_eq!(offset_of!(RepackedBase, int32), 0); - assert_eq!(offset_of!(RepackedBase, boolean), 4); - - assert_eq!(size_of::(), 8); - assert_eq!(align_of::(), 4); - assert_eq!(size_of::>(), 8); - assert_eq!(offset_of!(RepackedAroundNiche, A.0), 0); - assert_eq!(offset_of!(RepackedAroundNiche, B.1), 0); - assert_eq!(offset_of!(RepackedAroundNiche, B.2), 5); - assert_eq!(offset_of!(RepackedAroundNiche, B.0), 6); - - assert_eq!(size_of::(), 8); - assert_eq!(align_of::(), 4); - assert_eq!(size_of::>(), 8); - assert_eq!(offset_of!(NestedRepackedAroundNiche, A.0), 0); - assert_eq!(offset_of!(NestedRepackedAroundNiche, B.1), 0); - assert_eq!(offset_of!(NestedRepackedAroundNiche, B.2), 5); - assert_eq!(offset_of!(NestedRepackedAroundNiche, B.0), 6); - - assert_eq!(size_of::(), 4); - assert_eq!(align_of::(), 2); - assert_eq!(offset_of!(RepackedPair, field2), 0); - assert_eq!(offset_of!(RepackedPair, field3), 2); - - assert_eq!(size_of::(), 8); - assert_eq!(align_of::(), 4); - assert_eq!(size_of::>(), 8); - assert_eq!(offset_of!(RepackedNestedAggregate, B.0), 0); - assert_eq!(offset_of!(RepackedNestedAggregate, A.1), 0); - assert_eq!(offset_of!(RepackedNestedAggregate, A.0), 6); - - assert_eq!(size_of::(), 8); - assert_eq!(align_of::(), 4); - assert_eq!(offset_of!(FittingNichePayload, word), 0); - assert_eq!(offset_of!(FittingNichePayload, flag), 4); - - assert_eq!(size_of::(), 8); - assert_eq!(align_of::(), 4); - assert_eq!(size_of::>(), 8); - assert_eq!(offset_of!(ChooseFittingNiche, WideNiche.0), 0); - assert_eq!(offset_of!(ChooseFittingNiche, SmallNicheFront.1), 0); - assert_eq!(offset_of!(ChooseFittingNiche, SmallNicheFront.2), 5); - assert_eq!(offset_of!(ChooseFittingNiche, SmallNicheFront.0), 6); - assert_eq!(offset_of!(ChooseFittingNiche, SmallNicheBack.1), 0); - assert_eq!(offset_of!(ChooseFittingNiche, SmallNicheBack.2), 5); - assert_eq!(offset_of!(ChooseFittingNiche, SmallNicheBack.0), 6); - } + assert_eq!(size_of::(), 8); + assert_eq!(size_of::(), 8); + + assert_eq!(size_of::(), 8); + assert_eq!(align_of::(), 4); + assert_eq!(offset_of!(RepackedBase, int32), 0); + assert_eq!(offset_of!(RepackedBase, boolean), 4); + + assert_eq!(size_of::(), 8); + assert_eq!(align_of::(), 4); + assert_eq!(size_of::>(), 8); + assert_eq!(offset_of!(RepackedAroundNiche, A.0), 0); + assert_eq!(offset_of!(RepackedAroundNiche, B.1), 0); + assert_eq!(offset_of!(RepackedAroundNiche, B.2), 5); + assert_eq!(offset_of!(RepackedAroundNiche, B.0), 6); + + assert_eq!(size_of::(), 8); + assert_eq!(align_of::(), 4); + assert_eq!(size_of::>(), 8); + assert_eq!(offset_of!(NestedRepackedAroundNiche, A.0), 0); + assert_eq!(offset_of!(NestedRepackedAroundNiche, B.1), 0); + assert_eq!(offset_of!(NestedRepackedAroundNiche, B.2), 5); + assert_eq!(offset_of!(NestedRepackedAroundNiche, B.0), 6); + + assert_eq!(size_of::(), 4); + assert_eq!(align_of::(), 2); + assert_eq!(offset_of!(RepackedPair, field2), 0); + assert_eq!(offset_of!(RepackedPair, field3), 2); + + assert_eq!(size_of::(), 8); + assert_eq!(align_of::(), 4); + assert_eq!(size_of::>(), 8); + assert_eq!(offset_of!(RepackedNestedAggregate, B.0), 0); + assert_eq!(offset_of!(RepackedNestedAggregate, A.1), 0); + assert_eq!(offset_of!(RepackedNestedAggregate, A.0), 6); + + assert_eq!(size_of::(), 8); + assert_eq!(align_of::(), 4); + assert_eq!(offset_of!(FittingNichePayload, word), 0); + assert_eq!(offset_of!(FittingNichePayload, flag), 4); + + assert_eq!(size_of::(), 8); + assert_eq!(align_of::(), 4); + assert_eq!(size_of::>(), 8); + assert_eq!(offset_of!(ChooseFittingNiche, WideNiche.0), 0); + assert_eq!(offset_of!(ChooseFittingNiche, SmallNicheFront.1), 0); + assert_eq!(offset_of!(ChooseFittingNiche, SmallNicheFront.2), 5); + assert_eq!(offset_of!(ChooseFittingNiche, SmallNicheFront.0), 6); + assert_eq!(offset_of!(ChooseFittingNiche, SmallNicheBack.1), 0); + assert_eq!(offset_of!(ChooseFittingNiche, SmallNicheBack.2), 5); + assert_eq!(offset_of!(ChooseFittingNiche, SmallNicheBack.0), 6); } From e2707ecf8d98ffca33a976b50186fadd971aacc6 Mon Sep 17 00:00:00 2001 From: vicentegusmao Date: Sun, 28 Jun 2026 16:18:08 +0100 Subject: [PATCH 5/5] fix: update AST size assertions after enum layout changes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Account for the old bootstrap compiler still seeing the previous Item and ItemKind sizes, while the new compiler sees the smaller layouts produced by the enum layout optimization. Update print-type-size output for enums that now fit in less space. Signed-off-by: Vicente Gusmão Co-authored-by: Miguel Marques --- compiler/rustc_ast/src/ast.rs | 6 ++++++ tests/ui/print_type_sizes/padding.stdout | 26 +++++++++--------------- tests/ui/stats/input-stats.rs | 1 + tests/ui/stats/input-stats.stderr | 18 ++++++++-------- 4 files changed, 26 insertions(+), 25 deletions(-) diff --git a/compiler/rustc_ast/src/ast.rs b/compiler/rustc_ast/src/ast.rs index b8339c74dac03..a13d41fce8ecc 100644 --- a/compiler/rustc_ast/src/ast.rs +++ b/compiler/rustc_ast/src/ast.rs @@ -4359,7 +4359,13 @@ mod size_asserts { static_assert_size!(GenericBound, 88); static_assert_size!(Generics, 40); static_assert_size!(Impl, 80); + #[cfg(not(bootstrap))] + static_assert_size!(Item, 144); + #[cfg(bootstrap)] static_assert_size!(Item, 152); + #[cfg(not(bootstrap))] + static_assert_size!(ItemKind, 80); + #[cfg(bootstrap)] static_assert_size!(ItemKind, 88); static_assert_size!(LitKind, 24); static_assert_size!(Local, 96); diff --git a/tests/ui/print_type_sizes/padding.stdout b/tests/ui/print_type_sizes/padding.stdout index 9afdf76245df7..d12ff401f47f3 100644 --- a/tests/ui/print_type_sizes/padding.stdout +++ b/tests/ui/print_type_sizes/padding.stdout @@ -1,21 +1,15 @@ -print-type-size type: `E1`: 12 bytes, alignment: 4 bytes -print-type-size discriminant: 1 bytes -print-type-size variant `B`: 11 bytes -print-type-size padding: 3 bytes -print-type-size field `.0`: 8 bytes, alignment: 4 bytes -print-type-size variant `A`: 7 bytes +print-type-size type: `E1`: 8 bytes, alignment: 4 bytes +print-type-size variant `B`: 8 bytes +print-type-size field `.0`: 8 bytes +print-type-size variant `A`: 5 bytes +print-type-size field `.0`: 4 bytes print-type-size field `.1`: 1 bytes -print-type-size padding: 2 bytes -print-type-size field `.0`: 4 bytes, alignment: 4 bytes -print-type-size type: `E2`: 12 bytes, alignment: 4 bytes -print-type-size discriminant: 1 bytes -print-type-size variant `B`: 11 bytes -print-type-size padding: 3 bytes -print-type-size field `.0`: 8 bytes, alignment: 4 bytes -print-type-size variant `A`: 7 bytes +print-type-size type: `E2`: 8 bytes, alignment: 4 bytes +print-type-size variant `B`: 8 bytes +print-type-size field `.0`: 8 bytes +print-type-size variant `A`: 5 bytes +print-type-size field `.1`: 4 bytes print-type-size field `.0`: 1 bytes -print-type-size padding: 2 bytes -print-type-size field `.1`: 4 bytes, alignment: 4 bytes print-type-size type: `S`: 8 bytes, alignment: 4 bytes print-type-size field `.g`: 4 bytes print-type-size field `.a`: 1 bytes diff --git a/tests/ui/stats/input-stats.rs b/tests/ui/stats/input-stats.rs index 86e904202c1b1..ee4706ce30705 100644 --- a/tests/ui/stats/input-stats.rs +++ b/tests/ui/stats/input-stats.rs @@ -14,6 +14,7 @@ // bump occurs, stage1 and stage2 will give different outputs for this test. // Add an `ignore-stage1` comment marker to work around that problem during // that time. +//@ ignore-stage1 // The aim here is to include at least one of every different type of top-level diff --git a/tests/ui/stats/input-stats.stderr b/tests/ui/stats/input-stats.stderr index b4d8277a46c00..3f68921947da4 100644 --- a/tests/ui/stats/input-stats.stderr +++ b/tests/ui/stats/input-stats.stderr @@ -2,14 +2,14 @@ ast-stats ================================================================ ast-stats POST EXPANSION AST STATS: input_stats ast-stats Name Accumulated Size Count Item Size ast-stats ---------------------------------------------------------------- -ast-stats Item 1_672 (NN.N%) 11 152 -ast-stats - Enum 152 (NN.N%) 1 -ast-stats - ExternCrate 152 (NN.N%) 1 -ast-stats - ForeignMod 152 (NN.N%) 1 -ast-stats - Impl 152 (NN.N%) 1 -ast-stats - Trait 152 (NN.N%) 1 -ast-stats - Fn 304 (NN.N%) 2 -ast-stats - Use 608 (NN.N%) 4 +ast-stats Item 1_584 (NN.N%) 11 144 +ast-stats - Enum 144 (NN.N%) 1 +ast-stats - ExternCrate 144 (NN.N%) 1 +ast-stats - ForeignMod 144 (NN.N%) 1 +ast-stats - Impl 144 (NN.N%) 1 +ast-stats - Trait 144 (NN.N%) 1 +ast-stats - Fn 288 (NN.N%) 2 +ast-stats - Use 576 (NN.N%) 4 ast-stats Ty 896 (NN.N%) 14 64 ast-stats - Ptr 64 (NN.N%) 1 ast-stats - Ref 64 (NN.N%) 1 @@ -57,7 +57,7 @@ ast-stats GenericArgs 40 (NN.N%) 1 40 ast-stats - AngleBracketed 40 (NN.N%) 1 ast-stats Crate 40 (NN.N%) 1 40 ast-stats ---------------------------------------------------------------- -ast-stats Total 7_624 127 +ast-stats Total 7_536 127 ast-stats ================================================================ hir-stats ================================================================ hir-stats HIR STATS: input_stats