diff --git a/compiler/rustc_abi/src/layout.rs b/compiler/rustc_abi/src/layout.rs index 733a1956231b1..8e36e92c3413d 100644 --- a/compiler/rustc_abi/src/layout.rs +++ b/compiler/rustc_abi/src/layout.rs @@ -608,7 +608,7 @@ impl LayoutCalculator { let mut combined_seed = repr.field_shuffle_seed; let mut variants_info = IndexVec::::with_capacity(variants.len()); - let mut variant_layouts = variants + let variant_layouts = variants .iter() .map(|v| { let st = self.univariant(v, repr, StructKind::AlwaysSized).ok()?; @@ -624,122 +624,156 @@ impl LayoutCalculator { }) .collect::>>()?; - let largest_variant_index = variant_layouts - .iter_enumerated() - .max_by_key(|(_i, layout)| layout.size.bytes()) - .map(|(i, _layout)| i)?; - - let all_indices = variants.indices(); - let needs_disc = - |index: VariantIdx| index != largest_variant_index && !absent(&variants[index]); - let niche_variants = RangeInclusive { - start: all_indices.clone().find(|v| needs_disc(*v)).unwrap(), - last: all_indices.rev().find(|v| needs_disc(*v)).unwrap(), - }; + let try_niche_variant = + |largest_variant_index: VariantIdx| -> Option> { + //so niche_variant candidates don't contaminate each other if one fails + let mut variant_layouts = variant_layouts.clone(); - let count = - (niche_variants.last.index() as u128 - niche_variants.start.index() as u128) + 1; + let all_indices = variants.indices(); + let needs_disc = |index: VariantIdx| { + index != largest_variant_index && !absent(&variants[index]) + }; + let niche_variants = RangeInclusive { + start: all_indices.clone().find(|v| needs_disc(*v)).unwrap(), + last: all_indices.rev().find(|v| needs_disc(*v)).unwrap(), + }; - // Use the largest niche in the largest variant. - let niche = variant_layouts[largest_variant_index].largest_niche?; - let (niche_start, niche_scalar) = niche.reserve(dl, count)?; - let niche_offset = niche.offset; - let niche_size = niche.value.size(dl); - let size = variant_layouts[largest_variant_index].size.align_to(align); + let count = (niche_variants.last.index() as u128 + - niche_variants.start.index() as u128) + + 1; + + // Use the largest niche in the largest variant. + let niche = variant_layouts[largest_variant_index].largest_niche?; + let (niche_start, niche_scalar) = niche.reserve(dl, count)?; + let niche_offset = niche.offset; + let niche_size = niche.value.size(dl); + let size = variant_layouts[largest_variant_index].size.align_to(align); + + let all_variants_fit = + variant_layouts.iter_enumerated_mut().all(|(i, layout)| { + if i == largest_variant_index { + return true; + } - let all_variants_fit = variant_layouts.iter_enumerated_mut().all(|(i, layout)| { - if i == largest_variant_index { - return true; - } + layout.largest_niche = None; - layout.largest_niche = None; + if layout.size <= niche_offset { + // This variant will fit before the niche. + return true; + } - if layout.size <= niche_offset { - // This variant will fit before the niche. - return true; - } + // Determine if it'll fit after the niche. + let this_align = variants_info[i].align_abi; + let this_offset = (niche_offset + niche_size).align_to(this_align); - // Determine if it'll fit after the niche. - let this_align = variants_info[i].align_abi; - let this_offset = (niche_offset + niche_size).align_to(this_align); + if this_offset + layout.size <= size { + // It'll fit, but we need to make some adjustments. + for offset in layout.field_offsets.iter_mut() { + *offset += this_offset; + } - if this_offset + layout.size > size { - return false; - } + // It can't be a Scalar or ScalarPair because the offset isn't 0. + if !layout.is_uninhabited() { + layout.backend_repr = BackendRepr::Memory { sized: true }; + } + layout.size += this_offset; - // It'll fit, but we need to make some adjustments. - for offset in layout.field_offsets.iter_mut() { - *offset += this_offset; - } + return true; + } - // It can't be a Scalar or ScalarPair because the offset isn't 0. - if !layout.is_uninhabited() { - layout.backend_repr = BackendRepr::Memory { sized: true }; - } - layout.size += this_offset; + // The ordinary niche-filling path can only move a non-largest variant as a + // whole before or after the chosen niche. If that fails, try placing the + // variant's fields individually while treating the niche bytes as reserved. + if let Some(repacked) = self.try_layout_variant_around_niche( + &variants[i], + repr, + i, + size, + align, + niche_offset, + niche_size, + ) { + *layout = VariantLayout::from_layout(repacked); + return true; + } - true - }); + false + }); - if !all_variants_fit { - return None; - } + if !all_variants_fit { + return None; + } - let largest_niche = Niche::from_scalar(dl, niche_offset, niche_scalar); - - let others_zst = variant_layouts - .iter_enumerated() - .all(|(i, layout)| i == largest_variant_index || layout.size == Size::ZERO); - let same_size = size == variant_layouts[largest_variant_index].size; - let same_align = align == variants_info[largest_variant_index].align_abi; - - let uninhabited = variant_layouts.iter().all(|v| v.is_uninhabited()); - let abi = if same_size && same_align && others_zst { - match variant_layouts[largest_variant_index].backend_repr { - // When the total alignment and size match, we can use the - // same ABI as the scalar variant with the reserved niche. - BackendRepr::Scalar(_) => BackendRepr::Scalar(niche_scalar), - BackendRepr::ScalarPair(first, second) => { - // Only the niche is guaranteed to be initialised, - // so use union layouts for the other primitive. - if niche_offset == Size::ZERO { - BackendRepr::ScalarPair(niche_scalar, second.to_union()) - } else { - BackendRepr::ScalarPair(first.to_union(), niche_scalar) + let largest_niche = Niche::from_scalar(dl, niche_offset, niche_scalar); + + let others_zst = variant_layouts + .iter_enumerated() + .all(|(i, layout)| i == largest_variant_index || layout.size == Size::ZERO); + let same_size = size == variant_layouts[largest_variant_index].size; + let same_align = align == variants_info[largest_variant_index].align_abi; + + let uninhabited = variant_layouts.iter().all(|v| v.is_uninhabited()); + let abi = if same_size && same_align && others_zst { + match variant_layouts[largest_variant_index].backend_repr { + // When the total alignment and size match, we can use the + // same ABI as the scalar variant with the reserved niche. + BackendRepr::Scalar(_) => BackendRepr::Scalar(niche_scalar), + BackendRepr::ScalarPair(first, second) => { + // Only the niche is guaranteed to be initialised, + // so use union layouts for the other primitive. + if niche_offset == Size::ZERO { + BackendRepr::ScalarPair(niche_scalar, second.to_union()) + } else { + BackendRepr::ScalarPair(first.to_union(), niche_scalar) + } + } + _ => BackendRepr::Memory { sized: true }, } - } - _ => BackendRepr::Memory { sized: true }, - } - } else { - BackendRepr::Memory { sized: true } - }; + } else { + BackendRepr::Memory { sized: true } + }; - let layout = LayoutData { - variants: Variants::Multiple { - tag: niche_scalar, - tag_encoding: TagEncoding::Niche { - untagged_variant: largest_variant_index, - niche_variants, - niche_start, - }, - tag_field: FieldIdx::new(0), - variants: variant_layouts, - }, - fields: FieldsShape::Arbitrary { - offsets: [niche_offset].into(), - in_memory_order: [FieldIdx::new(0)].into(), - }, - backend_repr: abi, - largest_niche, - uninhabited, - size, - align: AbiAlign::new(align), - max_repr_align, - unadjusted_abi_align, - randomization_seed: combined_seed, - }; + let layout = LayoutData { + variants: Variants::Multiple { + tag: niche_scalar, + tag_encoding: TagEncoding::Niche { + untagged_variant: largest_variant_index, + niche_variants, + niche_start, + }, + tag_field: FieldIdx::new(0), + variants: variant_layouts, + }, + fields: FieldsShape::Arbitrary { + offsets: [niche_offset].into(), + in_memory_order: [FieldIdx::new(0)].into(), + }, + backend_repr: abi, + largest_niche, + uninhabited, + size, + align: AbiAlign::new(align), + max_repr_align, + unadjusted_abi_align, + randomization_seed: combined_seed, + }; + + Some(layout) + }; - Some(layout) + let max_variant_size = variant_layouts.iter().map(|layout| layout.size).max()?; + // Chooses the first max-sized niche-providing variant whose layout succeeds. + for (largest_variant_index, _) in + variant_layouts.iter_enumerated().filter(|&(_, layout)| { + layout.size == max_variant_size && layout.largest_niche.is_some() + }) + { + if let Some(layout) = try_niche_variant(largest_variant_index) { + return Some(layout); + } + } + + None }; let niche_filling_layout = calculate_niche_filling_layout(); @@ -1091,6 +1125,121 @@ impl LayoutCalculator { Ok(best_layout) } + fn try_layout_variant_around_niche< + 'a, + FieldIdx: Idx, + VariantIdx: Idx, + F: Deref> + fmt::Debug + Copy, + >( + &self, + fields: &IndexSlice, + repr: &ReprOptions, + variant_index: VariantIdx, + total_size: Size, + total_align: Align, + niche_offset: Size, + niche_size: Size, + ) -> Option> { + let dl = self.cx.data_layout(); + + if repr.inhibit_struct_field_reordering() { + return None; + } + if fields.iter().any(|f| f.is_unsized()) { + return None; + } + + // This is only a necessary condition: alignment can still make placement fail below. + let min_used_size = fields.iter().try_fold(niche_size, |size, field| { + if field.is_zst() { Some(size) } else { size.checked_add(field.size, dl) } + })?; + if min_used_size > total_size { + return None; + } + + let mut offsets = IndexVec::from_elem(Size::ZERO, fields); + let mut in_memory_order: IndexVec = fields.indices().collect(); + + in_memory_order.raw.sort_by_key(|&i| { + let field = &fields[i]; + + let field_align = if let Some(pack) = repr.pack { + field.align.min(AbiAlign::new(pack)) + } else { + field.align + }; + + (cmp::Reverse(field_align.abi.bytes()), cmp::Reverse(field.size.bytes())) + }); + + let niche_end = niche_offset.checked_add(niche_size, dl)?; + let mut occupied = vec![(niche_offset, niche_end)]; + + let mut max_repr_align = repr.align; + let mut unadjusted_abi_align = + if repr.pack.is_some() { dl.i8_align } else { dl.aggregate_align }; + + for &i in &in_memory_order { + let field = &fields[i]; + + let field_align = if let Some(pack) = repr.pack { + field.align.min(AbiAlign::new(pack)) + } else { + field.align + }; + + max_repr_align = max_repr_align.max(field.max_repr_align); + unadjusted_abi_align = unadjusted_abi_align.max(field_align.abi); + + let mut candidate = Size::ZERO; + 'search: loop { + candidate = candidate.align_to(field_align.abi); + + let end = candidate.checked_add(field.size, dl)?; + if end > total_size { + return None; + } + + for &(occupied_start, occupied_end) in &occupied { + if end <= occupied_start { + break; + } + if candidate < occupied_end { + candidate = occupied_end; + continue 'search; + } + } + + offsets[i] = candidate; + + if !field.is_zst() { + occupied.push((candidate, end)); + occupied.sort_by_key(|&(start, _end)| start); + } + + break; + } + } + + in_memory_order.raw.sort_by_key(|&i| offsets[i]); + + Some(LayoutData { + variants: Variants::Single { index: variant_index }, + fields: FieldsShape::Arbitrary { offsets, in_memory_order }, + backend_repr: BackendRepr::Memory { sized: true }, + largest_niche: None, + uninhabited: fields.iter().any(|f| f.is_uninhabited()), + align: AbiAlign::new(total_align), + size: total_size, + max_repr_align, + unadjusted_abi_align, + randomization_seed: fields + .iter() + .fold(Hash64::ZERO, |acc, f| acc.wrapping_add(f.randomization_seed)) + .wrapping_add(repr.field_shuffle_seed), + }) + } + fn univariant_biased< 'a, FieldIdx: Idx, diff --git a/compiler/rustc_abi/src/lib.rs b/compiler/rustc_abi/src/lib.rs index 166c8bea6f354..5aaa0a7e3eeb4 100644 --- a/compiler/rustc_abi/src/lib.rs +++ b/compiler/rustc_abi/src/lib.rs @@ -94,7 +94,6 @@ bitflags! { /// See [`TyAndLayout::pass_indirectly_in_non_rustic_abis`] for details. const PASS_INDIRECTLY_IN_NON_RUSTIC_ABIS = 1 << 5; const IS_SCALABLE = 1 << 6; - // Any of these flags being set prevent field reordering optimisation. const FIELD_ORDER_UNOPTIMIZABLE = ReprFlags::IS_C.bits() | ReprFlags::IS_SIMD.bits() | ReprFlags::IS_SCALABLE.bits() diff --git a/compiler/rustc_ast/src/ast.rs b/compiler/rustc_ast/src/ast.rs index b8339c74dac03..a13d41fce8ecc 100644 --- a/compiler/rustc_ast/src/ast.rs +++ b/compiler/rustc_ast/src/ast.rs @@ -4359,7 +4359,13 @@ mod size_asserts { static_assert_size!(GenericBound, 88); static_assert_size!(Generics, 40); static_assert_size!(Impl, 80); + #[cfg(not(bootstrap))] + static_assert_size!(Item, 144); + #[cfg(bootstrap)] static_assert_size!(Item, 152); + #[cfg(not(bootstrap))] + static_assert_size!(ItemKind, 80); + #[cfg(bootstrap)] static_assert_size!(ItemKind, 88); static_assert_size!(LitKind, 24); static_assert_size!(Local, 96); diff --git a/tests/ui/print_type_sizes/padding.stdout b/tests/ui/print_type_sizes/padding.stdout index 9afdf76245df7..d12ff401f47f3 100644 --- a/tests/ui/print_type_sizes/padding.stdout +++ b/tests/ui/print_type_sizes/padding.stdout @@ -1,21 +1,15 @@ -print-type-size type: `E1`: 12 bytes, alignment: 4 bytes -print-type-size discriminant: 1 bytes -print-type-size variant `B`: 11 bytes -print-type-size padding: 3 bytes -print-type-size field `.0`: 8 bytes, alignment: 4 bytes -print-type-size variant `A`: 7 bytes +print-type-size type: `E1`: 8 bytes, alignment: 4 bytes +print-type-size variant `B`: 8 bytes +print-type-size field `.0`: 8 bytes +print-type-size variant `A`: 5 bytes +print-type-size field `.0`: 4 bytes print-type-size field `.1`: 1 bytes -print-type-size padding: 2 bytes -print-type-size field `.0`: 4 bytes, alignment: 4 bytes -print-type-size type: `E2`: 12 bytes, alignment: 4 bytes -print-type-size discriminant: 1 bytes -print-type-size variant `B`: 11 bytes -print-type-size padding: 3 bytes -print-type-size field `.0`: 8 bytes, alignment: 4 bytes -print-type-size variant `A`: 7 bytes +print-type-size type: `E2`: 8 bytes, alignment: 4 bytes +print-type-size variant `B`: 8 bytes +print-type-size field `.0`: 8 bytes +print-type-size variant `A`: 5 bytes +print-type-size field `.1`: 4 bytes print-type-size field `.0`: 1 bytes -print-type-size padding: 2 bytes -print-type-size field `.1`: 4 bytes, alignment: 4 bytes print-type-size type: `S`: 8 bytes, alignment: 4 bytes print-type-size field `.g`: 4 bytes print-type-size field `.a`: 1 bytes diff --git a/tests/ui/stats/input-stats.rs b/tests/ui/stats/input-stats.rs index 86e904202c1b1..ee4706ce30705 100644 --- a/tests/ui/stats/input-stats.rs +++ b/tests/ui/stats/input-stats.rs @@ -14,6 +14,7 @@ // bump occurs, stage1 and stage2 will give different outputs for this test. // Add an `ignore-stage1` comment marker to work around that problem during // that time. +//@ ignore-stage1 // The aim here is to include at least one of every different type of top-level diff --git a/tests/ui/stats/input-stats.stderr b/tests/ui/stats/input-stats.stderr index b4d8277a46c00..3f68921947da4 100644 --- a/tests/ui/stats/input-stats.stderr +++ b/tests/ui/stats/input-stats.stderr @@ -2,14 +2,14 @@ ast-stats ================================================================ ast-stats POST EXPANSION AST STATS: input_stats ast-stats Name Accumulated Size Count Item Size ast-stats ---------------------------------------------------------------- -ast-stats Item 1_672 (NN.N%) 11 152 -ast-stats - Enum 152 (NN.N%) 1 -ast-stats - ExternCrate 152 (NN.N%) 1 -ast-stats - ForeignMod 152 (NN.N%) 1 -ast-stats - Impl 152 (NN.N%) 1 -ast-stats - Trait 152 (NN.N%) 1 -ast-stats - Fn 304 (NN.N%) 2 -ast-stats - Use 608 (NN.N%) 4 +ast-stats Item 1_584 (NN.N%) 11 144 +ast-stats - Enum 144 (NN.N%) 1 +ast-stats - ExternCrate 144 (NN.N%) 1 +ast-stats - ForeignMod 144 (NN.N%) 1 +ast-stats - Impl 144 (NN.N%) 1 +ast-stats - Trait 144 (NN.N%) 1 +ast-stats - Fn 288 (NN.N%) 2 +ast-stats - Use 576 (NN.N%) 4 ast-stats Ty 896 (NN.N%) 14 64 ast-stats - Ptr 64 (NN.N%) 1 ast-stats - Ref 64 (NN.N%) 1 @@ -57,7 +57,7 @@ ast-stats GenericArgs 40 (NN.N%) 1 40 ast-stats - AngleBracketed 40 (NN.N%) 1 ast-stats Crate 40 (NN.N%) 1 40 ast-stats ---------------------------------------------------------------- -ast-stats Total 7_624 127 +ast-stats Total 7_536 127 ast-stats ================================================================ hir-stats ================================================================ hir-stats HIR STATS: input_stats diff --git a/tests/ui/structs-enums/enum-niche-fill-around-tag.rs b/tests/ui/structs-enums/enum-niche-fill-around-tag.rs new file mode 100644 index 0000000000000..31f137e4057d1 --- /dev/null +++ b/tests/ui/structs-enums/enum-niche-fill-around-tag.rs @@ -0,0 +1,126 @@ +//@ run-pass + +#![allow(dead_code)] +#![feature(offset_of_enum)] + +use std::mem::{align_of, offset_of}; +use std::mem::size_of; +use std::num::NonZero; + +enum Inner { + A(u8, u32), + B(u32), +} + +enum Outer { + A(Inner), + B(u32, u8), +} + +struct WithPadding { + a: bool, + b: bool, + c: u32, +} + +enum RepackPadding { + A(u32, u8), + B(WithPadding), +} + +struct RepackedBase { + int32: u32, + boolean: bool, +} + +enum RepackedAroundNiche { + A(RepackedBase), + B(u16, u32, u8), +} + +enum NestedRepackedAroundNiche { + A(RepackedAroundNiche), + B(u16, u32, u8), +} + +struct RepackedPair { + field2: u16, + field3: u8, +} + +enum RepackedNestedAggregate { + A(u16, RepackedPair), + B(NestedRepackedAroundNiche), +} + +struct FittingNichePayload { + word: u32, + flag: bool, +} + +// The first and last max-sized niche candidates cannot encode all variants. +// The middle candidate can, so the enum layout has to keep looking. +enum ChooseFittingNiche { + SmallNicheFront(u16, u32, NonZero), + WideNiche(FittingNichePayload), + SmallNicheBack(u16, u32, NonZero), + V0, + V1, + V2, +} + +fn main() { + assert_eq!(size_of::(), 8); + + assert_eq!(size_of::(), 8); + assert_eq!(size_of::(), 8); + + assert_eq!(size_of::(), 8); + assert_eq!(align_of::(), 4); + assert_eq!(offset_of!(RepackedBase, int32), 0); + assert_eq!(offset_of!(RepackedBase, boolean), 4); + + assert_eq!(size_of::(), 8); + assert_eq!(align_of::(), 4); + assert_eq!(size_of::>(), 8); + assert_eq!(offset_of!(RepackedAroundNiche, A.0), 0); + assert_eq!(offset_of!(RepackedAroundNiche, B.1), 0); + assert_eq!(offset_of!(RepackedAroundNiche, B.2), 5); + assert_eq!(offset_of!(RepackedAroundNiche, B.0), 6); + + assert_eq!(size_of::(), 8); + assert_eq!(align_of::(), 4); + assert_eq!(size_of::>(), 8); + assert_eq!(offset_of!(NestedRepackedAroundNiche, A.0), 0); + assert_eq!(offset_of!(NestedRepackedAroundNiche, B.1), 0); + assert_eq!(offset_of!(NestedRepackedAroundNiche, B.2), 5); + assert_eq!(offset_of!(NestedRepackedAroundNiche, B.0), 6); + + assert_eq!(size_of::(), 4); + assert_eq!(align_of::(), 2); + assert_eq!(offset_of!(RepackedPair, field2), 0); + assert_eq!(offset_of!(RepackedPair, field3), 2); + + assert_eq!(size_of::(), 8); + assert_eq!(align_of::(), 4); + assert_eq!(size_of::>(), 8); + assert_eq!(offset_of!(RepackedNestedAggregate, B.0), 0); + assert_eq!(offset_of!(RepackedNestedAggregate, A.1), 0); + assert_eq!(offset_of!(RepackedNestedAggregate, A.0), 6); + + assert_eq!(size_of::(), 8); + assert_eq!(align_of::(), 4); + assert_eq!(offset_of!(FittingNichePayload, word), 0); + assert_eq!(offset_of!(FittingNichePayload, flag), 4); + + assert_eq!(size_of::(), 8); + assert_eq!(align_of::(), 4); + assert_eq!(size_of::>(), 8); + assert_eq!(offset_of!(ChooseFittingNiche, WideNiche.0), 0); + assert_eq!(offset_of!(ChooseFittingNiche, SmallNicheFront.1), 0); + assert_eq!(offset_of!(ChooseFittingNiche, SmallNicheFront.2), 5); + assert_eq!(offset_of!(ChooseFittingNiche, SmallNicheFront.0), 6); + assert_eq!(offset_of!(ChooseFittingNiche, SmallNicheBack.1), 0); + assert_eq!(offset_of!(ChooseFittingNiche, SmallNicheBack.2), 5); + assert_eq!(offset_of!(ChooseFittingNiche, SmallNicheBack.0), 6); +}