From 60b3d05326af353f73312291f8dfe0eb31f993c0 Mon Sep 17 00:00:00 2001 From: James Barford-Evans Date: Tue, 26 May 2026 08:18:37 +0100 Subject: [PATCH 1/6] Experimenting with lazy creating the bitsets vector in a niave way --- compiler/rustc_index/src/bit_set.rs | 183 ++++++++++++++++------ compiler/rustc_index/src/bit_set/tests.rs | 55 +++++++ 2 files changed, 194 insertions(+), 44 deletions(-) diff --git a/compiler/rustc_index/src/bit_set.rs b/compiler/rustc_index/src/bit_set.rs index 2910ba7c46851..4cfc17167c3f7 100644 --- a/compiler/rustc_index/src/bit_set.rs +++ b/compiler/rustc_index/src/bit_set.rs @@ -1,3 +1,4 @@ +use std::hash::{Hash, Hasher}; use std::marker::PhantomData; use std::ops::{Bound, Range, RangeBounds}; use std::rc::Rc; @@ -112,10 +113,10 @@ macro_rules! bit_relations_inherent_impls { /// will panic if the bitsets have differing domain sizes. /// #[cfg_attr(feature = "nightly", derive(Decodable_NoContext, Encodable_NoContext))] -#[derive(Eq, PartialEq, Hash)] pub struct DenseBitSet { domain_size: usize, - words: Vec, + // Lazily allocated. `None` represents a full set of zero words. + words: Option>, marker: PhantomData, } @@ -124,22 +125,42 @@ impl DenseBitSet { pub fn domain_size(&self) -> usize { self.domain_size } + + #[inline] + fn num_words(&self) -> usize { + num_words(self.domain_size) + } + + #[inline] + fn words(&self) -> &[Word] { + self.words.as_deref().unwrap_or(&[]) + } + + #[inline] + fn ensure_words(&mut self) -> &mut Vec { + let num_words = self.num_words(); + self.words.get_or_insert_with(|| vec![0; num_words]) + } + + #[inline] + fn word(&self, word_index: usize) -> Word { + self.words().get(word_index).copied().unwrap_or(0) + } } impl DenseBitSet { /// Creates a new, empty bitset with a given `domain_size`. #[inline] pub fn new_empty(domain_size: usize) -> DenseBitSet { - let num_words = num_words(domain_size); - DenseBitSet { domain_size, words: vec![0; num_words], marker: PhantomData } + DenseBitSet { domain_size, words: None, marker: PhantomData } } /// Creates a new, filled bitset with a given `domain_size`. #[inline] pub fn new_filled(domain_size: usize) -> DenseBitSet { let num_words = num_words(domain_size); - let mut result = - DenseBitSet { domain_size, words: vec![!0; num_words], marker: PhantomData }; + let words = if num_words == 0 { None } else { Some(vec![!0; num_words]) }; + let mut result = DenseBitSet { domain_size, words, marker: PhantomData }; result.clear_excess_bits(); result } @@ -147,17 +168,21 @@ impl DenseBitSet { /// Clear all elements. #[inline] pub fn clear(&mut self) { - self.words.fill(0); + if let Some(words) = &mut self.words { + words.fill(0); + } } /// Clear excess bits in the final word. fn clear_excess_bits(&mut self) { - clear_excess_bits_in_final_word(self.domain_size, &mut self.words); + if let Some(words) = &mut self.words { + clear_excess_bits_in_final_word(self.domain_size, words); + } } /// Count the number of set bits in the set. pub fn count(&self) -> usize { - count_ones(&self.words) + count_ones(self.words()) } /// Returns `true` if `self` contains `elem`. @@ -165,20 +190,25 @@ impl DenseBitSet { pub fn contains(&self, elem: T) -> bool { assert!(elem.index() < self.domain_size); let (word_index, mask) = word_index_and_mask(elem); - (self.words[word_index] & mask) != 0 + (self.word(word_index) & mask) != 0 } /// Is `self` is a (non-strict) superset of `other`? #[inline] pub fn superset(&self, other: &DenseBitSet) -> bool { assert_eq!(self.domain_size, other.domain_size); - self.words.iter().zip(&other.words).all(|(a, b)| (a & b) == *b) + match (&self.words, &other.words) { + (_, None) => true, + (None, Some(other_words)) => other_words.iter().all(|&word| word == 0), + (Some(_), Some(_)) => (0..self.num_words()) + .all(|index| (self.word(index) & other.word(index)) == other.word(index)), + } } /// Is the set empty? #[inline] pub fn is_empty(&self) -> bool { - self.words.iter().all(|a| *a == 0) + self.words().iter().all(|a| *a == 0) } /// Insert `elem`. Returns whether the set has changed. @@ -191,7 +221,7 @@ impl DenseBitSet { self.domain_size, ); let (word_index, mask) = word_index_and_mask(elem); - let word_ref = &mut self.words[word_index]; + let word_ref = &mut self.ensure_words()[word_index]; let word = *word_ref; let new_word = word | mask; *word_ref = new_word; @@ -206,28 +236,32 @@ impl DenseBitSet { let (start_word_index, start_mask) = word_index_and_mask(start); let (end_word_index, end_mask) = word_index_and_mask(end); + let words = self.ensure_words(); // Set all words in between start and end (exclusively of both). for word_index in (start_word_index + 1)..end_word_index { - self.words[word_index] = !0; + words[word_index] = !0; } if start_word_index != end_word_index { // Start and end are in different words, so we handle each in turn. // // We set all leading bits. This includes the start_mask bit. - self.words[start_word_index] |= !(start_mask - 1); + words[start_word_index] |= !(start_mask - 1); // And all trailing bits (i.e. from 0..=end) in the end word, // including the end. - self.words[end_word_index] |= end_mask | (end_mask - 1); + words[end_word_index] |= end_mask | (end_mask - 1); } else { - self.words[start_word_index] |= end_mask | (end_mask - start_mask); + words[start_word_index] |= end_mask | (end_mask - start_mask); } } /// Sets all bits to true. pub fn insert_all(&mut self) { - self.words.fill(!0); + if self.domain_size == 0 { + return; + } + self.ensure_words().fill(!0); self.clear_excess_bits(); } @@ -241,16 +275,16 @@ impl DenseBitSet { let (end_word_index, end_mask) = word_index_and_mask(end); if start_word_index == end_word_index { - self.words[start_word_index] & (end_mask | (end_mask - start_mask)) != 0 + self.word(start_word_index) & (end_mask | (end_mask - start_mask)) != 0 } else { - if self.words[start_word_index] & !(start_mask - 1) != 0 { + if self.word(start_word_index) & !(start_mask - 1) != 0 { return true; } let remaining = start_word_index + 1..end_word_index; if remaining.start <= remaining.end { - self.words[remaining].iter().any(|&w| w != 0) - || self.words[end_word_index] & (end_mask | (end_mask - 1)) != 0 + self.words().get(remaining).is_some_and(|words| words.iter().any(|&w| w != 0)) + || self.word(end_word_index) & (end_mask | (end_mask - 1)) != 0 } else { false } @@ -261,8 +295,11 @@ impl DenseBitSet { #[inline] pub fn remove(&mut self, elem: T) -> bool { assert!(elem.index() < self.domain_size); + let Some(words) = &mut self.words else { + return false; + }; let (word_index, mask) = word_index_and_mask(elem); - let word_ref = &mut self.words[word_index]; + let word_ref = &mut words[word_index]; let word = *word_ref; let new_word = word & !mask; *word_ref = new_word; @@ -272,15 +309,18 @@ impl DenseBitSet { /// Iterates over the indices of set bits in a sorted order. #[inline] pub fn iter(&self) -> BitIter<'_, T> { - BitIter::new(&self.words) + BitIter::new(self.words()) } pub fn last_set_in(&self, range: impl RangeBounds) -> Option { let (start, end) = inclusive_start_end(range, self.domain_size)?; + let Some(words) = &self.words else { + return None; + }; let (start_word_index, _) = word_index_and_mask(start); let (end_word_index, end_mask) = word_index_and_mask(end); - let end_word = self.words[end_word_index] & (end_mask | (end_mask - 1)); + let end_word = words[end_word_index] & (end_mask | (end_mask - 1)); if end_word != 0 { let pos = max_bit(end_word) + WORD_BITS * end_word_index; if start <= pos { @@ -291,11 +331,10 @@ impl DenseBitSet { // We exclude end_word_index from the range here, because we don't want // to limit ourselves to *just* the last word: the bits set it in may be // after `end`, so it may not work out. - if let Some(offset) = - self.words[start_word_index..end_word_index].iter().rposition(|&w| w != 0) + if let Some(offset) = words[start_word_index..end_word_index].iter().rposition(|&w| w != 0) { let word_idx = start_word_index + offset; - let start_word = self.words[word_idx]; + let start_word = words[word_idx]; let pos = max_bit(start_word) + WORD_BITS * word_idx; if start <= pos { return Some(T::new(pos)); @@ -319,7 +358,13 @@ impl DenseBitSet { // quickly and accurately detect whether the update changed anything. // But that's only worth doing if there's an actual use-case. - update_words(&mut self.words, &other.words, |a, b| a | !b); + let other_words = other.words(); + let words = self.ensure_words(); + if other_words.is_empty() { + words.fill(!0); + } else { + update_words(words, other_words, |a, b| a | !b); + } // The bitwise update `a | !b` can result in the last word containing // out-of-domain bits, so we need to clear them. self.clear_excess_bits(); @@ -330,17 +375,38 @@ impl DenseBitSet { impl BitRelations> for DenseBitSet { fn union(&mut self, other: &DenseBitSet) -> bool { assert_eq!(self.domain_size, other.domain_size); - update_words(&mut self.words, &other.words, |a, b| a | b) + let Some(other_words) = &other.words else { + return false; + }; + let Some(words) = &mut self.words else { + if other_words.iter().all(|&word| word == 0) { + return false; + } + self.words = Some(other_words.clone()); + return true; + }; + update_words(words, other_words, |a, b| a | b) } fn subtract(&mut self, other: &DenseBitSet) -> bool { assert_eq!(self.domain_size, other.domain_size); - update_words(&mut self.words, &other.words, |a, b| a & !b) + let (Some(words), Some(other_words)) = (&mut self.words, &other.words) else { + return false; + }; + update_words(words, other_words, |a, b| a & !b) } fn intersect(&mut self, other: &DenseBitSet) -> bool { assert_eq!(self.domain_size, other.domain_size); - update_words(&mut self.words, &other.words, |a, b| a & b) + let Some(words) = &mut self.words else { + return false; + }; + let Some(other_words) = &other.words else { + let changed = words.iter().any(|&word| word != 0); + words.fill(0); + return changed; + }; + update_words(words, other_words, |a, b| a & b) } } @@ -365,6 +431,27 @@ impl Clone for DenseBitSet { } } +impl PartialEq for DenseBitSet { + fn eq(&self, other: &Self) -> bool { + self.domain_size == other.domain_size + && (0..self.num_words()).all(|index| self.word(index) == other.word(index)) + } +} + +impl Eq for DenseBitSet {} + +impl Hash for DenseBitSet { + fn hash(&self, state: &mut H) { + self.domain_size.hash(state); + + let num_words = self.num_words(); + num_words.hash(state); + for index in 0..num_words { + self.word(index).hash(state); + } + } +} + impl fmt::Debug for DenseBitSet { fn fmt(&self, w: &mut fmt::Formatter<'_>) -> fmt::Result { w.debug_list().entries(self.iter()).finish() @@ -380,8 +467,8 @@ impl ToString for DenseBitSet { // i tracks how many bits we have printed so far. let mut i = 0; - for word in &self.words { - let mut word = *word; + for word_index in 0..self.num_words() { + let mut word = self.word(word_index); for _ in 0..WORD_BYTES { // for each byte in `word`: let remain = self.domain_size - i; @@ -1306,8 +1393,10 @@ impl GrowableBitSet { } let min_num_words = num_words(min_domain_size); - if self.bit_set.words.len() < min_num_words { - self.bit_set.words.resize(min_num_words, 0) + if let Some(words) = &mut self.bit_set.words { + if words.len() < min_num_words { + words.resize(min_num_words, 0) + } } } @@ -1357,7 +1446,7 @@ impl GrowableBitSet { #[inline] pub fn contains(&self, elem: T) -> bool { let (word_index, mask) = word_index_and_mask(elem); - self.bit_set.words.get(word_index).is_some_and(|word| (word & mask) != 0) + self.bit_set.words().get(word_index).is_some_and(|word| (word & mask) != 0) } #[inline] @@ -1419,13 +1508,16 @@ impl BitMatrix { pub fn from_row_n(row: &DenseBitSet, num_rows: usize) -> BitMatrix { let num_columns = row.domain_size(); let words_per_row = num_words(num_columns); - assert_eq!(words_per_row, row.words.len()); - BitMatrix { - num_rows, - num_columns, - words: iter::repeat_n(&row.words, num_rows).flatten().cloned().collect(), - marker: PhantomData, + let mut words = vec![0; num_rows * words_per_row]; + if let Some(row_words) = &row.words { + assert_eq!(words_per_row, row_words.len()); + if words_per_row > 0 { + for matrix_row in words.chunks_exact_mut(words_per_row) { + matrix_row.copy_from_slice(row_words); + } + } } + BitMatrix { num_rows, num_columns, words, marker: PhantomData } } pub fn rows(&self) -> impl Iterator { @@ -1517,8 +1609,11 @@ impl BitMatrix { pub fn union_row_with(&mut self, with: &DenseBitSet, write: R) -> bool { assert!(write.index() < self.num_rows); assert_eq!(with.domain_size(), self.num_columns); + let Some(with_words) = &with.words else { + return false; + }; let (write_start, write_end) = self.range(write); - update_words(&mut self.words[write_start..write_end], &with.words, |a, b| a | b) + update_words(&mut self.words[write_start..write_end], with_words, |a, b| a | b) } /// Sets every cell in `row` to true. diff --git a/compiler/rustc_index/src/bit_set/tests.rs b/compiler/rustc_index/src/bit_set/tests.rs index 243ac20ee441c..2a08c72a02af2 100644 --- a/compiler/rustc_index/src/bit_set/tests.rs +++ b/compiler/rustc_index/src/bit_set/tests.rs @@ -1,6 +1,8 @@ use super::*; extern crate test; +use std::collections::hash_map::DefaultHasher; +use std::hash::{Hash, Hasher}; use std::hint::black_box; use test::Bencher; @@ -58,6 +60,59 @@ fn bitset_clone_from() { assert_eq!(b.iter().collect::>(), []); } +#[test] +fn dense_empty_bitset_is_lazy() { + fn hash(value: &T) -> u64 { + let mut hasher = DefaultHasher::new(); + value.hash(&mut hasher); + hasher.finish() + } + + let mut lazy: DenseBitSet = DenseBitSet::new_empty(130); + assert!(lazy.words.is_none()); + assert!(lazy.is_empty()); + assert_eq!(lazy.count(), 0); + assert!(!lazy.contains(64)); + assert!(!lazy.contains_any(0..130)); + assert_eq!(lazy.last_set_in(0..130), None); + assert_eq!(lazy.iter().next(), None); + assert!(!lazy.remove(64)); + assert!(lazy.words.is_none()); + + let mut allocated_empty: DenseBitSet = DenseBitSet::new_empty(130); + assert!(allocated_empty.insert(64)); + assert!(allocated_empty.remove(64)); + assert!(allocated_empty.words.is_some()); + + assert_eq!(lazy, allocated_empty); + assert_eq!(hash(&lazy), hash(&allocated_empty)); + assert!(lazy.superset(&allocated_empty)); + assert!(allocated_empty.superset(&lazy)); + assert!(!lazy.union(&allocated_empty)); + assert!(lazy.words.is_none()); + + assert!(lazy.insert(129)); + assert!(lazy.words.is_some()); + assert!(lazy.contains(129)); +} + +#[test] +fn lazy_dense_bitset_adapters() { + let row = DenseBitSet::::new_empty(130); + let mut matrix = BitMatrix::::from_row_n(&row, 3); + assert_eq!(matrix.words().len(), 3 * num_words(130)); + assert!(matrix.words().iter().all(|&word| word == 0)); + assert!(!matrix.union_row_with(&row, 1)); + + let mut growable = GrowableBitSet::::with_capacity(130); + assert!(growable.bit_set.words.is_none()); + assert!(!growable.remove(64)); + assert!(growable.bit_set.words.is_none()); + assert!(growable.insert(64)); + assert!(growable.bit_set.words.is_some()); + assert!(growable.contains(64)); +} + #[test] fn union_two_sets() { let mut set1: DenseBitSet = DenseBitSet::new_empty(65); From c7b6e218d856c9727294638ea8a16070a53cde56 Mon Sep 17 00:00:00 2001 From: James Barford-Evans Date: Wed, 10 Jun 2026 13:10:19 +0100 Subject: [PATCH 2/6] reset experiments --- compiler/rustc_index/src/bit_set.rs | 183 ++++++---------------- compiler/rustc_index/src/bit_set/tests.rs | 55 ------- 2 files changed, 44 insertions(+), 194 deletions(-) diff --git a/compiler/rustc_index/src/bit_set.rs b/compiler/rustc_index/src/bit_set.rs index 4cfc17167c3f7..2910ba7c46851 100644 --- a/compiler/rustc_index/src/bit_set.rs +++ b/compiler/rustc_index/src/bit_set.rs @@ -1,4 +1,3 @@ -use std::hash::{Hash, Hasher}; use std::marker::PhantomData; use std::ops::{Bound, Range, RangeBounds}; use std::rc::Rc; @@ -113,10 +112,10 @@ macro_rules! bit_relations_inherent_impls { /// will panic if the bitsets have differing domain sizes. /// #[cfg_attr(feature = "nightly", derive(Decodable_NoContext, Encodable_NoContext))] +#[derive(Eq, PartialEq, Hash)] pub struct DenseBitSet { domain_size: usize, - // Lazily allocated. `None` represents a full set of zero words. - words: Option>, + words: Vec, marker: PhantomData, } @@ -125,42 +124,22 @@ impl DenseBitSet { pub fn domain_size(&self) -> usize { self.domain_size } - - #[inline] - fn num_words(&self) -> usize { - num_words(self.domain_size) - } - - #[inline] - fn words(&self) -> &[Word] { - self.words.as_deref().unwrap_or(&[]) - } - - #[inline] - fn ensure_words(&mut self) -> &mut Vec { - let num_words = self.num_words(); - self.words.get_or_insert_with(|| vec![0; num_words]) - } - - #[inline] - fn word(&self, word_index: usize) -> Word { - self.words().get(word_index).copied().unwrap_or(0) - } } impl DenseBitSet { /// Creates a new, empty bitset with a given `domain_size`. #[inline] pub fn new_empty(domain_size: usize) -> DenseBitSet { - DenseBitSet { domain_size, words: None, marker: PhantomData } + let num_words = num_words(domain_size); + DenseBitSet { domain_size, words: vec![0; num_words], marker: PhantomData } } /// Creates a new, filled bitset with a given `domain_size`. #[inline] pub fn new_filled(domain_size: usize) -> DenseBitSet { let num_words = num_words(domain_size); - let words = if num_words == 0 { None } else { Some(vec![!0; num_words]) }; - let mut result = DenseBitSet { domain_size, words, marker: PhantomData }; + let mut result = + DenseBitSet { domain_size, words: vec![!0; num_words], marker: PhantomData }; result.clear_excess_bits(); result } @@ -168,21 +147,17 @@ impl DenseBitSet { /// Clear all elements. #[inline] pub fn clear(&mut self) { - if let Some(words) = &mut self.words { - words.fill(0); - } + self.words.fill(0); } /// Clear excess bits in the final word. fn clear_excess_bits(&mut self) { - if let Some(words) = &mut self.words { - clear_excess_bits_in_final_word(self.domain_size, words); - } + clear_excess_bits_in_final_word(self.domain_size, &mut self.words); } /// Count the number of set bits in the set. pub fn count(&self) -> usize { - count_ones(self.words()) + count_ones(&self.words) } /// Returns `true` if `self` contains `elem`. @@ -190,25 +165,20 @@ impl DenseBitSet { pub fn contains(&self, elem: T) -> bool { assert!(elem.index() < self.domain_size); let (word_index, mask) = word_index_and_mask(elem); - (self.word(word_index) & mask) != 0 + (self.words[word_index] & mask) != 0 } /// Is `self` is a (non-strict) superset of `other`? #[inline] pub fn superset(&self, other: &DenseBitSet) -> bool { assert_eq!(self.domain_size, other.domain_size); - match (&self.words, &other.words) { - (_, None) => true, - (None, Some(other_words)) => other_words.iter().all(|&word| word == 0), - (Some(_), Some(_)) => (0..self.num_words()) - .all(|index| (self.word(index) & other.word(index)) == other.word(index)), - } + self.words.iter().zip(&other.words).all(|(a, b)| (a & b) == *b) } /// Is the set empty? #[inline] pub fn is_empty(&self) -> bool { - self.words().iter().all(|a| *a == 0) + self.words.iter().all(|a| *a == 0) } /// Insert `elem`. Returns whether the set has changed. @@ -221,7 +191,7 @@ impl DenseBitSet { self.domain_size, ); let (word_index, mask) = word_index_and_mask(elem); - let word_ref = &mut self.ensure_words()[word_index]; + let word_ref = &mut self.words[word_index]; let word = *word_ref; let new_word = word | mask; *word_ref = new_word; @@ -236,32 +206,28 @@ impl DenseBitSet { let (start_word_index, start_mask) = word_index_and_mask(start); let (end_word_index, end_mask) = word_index_and_mask(end); - let words = self.ensure_words(); // Set all words in between start and end (exclusively of both). for word_index in (start_word_index + 1)..end_word_index { - words[word_index] = !0; + self.words[word_index] = !0; } if start_word_index != end_word_index { // Start and end are in different words, so we handle each in turn. // // We set all leading bits. This includes the start_mask bit. - words[start_word_index] |= !(start_mask - 1); + self.words[start_word_index] |= !(start_mask - 1); // And all trailing bits (i.e. from 0..=end) in the end word, // including the end. - words[end_word_index] |= end_mask | (end_mask - 1); + self.words[end_word_index] |= end_mask | (end_mask - 1); } else { - words[start_word_index] |= end_mask | (end_mask - start_mask); + self.words[start_word_index] |= end_mask | (end_mask - start_mask); } } /// Sets all bits to true. pub fn insert_all(&mut self) { - if self.domain_size == 0 { - return; - } - self.ensure_words().fill(!0); + self.words.fill(!0); self.clear_excess_bits(); } @@ -275,16 +241,16 @@ impl DenseBitSet { let (end_word_index, end_mask) = word_index_and_mask(end); if start_word_index == end_word_index { - self.word(start_word_index) & (end_mask | (end_mask - start_mask)) != 0 + self.words[start_word_index] & (end_mask | (end_mask - start_mask)) != 0 } else { - if self.word(start_word_index) & !(start_mask - 1) != 0 { + if self.words[start_word_index] & !(start_mask - 1) != 0 { return true; } let remaining = start_word_index + 1..end_word_index; if remaining.start <= remaining.end { - self.words().get(remaining).is_some_and(|words| words.iter().any(|&w| w != 0)) - || self.word(end_word_index) & (end_mask | (end_mask - 1)) != 0 + self.words[remaining].iter().any(|&w| w != 0) + || self.words[end_word_index] & (end_mask | (end_mask - 1)) != 0 } else { false } @@ -295,11 +261,8 @@ impl DenseBitSet { #[inline] pub fn remove(&mut self, elem: T) -> bool { assert!(elem.index() < self.domain_size); - let Some(words) = &mut self.words else { - return false; - }; let (word_index, mask) = word_index_and_mask(elem); - let word_ref = &mut words[word_index]; + let word_ref = &mut self.words[word_index]; let word = *word_ref; let new_word = word & !mask; *word_ref = new_word; @@ -309,18 +272,15 @@ impl DenseBitSet { /// Iterates over the indices of set bits in a sorted order. #[inline] pub fn iter(&self) -> BitIter<'_, T> { - BitIter::new(self.words()) + BitIter::new(&self.words) } pub fn last_set_in(&self, range: impl RangeBounds) -> Option { let (start, end) = inclusive_start_end(range, self.domain_size)?; - let Some(words) = &self.words else { - return None; - }; let (start_word_index, _) = word_index_and_mask(start); let (end_word_index, end_mask) = word_index_and_mask(end); - let end_word = words[end_word_index] & (end_mask | (end_mask - 1)); + let end_word = self.words[end_word_index] & (end_mask | (end_mask - 1)); if end_word != 0 { let pos = max_bit(end_word) + WORD_BITS * end_word_index; if start <= pos { @@ -331,10 +291,11 @@ impl DenseBitSet { // We exclude end_word_index from the range here, because we don't want // to limit ourselves to *just* the last word: the bits set it in may be // after `end`, so it may not work out. - if let Some(offset) = words[start_word_index..end_word_index].iter().rposition(|&w| w != 0) + if let Some(offset) = + self.words[start_word_index..end_word_index].iter().rposition(|&w| w != 0) { let word_idx = start_word_index + offset; - let start_word = words[word_idx]; + let start_word = self.words[word_idx]; let pos = max_bit(start_word) + WORD_BITS * word_idx; if start <= pos { return Some(T::new(pos)); @@ -358,13 +319,7 @@ impl DenseBitSet { // quickly and accurately detect whether the update changed anything. // But that's only worth doing if there's an actual use-case. - let other_words = other.words(); - let words = self.ensure_words(); - if other_words.is_empty() { - words.fill(!0); - } else { - update_words(words, other_words, |a, b| a | !b); - } + update_words(&mut self.words, &other.words, |a, b| a | !b); // The bitwise update `a | !b` can result in the last word containing // out-of-domain bits, so we need to clear them. self.clear_excess_bits(); @@ -375,38 +330,17 @@ impl DenseBitSet { impl BitRelations> for DenseBitSet { fn union(&mut self, other: &DenseBitSet) -> bool { assert_eq!(self.domain_size, other.domain_size); - let Some(other_words) = &other.words else { - return false; - }; - let Some(words) = &mut self.words else { - if other_words.iter().all(|&word| word == 0) { - return false; - } - self.words = Some(other_words.clone()); - return true; - }; - update_words(words, other_words, |a, b| a | b) + update_words(&mut self.words, &other.words, |a, b| a | b) } fn subtract(&mut self, other: &DenseBitSet) -> bool { assert_eq!(self.domain_size, other.domain_size); - let (Some(words), Some(other_words)) = (&mut self.words, &other.words) else { - return false; - }; - update_words(words, other_words, |a, b| a & !b) + update_words(&mut self.words, &other.words, |a, b| a & !b) } fn intersect(&mut self, other: &DenseBitSet) -> bool { assert_eq!(self.domain_size, other.domain_size); - let Some(words) = &mut self.words else { - return false; - }; - let Some(other_words) = &other.words else { - let changed = words.iter().any(|&word| word != 0); - words.fill(0); - return changed; - }; - update_words(words, other_words, |a, b| a & b) + update_words(&mut self.words, &other.words, |a, b| a & b) } } @@ -431,27 +365,6 @@ impl Clone for DenseBitSet { } } -impl PartialEq for DenseBitSet { - fn eq(&self, other: &Self) -> bool { - self.domain_size == other.domain_size - && (0..self.num_words()).all(|index| self.word(index) == other.word(index)) - } -} - -impl Eq for DenseBitSet {} - -impl Hash for DenseBitSet { - fn hash(&self, state: &mut H) { - self.domain_size.hash(state); - - let num_words = self.num_words(); - num_words.hash(state); - for index in 0..num_words { - self.word(index).hash(state); - } - } -} - impl fmt::Debug for DenseBitSet { fn fmt(&self, w: &mut fmt::Formatter<'_>) -> fmt::Result { w.debug_list().entries(self.iter()).finish() @@ -467,8 +380,8 @@ impl ToString for DenseBitSet { // i tracks how many bits we have printed so far. let mut i = 0; - for word_index in 0..self.num_words() { - let mut word = self.word(word_index); + for word in &self.words { + let mut word = *word; for _ in 0..WORD_BYTES { // for each byte in `word`: let remain = self.domain_size - i; @@ -1393,10 +1306,8 @@ impl GrowableBitSet { } let min_num_words = num_words(min_domain_size); - if let Some(words) = &mut self.bit_set.words { - if words.len() < min_num_words { - words.resize(min_num_words, 0) - } + if self.bit_set.words.len() < min_num_words { + self.bit_set.words.resize(min_num_words, 0) } } @@ -1446,7 +1357,7 @@ impl GrowableBitSet { #[inline] pub fn contains(&self, elem: T) -> bool { let (word_index, mask) = word_index_and_mask(elem); - self.bit_set.words().get(word_index).is_some_and(|word| (word & mask) != 0) + self.bit_set.words.get(word_index).is_some_and(|word| (word & mask) != 0) } #[inline] @@ -1508,16 +1419,13 @@ impl BitMatrix { pub fn from_row_n(row: &DenseBitSet, num_rows: usize) -> BitMatrix { let num_columns = row.domain_size(); let words_per_row = num_words(num_columns); - let mut words = vec![0; num_rows * words_per_row]; - if let Some(row_words) = &row.words { - assert_eq!(words_per_row, row_words.len()); - if words_per_row > 0 { - for matrix_row in words.chunks_exact_mut(words_per_row) { - matrix_row.copy_from_slice(row_words); - } - } + assert_eq!(words_per_row, row.words.len()); + BitMatrix { + num_rows, + num_columns, + words: iter::repeat_n(&row.words, num_rows).flatten().cloned().collect(), + marker: PhantomData, } - BitMatrix { num_rows, num_columns, words, marker: PhantomData } } pub fn rows(&self) -> impl Iterator { @@ -1609,11 +1517,8 @@ impl BitMatrix { pub fn union_row_with(&mut self, with: &DenseBitSet, write: R) -> bool { assert!(write.index() < self.num_rows); assert_eq!(with.domain_size(), self.num_columns); - let Some(with_words) = &with.words else { - return false; - }; let (write_start, write_end) = self.range(write); - update_words(&mut self.words[write_start..write_end], with_words, |a, b| a | b) + update_words(&mut self.words[write_start..write_end], &with.words, |a, b| a | b) } /// Sets every cell in `row` to true. diff --git a/compiler/rustc_index/src/bit_set/tests.rs b/compiler/rustc_index/src/bit_set/tests.rs index 2a08c72a02af2..243ac20ee441c 100644 --- a/compiler/rustc_index/src/bit_set/tests.rs +++ b/compiler/rustc_index/src/bit_set/tests.rs @@ -1,8 +1,6 @@ use super::*; extern crate test; -use std::collections::hash_map::DefaultHasher; -use std::hash::{Hash, Hasher}; use std::hint::black_box; use test::Bencher; @@ -60,59 +58,6 @@ fn bitset_clone_from() { assert_eq!(b.iter().collect::>(), []); } -#[test] -fn dense_empty_bitset_is_lazy() { - fn hash(value: &T) -> u64 { - let mut hasher = DefaultHasher::new(); - value.hash(&mut hasher); - hasher.finish() - } - - let mut lazy: DenseBitSet = DenseBitSet::new_empty(130); - assert!(lazy.words.is_none()); - assert!(lazy.is_empty()); - assert_eq!(lazy.count(), 0); - assert!(!lazy.contains(64)); - assert!(!lazy.contains_any(0..130)); - assert_eq!(lazy.last_set_in(0..130), None); - assert_eq!(lazy.iter().next(), None); - assert!(!lazy.remove(64)); - assert!(lazy.words.is_none()); - - let mut allocated_empty: DenseBitSet = DenseBitSet::new_empty(130); - assert!(allocated_empty.insert(64)); - assert!(allocated_empty.remove(64)); - assert!(allocated_empty.words.is_some()); - - assert_eq!(lazy, allocated_empty); - assert_eq!(hash(&lazy), hash(&allocated_empty)); - assert!(lazy.superset(&allocated_empty)); - assert!(allocated_empty.superset(&lazy)); - assert!(!lazy.union(&allocated_empty)); - assert!(lazy.words.is_none()); - - assert!(lazy.insert(129)); - assert!(lazy.words.is_some()); - assert!(lazy.contains(129)); -} - -#[test] -fn lazy_dense_bitset_adapters() { - let row = DenseBitSet::::new_empty(130); - let mut matrix = BitMatrix::::from_row_n(&row, 3); - assert_eq!(matrix.words().len(), 3 * num_words(130)); - assert!(matrix.words().iter().all(|&word| word == 0)); - assert!(!matrix.union_row_with(&row, 1)); - - let mut growable = GrowableBitSet::::with_capacity(130); - assert!(growable.bit_set.words.is_none()); - assert!(!growable.remove(64)); - assert!(growable.bit_set.words.is_none()); - assert!(growable.insert(64)); - assert!(growable.bit_set.words.is_some()); - assert!(growable.contains(64)); -} - #[test] fn union_two_sets() { let mut set1: DenseBitSet = DenseBitSet::new_empty(65); From b8be5c33c90974d829add42fff25804d1e6cb5db Mon Sep 17 00:00:00 2001 From: James Barford-Evans Date: Wed, 10 Jun 2026 13:22:15 +0100 Subject: [PATCH 3/6] Try and use a small vec as opposed to `vec![...]` --- compiler/rustc_index/Cargo.toml | 3 +-- compiler/rustc_index/src/bit_set.rs | 8 +++++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/compiler/rustc_index/Cargo.toml b/compiler/rustc_index/Cargo.toml index edcd7816a60e0..49fc13ac93cce 100644 --- a/compiler/rustc_index/Cargo.toml +++ b/compiler/rustc_index/Cargo.toml @@ -8,7 +8,7 @@ edition = "2024" rustc_index_macros = { path = "../rustc_index_macros" } rustc_macros = { path = "../rustc_macros", optional = true } rustc_serialize = { path = "../rustc_serialize", optional = true } -smallvec = { version = "1.8.1", optional = true } +smallvec = { version = "1.8.1" } # tidy-alphabetical-end [features] @@ -17,7 +17,6 @@ default = ["nightly"] nightly = [ "dep:rustc_macros", "dep:rustc_serialize", - "dep:smallvec", "rustc_index_macros/nightly", ] rustc_randomized_layouts = [] diff --git a/compiler/rustc_index/src/bit_set.rs b/compiler/rustc_index/src/bit_set.rs index 2910ba7c46851..76cac2506d6cb 100644 --- a/compiler/rustc_index/src/bit_set.rs +++ b/compiler/rustc_index/src/bit_set.rs @@ -6,6 +6,7 @@ use std::{fmt, iter, slice}; use Chunk::*; #[cfg(feature = "nightly")] use rustc_macros::{Decodable_NoContext, Encodable_NoContext}; +use smallvec::{SmallVec, smallvec}; use crate::{Idx, IndexVec}; @@ -13,6 +14,7 @@ use crate::{Idx, IndexVec}; mod tests; type Word = u64; +type DenseBitSetWords = SmallVec<[Word; 1]>; const WORD_BYTES: usize = size_of::(); const WORD_BITS: usize = WORD_BYTES * 8; @@ -115,7 +117,7 @@ macro_rules! bit_relations_inherent_impls { #[derive(Eq, PartialEq, Hash)] pub struct DenseBitSet { domain_size: usize, - words: Vec, + words: DenseBitSetWords, marker: PhantomData, } @@ -131,7 +133,7 @@ impl DenseBitSet { #[inline] pub fn new_empty(domain_size: usize) -> DenseBitSet { let num_words = num_words(domain_size); - DenseBitSet { domain_size, words: vec![0; num_words], marker: PhantomData } + DenseBitSet { domain_size, words: smallvec![0; num_words], marker: PhantomData } } /// Creates a new, filled bitset with a given `domain_size`. @@ -139,7 +141,7 @@ impl DenseBitSet { pub fn new_filled(domain_size: usize) -> DenseBitSet { let num_words = num_words(domain_size); let mut result = - DenseBitSet { domain_size, words: vec![!0; num_words], marker: PhantomData }; + DenseBitSet { domain_size, words: smallvec![!0; num_words], marker: PhantomData }; result.clear_excess_bits(); result } From 71a3dba5d7bfe1d64b5737f394f8a202ed5bb8a1 Mon Sep 17 00:00:00 2001 From: Jamesbarford Date: Thu, 11 Jun 2026 13:09:48 +0000 Subject: [PATCH 4/6] reset changes --- compiler/rustc_index/Cargo.toml | 3 ++- compiler/rustc_index/src/bit_set.rs | 8 +++----- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/compiler/rustc_index/Cargo.toml b/compiler/rustc_index/Cargo.toml index 49fc13ac93cce..edcd7816a60e0 100644 --- a/compiler/rustc_index/Cargo.toml +++ b/compiler/rustc_index/Cargo.toml @@ -8,7 +8,7 @@ edition = "2024" rustc_index_macros = { path = "../rustc_index_macros" } rustc_macros = { path = "../rustc_macros", optional = true } rustc_serialize = { path = "../rustc_serialize", optional = true } -smallvec = { version = "1.8.1" } +smallvec = { version = "1.8.1", optional = true } # tidy-alphabetical-end [features] @@ -17,6 +17,7 @@ default = ["nightly"] nightly = [ "dep:rustc_macros", "dep:rustc_serialize", + "dep:smallvec", "rustc_index_macros/nightly", ] rustc_randomized_layouts = [] diff --git a/compiler/rustc_index/src/bit_set.rs b/compiler/rustc_index/src/bit_set.rs index 76cac2506d6cb..2910ba7c46851 100644 --- a/compiler/rustc_index/src/bit_set.rs +++ b/compiler/rustc_index/src/bit_set.rs @@ -6,7 +6,6 @@ use std::{fmt, iter, slice}; use Chunk::*; #[cfg(feature = "nightly")] use rustc_macros::{Decodable_NoContext, Encodable_NoContext}; -use smallvec::{SmallVec, smallvec}; use crate::{Idx, IndexVec}; @@ -14,7 +13,6 @@ use crate::{Idx, IndexVec}; mod tests; type Word = u64; -type DenseBitSetWords = SmallVec<[Word; 1]>; const WORD_BYTES: usize = size_of::(); const WORD_BITS: usize = WORD_BYTES * 8; @@ -117,7 +115,7 @@ macro_rules! bit_relations_inherent_impls { #[derive(Eq, PartialEq, Hash)] pub struct DenseBitSet { domain_size: usize, - words: DenseBitSetWords, + words: Vec, marker: PhantomData, } @@ -133,7 +131,7 @@ impl DenseBitSet { #[inline] pub fn new_empty(domain_size: usize) -> DenseBitSet { let num_words = num_words(domain_size); - DenseBitSet { domain_size, words: smallvec![0; num_words], marker: PhantomData } + DenseBitSet { domain_size, words: vec![0; num_words], marker: PhantomData } } /// Creates a new, filled bitset with a given `domain_size`. @@ -141,7 +139,7 @@ impl DenseBitSet { pub fn new_filled(domain_size: usize) -> DenseBitSet { let num_words = num_words(domain_size); let mut result = - DenseBitSet { domain_size, words: smallvec![!0; num_words], marker: PhantomData }; + DenseBitSet { domain_size, words: vec![!0; num_words], marker: PhantomData }; result.clear_excess_bits(); result } From ad47fc478e39bbe953354cd8474d9ac2e4566a7b Mon Sep 17 00:00:00 2001 From: Jamesbarford Date: Thu, 11 Jun 2026 13:57:30 +0000 Subject: [PATCH 5/6] More aggressively tried to port across some of the ideas from 141325 --- compiler/rustc_index/src/bit_set.rs | 588 +++++++++++++++++++--- compiler/rustc_index/src/bit_set/tests.rs | 109 ++++ 2 files changed, 639 insertions(+), 58 deletions(-) diff --git a/compiler/rustc_index/src/bit_set.rs b/compiler/rustc_index/src/bit_set.rs index 2910ba7c46851..19dc3de1685f0 100644 --- a/compiler/rustc_index/src/bit_set.rs +++ b/compiler/rustc_index/src/bit_set.rs @@ -1,11 +1,17 @@ +use std::alloc::{Layout, alloc, alloc_zeroed, dealloc, handle_alloc_error, realloc}; +use std::hash::{Hash, Hasher}; use std::marker::PhantomData; +use std::mem::ManuallyDrop; use std::ops::{Bound, Range, RangeBounds}; +use std::ptr::NonNull; use std::rc::Rc; use std::{fmt, iter, slice}; use Chunk::*; #[cfg(feature = "nightly")] use rustc_macros::{Decodable_NoContext, Encodable_NoContext}; +#[cfg(feature = "nightly")] +use rustc_serialize::{Decodable, Decoder, Encodable, Encoder}; use crate::{Idx, IndexVec}; @@ -111,53 +117,97 @@ macro_rules! bit_relations_inherent_impls { /// to or greater than the domain size. All operations that involve two bitsets /// will panic if the bitsets have differing domain sizes. /// -#[cfg_attr(feature = "nightly", derive(Decodable_NoContext, Encodable_NoContext))] -#[derive(Eq, PartialEq, Hash)] pub struct DenseBitSet { domain_size: usize, - words: Vec, + storage: DenseBitSetStorage, marker: PhantomData, } impl DenseBitSet { + const INLINE_CAPACITY: usize = WORD_BITS; + /// Gets the domain size. pub fn domain_size(&self) -> usize { self.domain_size } + + #[inline] + fn num_words(&self) -> usize { + num_words(self.domain_size) + } + + #[inline] + fn storage_words(&self) -> Option<&[Word]> { + self.storage.as_slice(self.domain_size) + } + + #[inline] + fn storage_words_mut(&mut self) -> Option<&mut [Word]> { + self.storage.as_mut_slice(self.domain_size) + } + + #[inline] + fn storage_words_mut_or_alloc(&mut self) -> &mut [Word] { + self.storage.as_mut_slice_or_alloc(self.domain_size) + } + + #[inline] + fn word(&self, word_index: usize) -> Word { + self.storage.word(self.domain_size, word_index) + } + + #[inline] + fn copy_words_to(&self, words: &mut [Word]) { + self.storage.copy_words_to(self.domain_size, words); + } + + #[inline] + fn ensure_domain_size(&mut self, min_domain_size: usize) { + if self.domain_size < min_domain_size { + self.storage.ensure_capacity(self.domain_size, min_domain_size); + self.domain_size = min_domain_size; + } + } } impl DenseBitSet { /// Creates a new, empty bitset with a given `domain_size`. #[inline] pub fn new_empty(domain_size: usize) -> DenseBitSet { - let num_words = num_words(domain_size); - DenseBitSet { domain_size, words: vec![0; num_words], marker: PhantomData } + DenseBitSet { + domain_size, + storage: DenseBitSetStorage::new_empty(domain_size), + marker: PhantomData, + } } /// Creates a new, filled bitset with a given `domain_size`. #[inline] pub fn new_filled(domain_size: usize) -> DenseBitSet { - let num_words = num_words(domain_size); - let mut result = - DenseBitSet { domain_size, words: vec![!0; num_words], marker: PhantomData }; - result.clear_excess_bits(); - result + DenseBitSet { + domain_size, + storage: DenseBitSetStorage::new_filled(domain_size), + marker: PhantomData, + } } /// Clear all elements. #[inline] pub fn clear(&mut self) { - self.words.fill(0); + self.storage.clear(self.domain_size); } /// Clear excess bits in the final word. fn clear_excess_bits(&mut self) { - clear_excess_bits_in_final_word(self.domain_size, &mut self.words); + let domain_size = self.domain_size; + if let Some(words) = self.storage_words_mut() { + clear_excess_bits_in_final_word(domain_size, words); + } } /// Count the number of set bits in the set. pub fn count(&self) -> usize { - count_ones(&self.words) + self.storage_words().map_or(0, count_ones) } /// Returns `true` if `self` contains `elem`. @@ -165,20 +215,29 @@ impl DenseBitSet { pub fn contains(&self, elem: T) -> bool { assert!(elem.index() < self.domain_size); let (word_index, mask) = word_index_and_mask(elem); - (self.words[word_index] & mask) != 0 + (self.word(word_index) & mask) != 0 } /// Is `self` is a (non-strict) superset of `other`? #[inline] pub fn superset(&self, other: &DenseBitSet) -> bool { assert_eq!(self.domain_size, other.domain_size); - self.words.iter().zip(&other.words).all(|(a, b)| (a & b) == *b) + if other.is_empty() { + return true; + } + if self.is_empty() { + return false; + } + + let self_words = self.storage_words().unwrap(); + let other_words = other.storage_words().unwrap(); + self_words.iter().zip(other_words).all(|(a, b)| (a & b) == *b) } /// Is the set empty? #[inline] pub fn is_empty(&self) -> bool { - self.words.iter().all(|a| *a == 0) + self.storage.is_empty(self.domain_size) } /// Insert `elem`. Returns whether the set has changed. @@ -191,7 +250,7 @@ impl DenseBitSet { self.domain_size, ); let (word_index, mask) = word_index_and_mask(elem); - let word_ref = &mut self.words[word_index]; + let word_ref = &mut self.storage_words_mut_or_alloc()[word_index]; let word = *word_ref; let new_word = word | mask; *word_ref = new_word; @@ -206,28 +265,29 @@ impl DenseBitSet { let (start_word_index, start_mask) = word_index_and_mask(start); let (end_word_index, end_mask) = word_index_and_mask(end); + let words = self.storage_words_mut_or_alloc(); // Set all words in between start and end (exclusively of both). for word_index in (start_word_index + 1)..end_word_index { - self.words[word_index] = !0; + words[word_index] = !0; } if start_word_index != end_word_index { // Start and end are in different words, so we handle each in turn. // // We set all leading bits. This includes the start_mask bit. - self.words[start_word_index] |= !(start_mask - 1); + words[start_word_index] |= !(start_mask - 1); // And all trailing bits (i.e. from 0..=end) in the end word, // including the end. - self.words[end_word_index] |= end_mask | (end_mask - 1); + words[end_word_index] |= end_mask | (end_mask - 1); } else { - self.words[start_word_index] |= end_mask | (end_mask - start_mask); + words[start_word_index] |= end_mask | (end_mask - start_mask); } } /// Sets all bits to true. pub fn insert_all(&mut self) { - self.words.fill(!0); + self.storage_words_mut_or_alloc().fill(!0); self.clear_excess_bits(); } @@ -239,18 +299,21 @@ impl DenseBitSet { }; let (start_word_index, start_mask) = word_index_and_mask(start); let (end_word_index, end_mask) = word_index_and_mask(end); + let Some(words) = self.storage_words() else { + return false; + }; if start_word_index == end_word_index { - self.words[start_word_index] & (end_mask | (end_mask - start_mask)) != 0 + words[start_word_index] & (end_mask | (end_mask - start_mask)) != 0 } else { - if self.words[start_word_index] & !(start_mask - 1) != 0 { + if words[start_word_index] & !(start_mask - 1) != 0 { return true; } let remaining = start_word_index + 1..end_word_index; if remaining.start <= remaining.end { - self.words[remaining].iter().any(|&w| w != 0) - || self.words[end_word_index] & (end_mask | (end_mask - 1)) != 0 + words[remaining].iter().any(|&w| w != 0) + || words[end_word_index] & (end_mask | (end_mask - 1)) != 0 } else { false } @@ -262,7 +325,10 @@ impl DenseBitSet { pub fn remove(&mut self, elem: T) -> bool { assert!(elem.index() < self.domain_size); let (word_index, mask) = word_index_and_mask(elem); - let word_ref = &mut self.words[word_index]; + let Some(words) = self.storage_words_mut() else { + return false; + }; + let word_ref = &mut words[word_index]; let word = *word_ref; let new_word = word & !mask; *word_ref = new_word; @@ -272,15 +338,19 @@ impl DenseBitSet { /// Iterates over the indices of set bits in a sorted order. #[inline] pub fn iter(&self) -> BitIter<'_, T> { - BitIter::new(&self.words) + match self.storage_words() { + Some(words) => BitIter::new(words), + None => BitIter::from_single_word(0), + } } pub fn last_set_in(&self, range: impl RangeBounds) -> Option { let (start, end) = inclusive_start_end(range, self.domain_size)?; let (start_word_index, _) = word_index_and_mask(start); let (end_word_index, end_mask) = word_index_and_mask(end); + let words = self.storage_words()?; - let end_word = self.words[end_word_index] & (end_mask | (end_mask - 1)); + let end_word = words[end_word_index] & (end_mask | (end_mask - 1)); if end_word != 0 { let pos = max_bit(end_word) + WORD_BITS * end_word_index; if start <= pos { @@ -291,11 +361,10 @@ impl DenseBitSet { // We exclude end_word_index from the range here, because we don't want // to limit ourselves to *just* the last word: the bits set it in may be // after `end`, so it may not work out. - if let Some(offset) = - self.words[start_word_index..end_word_index].iter().rposition(|&w| w != 0) + if let Some(offset) = words[start_word_index..end_word_index].iter().rposition(|&w| w != 0) { let word_idx = start_word_index + offset; - let start_word = self.words[word_idx]; + let start_word = words[word_idx]; let pos = max_bit(start_word) + WORD_BITS * word_idx; if start <= pos { return Some(T::new(pos)); @@ -319,7 +388,13 @@ impl DenseBitSet { // quickly and accurately detect whether the update changed anything. // But that's only worth doing if there's an actual use-case. - update_words(&mut self.words, &other.words, |a, b| a | !b); + if other.is_empty() { + self.insert_all(); + } else { + let lhs = self.storage_words_mut_or_alloc(); + let rhs = other.storage_words().unwrap(); + update_words(lhs, rhs, |a, b| a | !b); + } // The bitwise update `a | !b` can result in the last word containing // out-of-domain bits, so we need to clear them. self.clear_excess_bits(); @@ -330,17 +405,42 @@ impl DenseBitSet { impl BitRelations> for DenseBitSet { fn union(&mut self, other: &DenseBitSet) -> bool { assert_eq!(self.domain_size, other.domain_size); - update_words(&mut self.words, &other.words, |a, b| a | b) + if other.is_empty() { + false + } else if self.is_empty() { + *self = other.clone(); + true + } else { + let lhs = self.storage_words_mut_or_alloc(); + let rhs = other.storage_words().unwrap(); + update_words(lhs, rhs, |a, b| a | b) + } } fn subtract(&mut self, other: &DenseBitSet) -> bool { assert_eq!(self.domain_size, other.domain_size); - update_words(&mut self.words, &other.words, |a, b| a & !b) + if self.is_empty() || other.is_empty() { + false + } else { + let lhs = self.storage_words_mut().unwrap(); + let rhs = other.storage_words().unwrap(); + update_words(lhs, rhs, |a, b| a & !b) + } } fn intersect(&mut self, other: &DenseBitSet) -> bool { assert_eq!(self.domain_size, other.domain_size); - update_words(&mut self.words, &other.words, |a, b| a & b) + if self.is_empty() { + false + } else if other.is_empty() { + let changed = !self.is_empty(); + self.clear(); + changed + } else { + let lhs = self.storage_words_mut().unwrap(); + let rhs = other.storage_words().unwrap(); + update_words(lhs, rhs, |a, b| a & b) + } } } @@ -354,14 +454,74 @@ impl Clone for DenseBitSet { fn clone(&self) -> Self { DenseBitSet { domain_size: self.domain_size, - words: self.words.clone(), + storage: self.storage.clone(self.domain_size), marker: PhantomData, } } fn clone_from(&mut self, from: &Self) { - self.domain_size = from.domain_size; - self.words.clone_from(&from.words); + *self = from.clone(); + } +} + +impl Drop for DenseBitSet { + fn drop(&mut self) { + self.storage.drop(self.domain_size); + } +} + +impl PartialEq for DenseBitSet { + fn eq(&self, other: &Self) -> bool { + self.domain_size == other.domain_size + && (0..self.num_words()) + .all(|word_index| self.word(word_index) == other.word(word_index)) + } +} + +impl Eq for DenseBitSet {} + +impl Hash for DenseBitSet { + fn hash(&self, state: &mut H) { + self.domain_size.hash(state); + self.num_words().hash(state); + for word_index in 0..self.num_words() { + self.word(word_index).hash(state); + } + } +} + +#[cfg(feature = "nightly")] +impl Encodable for DenseBitSet { + fn encode(&self, s: &mut S) { + self.domain_size.encode(s); + self.num_words().encode(s); + for word_index in 0..self.num_words() { + self.word(word_index).encode(s); + } + } +} + +#[cfg(feature = "nightly")] +impl Decodable for DenseBitSet { + fn decode(d: &mut D) -> Self { + let domain_size = usize::decode(d); + let decoded_words = usize::decode(d); + assert_eq!(decoded_words, num_words(domain_size)); + + let mut storage = DenseBitSetStorage::new_empty(domain_size); + if decoded_words != 0 { + let mut words = Vec::with_capacity(decoded_words); + for _ in 0..decoded_words { + words.push(Word::decode(d)); + } + if words.iter().any(|&word| word != 0) + || domain_size <= DenseBitSet::::INLINE_CAPACITY + { + storage.as_mut_slice_or_alloc(domain_size).copy_from_slice(&words); + } + } + + DenseBitSet { domain_size, storage, marker: PhantomData } } } @@ -380,8 +540,8 @@ impl ToString for DenseBitSet { // i tracks how many bits we have printed so far. let mut i = 0; - for word in &self.words { - let mut word = *word; + for word_index in 0..self.num_words() { + let mut word = self.word(word_index); for _ in 0..WORD_BYTES { // for each byte in `word`: let remain = self.domain_size - i; @@ -407,6 +567,307 @@ impl ToString for DenseBitSet { } } +#[repr(C)] +union DenseBitSetStorage { + inline: Word, + empty_unallocated: usize, + on_heap: ManuallyDrop, +} + +impl DenseBitSetStorage { + const EMPTY_UNALLOCATED_TAG_BITS: usize = 0b01usize << (usize::BITS - 2); + const LARGE_TAG_MASK: usize = 0b11usize << (usize::BITS - 2); + + #[inline] + fn is_inline_domain(domain_size: usize) -> bool { + domain_size <= DenseBitSet::::INLINE_CAPACITY + } + + #[inline] + fn new_empty(domain_size: usize) -> Self { + if Self::is_inline_domain(domain_size) { + Self { inline: 0 } + } else { + Self::new_empty_unallocated(domain_size) + } + } + + #[inline] + fn new_filled(domain_size: usize) -> Self { + if Self::is_inline_domain(domain_size) { + Self { inline: domain_mask(domain_size) } + } else { + let mut on_heap = BitSetOnHeap::new_empty(num_words(domain_size)); + let words = on_heap.as_mut_slice(); + words.fill(!0); + clear_excess_bits_in_final_word(domain_size, words); + Self { on_heap: ManuallyDrop::new(on_heap) } + } + } + + #[inline] + fn new_empty_unallocated(domain_size: usize) -> Self { + let num_words = num_words(domain_size); + assert_eq!(num_words & Self::LARGE_TAG_MASK, 0); + Self { empty_unallocated: Self::EMPTY_UNALLOCATED_TAG_BITS | num_words } + } + + #[inline] + fn empty_unallocated_num_words(&self, domain_size: usize) -> Option { + if Self::is_inline_domain(domain_size) { + return None; + } + + let bits = unsafe { self.empty_unallocated }; + if bits & Self::LARGE_TAG_MASK == Self::EMPTY_UNALLOCATED_TAG_BITS { + Some(bits ^ Self::EMPTY_UNALLOCATED_TAG_BITS) + } else { + None + } + } + + #[inline] + fn on_heap(&self, domain_size: usize) -> Option<&BitSetOnHeap> { + if Self::is_inline_domain(domain_size) + || self.empty_unallocated_num_words(domain_size).is_some() + { + None + } else { + Some(unsafe { &*std::ptr::addr_of!(self.on_heap).cast::() }) + } + } + + #[inline] + fn on_heap_mut(&mut self, domain_size: usize) -> Option<&mut BitSetOnHeap> { + if Self::is_inline_domain(domain_size) + || self.empty_unallocated_num_words(domain_size).is_some() + { + None + } else { + Some(unsafe { &mut *std::ptr::addr_of_mut!(self.on_heap).cast::() }) + } + } + + #[inline] + fn as_slice(&self, domain_size: usize) -> Option<&[Word]> { + if Self::is_inline_domain(domain_size) { + let len = num_words(domain_size); + let ptr = std::ptr::addr_of!(self.inline); + Some(unsafe { slice::from_raw_parts(ptr, len) }) + } else { + self.on_heap(domain_size).map(BitSetOnHeap::as_slice) + } + } + + #[inline] + fn as_mut_slice(&mut self, domain_size: usize) -> Option<&mut [Word]> { + if Self::is_inline_domain(domain_size) { + let len = num_words(domain_size); + let ptr = std::ptr::addr_of_mut!(self.inline); + Some(unsafe { slice::from_raw_parts_mut(ptr, len) }) + } else { + self.on_heap_mut(domain_size).map(BitSetOnHeap::as_mut_slice) + } + } + + #[inline] + fn as_mut_slice_or_alloc(&mut self, domain_size: usize) -> &mut [Word] { + if Self::is_inline_domain(domain_size) { + let len = num_words(domain_size); + let ptr = std::ptr::addr_of_mut!(self.inline); + unsafe { slice::from_raw_parts_mut(ptr, len) } + } else if let Some(num_words) = self.empty_unallocated_num_words(domain_size) { + *self = Self { on_heap: ManuallyDrop::new(BitSetOnHeap::new_empty(num_words)) }; + self.on_heap_mut(domain_size).unwrap().as_mut_slice() + } else { + self.on_heap_mut(domain_size).unwrap().as_mut_slice() + } + } + + #[inline] + fn word(&self, domain_size: usize, word_index: usize) -> Word { + self.as_slice(domain_size).map_or(0, |words| words[word_index]) + } + + #[inline] + fn copy_words_to(&self, domain_size: usize, words: &mut [Word]) { + assert_eq!(words.len(), num_words(domain_size)); + if let Some(storage_words) = self.as_slice(domain_size) { + words.copy_from_slice(storage_words); + } else { + words.fill(0); + } + } + + #[inline] + fn clear(&mut self, domain_size: usize) { + if let Some(words) = self.as_mut_slice(domain_size) { + words.fill(0); + } + } + + #[inline] + fn is_empty(&self, domain_size: usize) -> bool { + match self.as_slice(domain_size) { + Some(words) => words.iter().all(|&word| word == 0), + None => true, + } + } + + fn ensure_capacity(&mut self, old_domain_size: usize, min_domain_size: usize) { + if Self::is_inline_domain(min_domain_size) { + return; + } + + if Self::is_inline_domain(old_domain_size) { + let word = unsafe { self.inline }; + if word == 0 { + *self = Self::new_empty_unallocated(min_domain_size); + } else { + let mut on_heap = BitSetOnHeap::new_empty(num_words(min_domain_size)); + on_heap.as_mut_slice()[0] = word; + *self = Self { on_heap: ManuallyDrop::new(on_heap) }; + } + } else if self.empty_unallocated_num_words(old_domain_size).is_some() { + *self = Self::new_empty_unallocated(min_domain_size); + } else { + self.on_heap_mut(old_domain_size).unwrap().ensure_capacity(num_words(min_domain_size)); + } + } + + fn clone(&self, domain_size: usize) -> Self { + if Self::is_inline_domain(domain_size) { + Self { inline: unsafe { self.inline } } + } else if self.empty_unallocated_num_words(domain_size).is_some() { + Self { empty_unallocated: unsafe { self.empty_unallocated } } + } else { + Self { on_heap: ManuallyDrop::new(self.on_heap(domain_size).unwrap().clone()) } + } + } + + fn drop(&mut self, domain_size: usize) { + if !Self::is_inline_domain(domain_size) + && self.empty_unallocated_num_words(domain_size).is_none() + { + unsafe { + ManuallyDrop::drop(&mut self.on_heap); + } + } + } +} + +#[repr(transparent)] +struct BitSetOnHeap(usize); + +impl BitSetOnHeap { + fn new_empty(len: usize) -> Self { + assert!(len > 0); + assert!(Word::try_from(len).is_ok()); + + let num_words = len + 1; + let layout = Layout::array::(num_words).expect("bit set too large"); + let ptr = unsafe { alloc_zeroed(layout).cast::() }; + let Some(ptr) = NonNull::new(ptr) else { + handle_alloc_error(layout); + }; + + unsafe { ptr.write(len as Word) }; + Self::from_ptr(ptr) + } + + #[inline] + fn from_ptr(ptr: NonNull) -> Self { + Self(Self::pack_ptr(ptr)) + } + + #[inline] + fn pack_ptr(ptr: NonNull) -> usize { + let addr = ptr.as_ptr() as usize; + debug_assert_eq!(addr & 0b11, 0); + let packed = addr >> 2; + assert_eq!(packed & DenseBitSetStorage::LARGE_TAG_MASK, 0); + packed + } + + #[inline] + fn ptr(&self) -> *mut Word { + (self.0 << 2) as *mut Word + } + + #[inline] + fn len(&self) -> usize { + unsafe { self.ptr().read() as usize } + } + + #[inline] + fn as_slice(&self) -> &[Word] { + let ptr = self.ptr(); + let len = self.len(); + unsafe { slice::from_raw_parts(ptr.add(1), len) } + } + + #[inline] + fn as_mut_slice(&mut self) -> &mut [Word] { + let ptr = self.ptr(); + let len = self.len(); + unsafe { slice::from_raw_parts_mut(ptr.add(1), len) } + } + + fn ensure_capacity(&mut self, min_len: usize) { + let old_len = self.len(); + if min_len <= old_len { + return; + } + assert!(Word::try_from(min_len).is_ok()); + + let old_num_words = old_len + 1; + let new_num_words = min_len + 1; + let old_layout = Layout::array::(old_num_words).expect("bit set too large"); + let new_layout = Layout::array::(new_num_words).expect("bit set too large"); + + let ptr = unsafe { + realloc(self.ptr().cast::(), old_layout, new_layout.size()).cast::() + }; + let Some(ptr) = NonNull::new(ptr) else { + handle_alloc_error(new_layout); + }; + + unsafe { + ptr.write(min_len as Word); + ptr.as_ptr().add(old_num_words).write_bytes(0, new_num_words - old_num_words); + } + + self.0 = Self::pack_ptr(ptr); + } +} + +impl Clone for BitSetOnHeap { + fn clone(&self) -> Self { + let num_words = self.len() + 1; + let layout = Layout::array::(num_words).expect("bit set too large"); + let new_ptr = unsafe { alloc(layout).cast::() }; + let Some(new_ptr) = NonNull::new(new_ptr) else { + handle_alloc_error(layout); + }; + + unsafe { + self.ptr().copy_to_nonoverlapping(new_ptr.as_ptr(), num_words); + } + + Self::from_ptr(new_ptr) + } +} + +impl Drop for BitSetOnHeap { + fn drop(&mut self) { + let num_words = self.len() + 1; + let layout = Layout::array::(num_words).expect("bit set too large"); + unsafe { + dealloc(self.ptr().cast::(), layout); + } + } +} + pub struct BitIter<'a, T: Idx> { /// A copy of the current word, but with any already-visited bits cleared. /// (This lets us use `trailing_zeros()` to find the next set bit.) When it @@ -437,6 +898,11 @@ impl<'a, T: Idx> BitIter<'a, T> { marker: PhantomData, } } + + #[inline] + fn from_single_word(word: Word) -> BitIter<'a, T> { + BitIter { word, offset: 0, iter: [].iter(), marker: PhantomData } + } } impl<'a, T: Idx> Iterator for BitIter<'a, T> { @@ -1301,14 +1767,7 @@ impl Default for GrowableBitSet { impl GrowableBitSet { /// Ensure that the set can hold at least `min_domain_size` elements. pub fn ensure(&mut self, min_domain_size: usize) { - if self.bit_set.domain_size < min_domain_size { - self.bit_set.domain_size = min_domain_size; - } - - let min_num_words = num_words(min_domain_size); - if self.bit_set.words.len() < min_num_words { - self.bit_set.words.resize(min_num_words, 0) - } + self.bit_set.ensure_domain_size(min_domain_size); } pub fn new_empty() -> GrowableBitSet { @@ -1356,8 +1815,7 @@ impl GrowableBitSet { #[inline] pub fn contains(&self, elem: T) -> bool { - let (word_index, mask) = word_index_and_mask(elem); - self.bit_set.words.get(word_index).is_some_and(|word| (word & mask) != 0) + elem.index() < self.bit_set.domain_size && self.bit_set.contains(elem) } #[inline] @@ -1419,13 +1877,14 @@ impl BitMatrix { pub fn from_row_n(row: &DenseBitSet, num_rows: usize) -> BitMatrix { let num_columns = row.domain_size(); let words_per_row = num_words(num_columns); - assert_eq!(words_per_row, row.words.len()); - BitMatrix { - num_rows, - num_columns, - words: iter::repeat_n(&row.words, num_rows).flatten().cloned().collect(), - marker: PhantomData, + assert_eq!(words_per_row, row.num_words()); + let mut words = vec![0; num_rows * words_per_row]; + if words_per_row != 0 { + for row_words in words.chunks_mut(words_per_row) { + row.copy_words_to(row_words); + } } + BitMatrix { num_rows, num_columns, words, marker: PhantomData } } pub fn rows(&self) -> impl Iterator { @@ -1517,8 +1976,15 @@ impl BitMatrix { pub fn union_row_with(&mut self, with: &DenseBitSet, write: R) -> bool { assert!(write.index() < self.num_rows); assert_eq!(with.domain_size(), self.num_columns); + if with.is_empty() { + return false; + } let (write_start, write_end) = self.range(write); - update_words(&mut self.words[write_start..write_end], &with.words, |a, b| a | b) + update_words( + &mut self.words[write_start..write_end], + with.storage_words().unwrap(), + |a, b| a | b, + ) } /// Sets every cell in `row` to true. @@ -1753,6 +2219,12 @@ fn max_bit(word: Word) -> usize { WORD_BITS - 1 - word.leading_zeros() as usize } +#[inline] +fn domain_mask(domain_size: usize) -> Word { + debug_assert!(domain_size <= WORD_BITS); + if domain_size == 0 { 0 } else { Word::MAX >> (WORD_BITS - domain_size) } +} + #[inline] fn count_ones(words: &[Word]) -> usize { words.iter().map(|word| word.count_ones() as usize).sum() diff --git a/compiler/rustc_index/src/bit_set/tests.rs b/compiler/rustc_index/src/bit_set/tests.rs index 243ac20ee441c..ffbd52451b0ef 100644 --- a/compiler/rustc_index/src/bit_set/tests.rs +++ b/compiler/rustc_index/src/bit_set/tests.rs @@ -58,6 +58,66 @@ fn bitset_clone_from() { assert_eq!(b.iter().collect::>(), []); } +#[test] +fn dense_storage_boundaries() { + for domain in [0, 1, WORD_BITS - 1, WORD_BITS, WORD_BITS + 1, WORD_BITS * 2 + 3] { + let empty = DenseBitSet::::new_empty(domain); + assert_eq!(empty.domain_size(), domain); + assert!(empty.is_empty()); + assert_eq!(empty.count(), 0); + assert_eq!(empty.iter().collect::>(), []); + + let mut filled = DenseBitSet::::new_filled(domain); + assert_eq!(filled.domain_size(), domain); + assert_eq!(filled.count(), domain); + assert_eq!(filled.iter().collect::>(), (0..domain).collect::>()); + + filled.clear(); + assert!(filled.is_empty()); + if domain != 0 { + assert!(filled.insert(domain - 1)); + assert!(filled.contains(domain - 1)); + } + } +} + +#[test] +fn dense_clone_from_crosses_storage_boundaries() { + let mut inline = DenseBitSet::::new_empty(WORD_BITS); + inline.insert(WORD_BITS - 1); + + let mut large = DenseBitSet::::new_empty(WORD_BITS + 1); + large.insert(WORD_BITS); + + inline.clone_from(&large); + assert_eq!(inline.domain_size(), WORD_BITS + 1); + assert_eq!(inline.iter().collect::>(), [WORD_BITS]); + + large.clone_from(&DenseBitSet::new_empty(WORD_BITS * 2 + 1)); + assert_eq!(large.domain_size(), WORD_BITS * 2 + 1); + assert!(large.is_empty()); + + large.clone_from(&DenseBitSet::new_filled(WORD_BITS - 1)); + assert_eq!(large.domain_size(), WORD_BITS - 1); + assert_eq!(large.count(), WORD_BITS - 1); +} + +#[test] +fn dense_relations_handle_lazy_empty_large_sets() { + let domain = WORD_BITS + 5; + let empty = DenseBitSet::::new_empty(domain); + let mut set = DenseBitSet::::new_empty(domain); + let mut other = DenseBitSet::::new_empty(domain); + other.insert(WORD_BITS + 4); + + assert!(!set.subtract(&empty)); + assert!(set.union(&other)); + assert!(set.contains(WORD_BITS + 4)); + assert!(!set.union(&other)); + assert!(set.intersect(&empty)); + assert!(set.is_empty()); +} + #[test] fn union_two_sets() { let mut set1: DenseBitSet = DenseBitSet::new_empty(65); @@ -548,6 +608,30 @@ fn grow() { } } +#[test] +fn growable_crosses_inline_to_heap_boundary() { + let mut set: GrowableBitSet = GrowableBitSet::with_capacity(1); + assert!(!set.contains(WORD_BITS + 3)); + + assert!(set.insert(WORD_BITS + 3)); + assert!(set.contains(WORD_BITS + 3)); + assert_eq!(set.count(), 1); + + assert!(set.remove(WORD_BITS + 3)); + assert!(set.is_empty()); + + set.insert_range((WORD_BITS - 1)..(WORD_BITS + 4)); + assert_eq!( + set.iter().collect::>(), + ((WORD_BITS - 1)..(WORD_BITS + 4)).collect::>() + ); + + set.ensure(WORD_BITS * 4 + 1); + assert!(set.contains(WORD_BITS + 3)); + assert!(!set.contains(WORD_BITS * 4)); + assert!(set.insert(WORD_BITS * 4)); +} + #[test] fn matrix_intersection() { let mut matrix: BitMatrix = BitMatrix::new(200, 200); @@ -637,6 +721,31 @@ fn matrix_iter() { } } +#[test] +fn matrix_from_row_and_union_with_lazy_dense_rows() { + let domain = WORD_BITS + 3; + let empty = DenseBitSet::::new_empty(domain); + let matrix = BitMatrix::::from_row_n(&empty, 3); + for row in 0..3 { + assert_eq!(matrix.iter(row).collect::>(), []); + } + + let mut row = DenseBitSet::::new_empty(domain); + row.insert(0); + row.insert(WORD_BITS + 2); + let matrix = BitMatrix::::from_row_n(&row, 2); + for matrix_row in 0..2 { + assert!(matrix.contains(matrix_row, 0)); + assert!(matrix.contains(matrix_row, WORD_BITS + 2)); + } + + let mut matrix = BitMatrix::::new(1, domain); + assert!(!matrix.union_row_with(&empty, 0)); + assert!(matrix.union_row_with(&row, 0)); + assert!(!matrix.union_row_with(&row, 0)); + assert_eq!(matrix.iter(0).collect::>(), [0, WORD_BITS + 2]); +} + #[test] fn sparse_matrix_iter() { let mut matrix: SparseBitMatrix = SparseBitMatrix::new(100); From 71af2e03424b1c279943cdd64d4f5810db4c3341 Mon Sep 17 00:00:00 2001 From: Jamesbarford Date: Thu, 11 Jun 2026 19:08:45 +0000 Subject: [PATCH 6/6] micro optimisation for clearing least significant bit --- compiler/rustc_index/src/bit_set.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler/rustc_index/src/bit_set.rs b/compiler/rustc_index/src/bit_set.rs index 19dc3de1685f0..356f61be3e608 100644 --- a/compiler/rustc_index/src/bit_set.rs +++ b/compiler/rustc_index/src/bit_set.rs @@ -913,7 +913,7 @@ impl<'a, T: Idx> Iterator for BitIter<'a, T> { // Get the position of the next set bit in the current word, // then clear the bit. let bit_pos = self.word.trailing_zeros() as usize; - self.word ^= 1 << bit_pos; + self.word &= self.word - 1; return Some(T::new(bit_pos + self.offset)); }