Skip to content

[EXPERIMENT] Use AtomicSharedVector for token streams #159048

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
nnethercote wants to merge 4 commits into
base: main
Choose a base branch
from
Draft

[EXPERIMENT] Use AtomicSharedVector for token streams #159048

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
48e0338
Remove unused `TokenStream::chunks`
nnethercote Jul 9, 2026
4b5a9f3
Adjust signature of `TokenStream::try_glue_to_last`
nnethercote Jul 9, 2026
d052ec8
Improve `StableHash` impl for `TokenStream`
nnethercote Jul 9, 2026
500af70
Use `AtomicSharedVector` for token streams
nnethercote Jul 9, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 20 additions & 1 deletion Cargo.lock
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,12 @@ version = "0.2.21"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923"

[[package]]
name = "allocator-api2"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c880a97d28a3681c0267bd29cff89621202715b065127cd445fa0f0fe0aa2880"

[[package]]
name = "android_system_properties"
version = "0.1.5"
Expand Down Expand Up @@ -1713,7 +1719,7 @@ version = "0.15.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1"
dependencies = [
"allocator-api2",
"allocator-api2 0.2.21",
"equivalent",
"foldhash 0.1.5",
]
Expand Down Expand Up @@ -3593,6 +3599,7 @@ dependencies = [
"rustc_macros",
"rustc_serialize",
"rustc_span",
"shared_vector",
"smallvec",
"thin-vec",
"tracing",
Expand Down Expand Up @@ -3872,6 +3879,7 @@ dependencies = [
"rustc_macros",
"rustc_serialize",
"rustc_thread_pool",
"shared_vector",
"smallvec",
"stacker",
"tempfile",
Expand Down Expand Up @@ -3995,6 +4003,7 @@ dependencies = [
"rustc_session",
"rustc_span",
"scoped-tls",
"shared_vector",
"smallvec",
"thin-vec",
"tracing",
Expand Down Expand Up @@ -4496,6 +4505,7 @@ dependencies = [
"rustc_macros",
"rustc_session",
"rustc_span",
"shared_vector",
"thin-vec",
"tracing",
"unicode-normalization",
Expand Down Expand Up @@ -5280,6 +5290,15 @@ dependencies = [
"lazy_static",
]

[[package]]
name = "shared_vector"
version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "db178a7b1e552c11440665d314451b8dc12bfc4da84212728292bbbd08f28f6f"
dependencies = [
"allocator-api2 0.4.0",
]

[[package]]
name = "shlex"
version = "1.3.0"
Expand Down
1 change: 1 addition & 0 deletions compiler/rustc_ast/Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ rustc_index = { path = "../rustc_index" }
rustc_macros = { path = "../rustc_macros" }
rustc_serialize = { path = "../rustc_serialize" }
rustc_span = { path = "../rustc_span" }
shared_vector = "0.5"
smallvec = { version = "1.8.1", features = ["union", "may_dangle"] }
thin-vec = "0.2.18"
tracing = "0.1"
Expand Down
7 changes: 4 additions & 3 deletions compiler/rustc_ast/src/attr/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ use std::sync::atomic::{AtomicU32, Ordering};

use rustc_index::bit_set::GrowableBitSet;
use rustc_span::{Ident, Span, Symbol, sym};
use shared_vector::{Vector, vector};
use smallvec::{SmallVec, smallvec};
use thin_vec::{ThinVec, thin_vec};

Expand Down Expand Up @@ -299,15 +300,15 @@ impl Attribute {
}
}

pub fn token_trees(&self) -> Vec<TokenTree> {
pub fn token_trees(&self) -> Vector<TokenTree> {
match self.kind {
AttrKind::Normal(ref normal) => normal
.tokens
.as_ref()
.unwrap_or_else(|| panic!("attribute is missing tokens: {self:?}"))
.to_attr_token_stream()
.to_token_trees(),
AttrKind::DocComment(comment_kind, data) => vec![TokenTree::token_alone(
AttrKind::DocComment(comment_kind, data) => vector![TokenTree::token_alone(
token::DocComment(comment_kind, self.style, data),
self.span,
)],
Expand Down Expand Up @@ -789,7 +790,7 @@ pub fn mk_attr_nested_word(
inner: Symbol,
span: Span,
) -> Attribute {
let inner_tokens = TokenStream::new(vec![TokenTree::Token(
let inner_tokens = TokenStream::new(vector![TokenTree::Token(
Token::from_ast_ident(Ident::new(inner, span)),
Spacing::Alone,
)]);
Expand Down
Loading
Loading