From 2903bafd751d3d3030d5d9d3a418ce9f9056972b Mon Sep 17 00:00:00 2001 From: Assaf Vayner Date: Mon, 13 Jul 2026 18:08:42 -0700 Subject: [PATCH] Upgrade hf-hub to 1.0 --- tokenizers/Cargo.toml | 4 ++-- tokenizers/src/utils/from_pretrained.rs | 17 ++++++++++------- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/tokenizers/Cargo.toml b/tokenizers/Cargo.toml index a4660fd33c..01f058a100 100644 --- a/tokenizers/Cargo.toml +++ b/tokenizers/Cargo.toml @@ -87,8 +87,8 @@ itertools = "0.14" log = "0.4" derive_builder = "0.20" spm_precompiled = "0.1.3" -hf-hub = { version = "0.4.1", features = [ - "ureq", +hf-hub = { version = "1.0", features = [ + "blocking", ], default-features = false, optional = true } daachorse = "3.0.0" paste = "1.0.14" diff --git a/tokenizers/src/utils/from_pretrained.rs b/tokenizers/src/utils/from_pretrained.rs index 7d99f05a5f..2466030319 100644 --- a/tokenizers/src/utils/from_pretrained.rs +++ b/tokenizers/src/utils/from_pretrained.rs @@ -1,5 +1,4 @@ use crate::Result; -use hf_hub::{api::sync::ApiBuilder, Repo, RepoType}; use std::collections::HashMap; use std::path::PathBuf; @@ -57,12 +56,16 @@ pub fn from_pretrained>( .into()); } - let mut builder = ApiBuilder::from_env(); + let mut builder = hf_hub::HFClient::builder(); if let Some(token) = params.token { - builder = builder.with_token(Some(token)); + builder = builder.token(token); } - let api = builder.build()?; - let repo = Repo::with_revision(identifier, RepoType::Model, params.revision); - let api = api.repo(repo); - Ok(api.get("tokenizer.json")?) + let client = hf_hub::HFClientSync::from_inner(builder.build()?)?; + let (owner, name) = hf_hub::split_id(&identifier); + let repo = client.model(owner, name); + Ok(repo + .download_file() + .filename("tokenizer.json") + .revision(params.revision) + .send()?) }