Pipeline-only tk-encode + tk-train on the pipeline (thin deps) — replaces #2200

bindings/python/src/lib.rs

@@ -18,13 +18,10 @@ static TOKIO_RUNTIME: Lazy<Arc<Runtime>> = Lazy::new(|| {
         .expect("Failed to create global Tokio runtime");
     Arc::new(rt)
 });
-mod decoders;
-mod encoding;
 mod error;
 mod models;
 mod normalizers;
 mod pre_tokenizers;
-mod processors;
 mod token;
 mod tokenizer;
 mod trainers;
@@ -51,8 +48,6 @@ extern "C" fn child_after_fork() {
 pub mod tokenizers {
     use super::*;
 
-    #[pymodule_export]
-    pub use super::encoding::PyEncoding;
     #[pymodule_export]
     pub use super::token::PyToken;
     #[pymodule_export]
@@ -62,21 +57,15 @@ pub mod tokenizers {
     #[pymodule_export]
     pub use super::utils::PyNormalizedString;
     #[pymodule_export]
-    pub use super::utils::PyPreTokenizedString;
-    #[pymodule_export]
     pub use super::utils::PyRegex;
 
-    #[pymodule_export]
-    pub use super::decoders::decoders;
     #[pymodule_export]
     pub use super::models::models;
     #[pymodule_export]
     pub use super::normalizers::normalizers;
     #[pymodule_export]
     pub use super::pre_tokenizers::pre_tokenizers;
     #[pymodule_export]
-    pub use super::processors::processors;
-    #[pymodule_export]
     pub use super::trainers::trainers;
 
     #[allow(non_upper_case_globals)]

bindings/python/src/models.rs

@@ -14,7 +14,6 @@ use tk::models::unigram::Unigram;
 use tk::models::wordlevel::WordLevel;
 use tk::models::wordpiece::{WordPiece, WordPieceBuilder};
 use tk::models::ModelWrapper;
-use tk::tokenizer::PreTokenizedString;
 use tk::{Model, Token, Trainable};
 use tokenizers as tk;
 
@@ -50,23 +49,6 @@ impl Model for PyModel {
         self.model.read().unwrap().tokenize(tokens)
     }
 
-    /// See [`Model::tokenize_in_pretokenized`] for the lock-once rationale.
-    fn tokenize_in_pretokenized(
-        &self,
-        pretokenized: &mut PreTokenizedString,
-        truncation: Option<(usize, tk::TruncationDirection)>,
-    ) -> tk::Result<()> {
-        let guard = self.model.read().unwrap();
-        match truncation {
-            Some((max_tokens, direction)) => pretokenized.tokenize_with_limit(
-                |normalized| guard.tokenize(normalized.get()),
-                max_tokens,
-                direction,
-            ),
-            None => pretokenized.tokenize(|normalized| guard.tokenize(normalized.get())),
-        }
-    }
-
     fn token_to_id(&self, token: &str) -> Option<u32> {
         self.model.read().unwrap().token_to_id(token)
     }

bindings/python/src/pre_tokenizers.rs

@@ -147,23 +147,6 @@ impl PyPreTokenizer {
         }
     }
 
-    /// Pre-tokenize a :class:`~tokenizers.PyPreTokenizedString` in-place
-    ///
-    /// This method allows to modify a :class:`~tokenizers.PreTokenizedString` to
-    /// keep track of the pre-tokenization, and leverage the capabilities of the
-    /// :class:`~tokenizers.PreTokenizedString`. If you just want to see the result of
-    /// the pre-tokenization of a raw string, you can use
-    /// :meth:`~tokenizers.pre_tokenizers.PreTokenizer.pre_tokenize_str`
-    ///
-    /// Args:
-    ///     pretok (:class:`~tokenizers.PreTokenizedString):
-    ///         The pre-tokenized string on which to apply this
-    ///         :class:`~tokenizers.pre_tokenizers.PreTokenizer`
-    #[pyo3(text_signature = "(self, pretok)")]
-    fn pre_tokenize(&self, pretok: &mut PyPreTokenizedString) -> PyResult<()> {
-        ToPyResult(self.pretok.pre_tokenize(&mut pretok.pretok)).into()
-    }
-
     /// Pre tokenize the given string
     ///
     /// This method provides a way to visualize the effect of a
@@ -900,13 +883,10 @@ impl CustomPreTokenizer {
 }
 
 impl tk::tokenizer::PreTokenizer for CustomPreTokenizer {
-    fn pre_tokenize(&self, sentence: &mut PreTokenizedString) -> tk::Result<()> {
-        Python::attach(|py| {
-            let pretok = PyPreTokenizedStringRefMut::new(sentence);
-            let py_pretok = self.inner.bind(py);
-            py_pretok.call_method("pre_tokenize", (pretok.get().clone(),), None)?;
-            Ok(())
-        })
+    fn pre_tokenize(&self, _sentence: &mut PreTokenizedString) -> tk::Result<()> {
+        // ponytail: custom Python pre-tokenizers relied on PyPreTokenizedStringRefMut, which was
+        // part of the removed offset/encoding machinery. The pipeline-only build can't drive them.
+        Err("custom Python PreTokenizers are not supported in this pipeline-only build".into())
     }
 }