Reuse resolved Hub commits throughout loading

src/transformers/feature_extraction_utils.py

@@ -40,7 +40,7 @@
     requires_backends,
     safe_load_json_file,
 )
-from .utils.hub import cached_file, hf_api
+from .utils.hub import cached_file, extract_commit_hash, hf_api
 
 
 if TYPE_CHECKING:
@@ -377,6 +377,7 @@ def from_pretrained(
             kwargs["token"] = token
 
         feature_extractor_dict, kwargs = cls.get_feature_extractor_dict(pretrained_model_name_or_path, **kwargs)
+        kwargs.pop("_commit_hash", None)
 
         return cls.from_dict(feature_extractor_dict, **kwargs)
 
@@ -450,6 +451,7 @@ def get_feature_extractor_dict(
         token = kwargs.pop("token", None)
         local_files_only = kwargs.pop("local_files_only", False)
         revision = kwargs.pop("revision", None)
+        commit_hash = kwargs.pop("_commit_hash", None)
 
         from_pipeline = kwargs.pop("_from_pipeline", None)
         from_auto_class = kwargs.pop("_from_auto", False)
@@ -486,7 +488,9 @@ def get_feature_extractor_dict(
                     revision=revision,
                     subfolder=subfolder,
                     _raise_exceptions_for_missing_entries=False,
+                    _commit_hash=commit_hash,
                 )
+                commit_hash = extract_commit_hash(resolved_processor_file, commit_hash)
                 resolved_feature_extractor_file = cached_file(
                     pretrained_model_name_or_path,
                     filename=feature_extractor_file,
@@ -499,7 +503,9 @@ def get_feature_extractor_dict(
                     revision=revision,
                     subfolder=subfolder,
                     _raise_exceptions_for_missing_entries=False,
+                    _commit_hash=commit_hash,
                 )
+                commit_hash = extract_commit_hash(resolved_feature_extractor_file, commit_hash)
             except OSError:
                 # Raise any environment error raise by `cached_file`. It will have a helpful error message adapted to
                 # the original exception.
@@ -540,6 +546,8 @@ def get_feature_extractor_dict(
                 f"loading configuration file {feature_extractor_file} from cache at {resolved_feature_extractor_file}"
             )
 
+        if commit_hash is not None:
+            kwargs["_commit_hash"] = commit_hash
         return feature_extractor_dict, kwargs
 
     @classmethod

src/transformers/image_processing_base.py

@@ -31,7 +31,7 @@
     logging,
     safe_load_json_file,
 )
-from .utils.hub import cached_file, hf_api
+from .utils.hub import cached_file, extract_commit_hash, hf_api
 
 
 ImageProcessorType = TypeVar("ImageProcessorType", bound="ImageProcessingMixin")
@@ -177,6 +177,7 @@ def from_pretrained(
             kwargs["token"] = token
 
         image_processor_dict, kwargs = cls.get_image_processor_dict(pretrained_model_name_or_path, **kwargs)
+        kwargs.pop("_commit_hash", None)
 
         return cls.from_dict(image_processor_dict, **kwargs)
 
@@ -256,6 +257,7 @@ def get_image_processor_dict(
         revision = kwargs.pop("revision", None)
         subfolder = kwargs.pop("subfolder", "")
         image_processor_filename = kwargs.pop("image_processor_filename", IMAGE_PROCESSOR_NAME)
+        commit_hash = kwargs.pop("_commit_hash", None)
 
         from_pipeline = kwargs.pop("_from_pipeline", None)
         from_auto_class = kwargs.pop("_from_auto", False)
@@ -291,7 +293,9 @@ def get_image_processor_dict(
                     revision=revision,
                     subfolder=subfolder,
                     _raise_exceptions_for_missing_entries=False,
+                    _commit_hash=commit_hash,
                 )
+                commit_hash = extract_commit_hash(resolved_processor_file, commit_hash)
                 resolved_image_processor_file = cached_file(
                     pretrained_model_name_or_path,
                     filename=image_processor_file,
@@ -304,7 +308,9 @@ def get_image_processor_dict(
                     revision=revision,
                     subfolder=subfolder,
                     _raise_exceptions_for_missing_entries=False,
+                    _commit_hash=commit_hash,
                 )
+                commit_hash = extract_commit_hash(resolved_image_processor_file, commit_hash)
             except OSError:
                 # Raise any environment error raise by `cached_file`. It will have a helpful error message adapted to
                 # the original exception.
@@ -345,6 +351,8 @@ def get_image_processor_dict(
                 f"loading configuration file {image_processor_file} from cache at {resolved_image_processor_file}"
             )
 
+        if commit_hash is not None:
+            kwargs["_commit_hash"] = commit_hash
         return image_processor_dict, kwargs
 
     @classmethod

src/transformers/models/auto/feature_extraction_auto.py

@@ -21,7 +21,15 @@
 from ...configuration_utils import PreTrainedConfig
 from ...dynamic_module_utils import get_class_from_dynamic_module, resolve_trust_remote_code
 from ...feature_extraction_utils import FeatureExtractionMixin
-from ...utils import CONFIG_NAME, FEATURE_EXTRACTOR_NAME, PROCESSOR_NAME, cached_file, logging, safe_load_json_file
+from ...utils import (
+    CONFIG_NAME,
+    FEATURE_EXTRACTOR_NAME,
+    PROCESSOR_NAME,
+    cached_file,
+    extract_commit_hash,
+    logging,
+    safe_load_json_file,
+)
 from .auto_factory import _LazyAutoMapping
 from .auto_mappings import FEATURE_EXTRACTOR_MAPPING_NAMES
 from .configuration_auto import (
@@ -167,6 +175,7 @@ def get_feature_extractor_config(
     feature_extractor.save_pretrained("feature-extractor-test")
     feature_extractor_config = get_feature_extractor_config("feature-extractor-test")
     ```"""
+    commit_hash = kwargs.get("_commit_hash")
     # Load with a priority given to the nested processor config, if available in repo
     resolved_processor_file = cached_file(
         pretrained_model_name_or_path,
@@ -179,7 +188,9 @@ def get_feature_extractor_config(
         local_files_only=local_files_only,
         _raise_exceptions_for_gated_repo=False,
         _raise_exceptions_for_missing_entries=False,
+        _commit_hash=commit_hash,
     )
+    commit_hash = extract_commit_hash(resolved_processor_file, commit_hash)
     resolved_feature_extractor_file = cached_file(
         pretrained_model_name_or_path,
         filename=FEATURE_EXTRACTOR_NAME,
@@ -191,6 +202,7 @@ def get_feature_extractor_config(
         local_files_only=local_files_only,
         _raise_exceptions_for_gated_repo=False,
         _raise_exceptions_for_missing_entries=False,
+        _commit_hash=commit_hash,
     )
 
     # An empty list if none of the possible files is found in the repo
@@ -299,8 +311,14 @@ def from_pretrained(cls, pretrained_model_name_or_path, **kwargs):
         config = kwargs.pop("config", None)
         trust_remote_code = kwargs.pop("trust_remote_code", None)
         kwargs["_from_auto"] = True
+        if kwargs.get("_commit_hash") is None and (commit_hash := getattr(config, "_commit_hash", None)) is not None:
+            kwargs["_commit_hash"] = commit_hash
 
-        config_dict, _ = FeatureExtractionMixin.get_feature_extractor_dict(pretrained_model_name_or_path, **kwargs)
+        config_dict, unused_kwargs = FeatureExtractionMixin.get_feature_extractor_dict(
+            pretrained_model_name_or_path, **kwargs
+        )
+        if (commit_hash := unused_kwargs.pop("_commit_hash", None)) is not None:
+            kwargs["_commit_hash"] = commit_hash
         feature_extractor_class = config_dict.get("feature_extractor_type", None)
         feature_extractor_auto_map = None
         if "AutoFeatureExtractor" in config_dict.get("auto_map", {}):
@@ -312,6 +330,8 @@ def from_pretrained(cls, pretrained_model_name_or_path, **kwargs):
                 config = AutoConfig.from_pretrained(
                     pretrained_model_name_or_path, trust_remote_code=trust_remote_code, **kwargs
                 )
+            if (commit_hash := getattr(config, "_commit_hash", None)) is not None:
+                kwargs["_commit_hash"] = commit_hash
             # It could be in `config.feature_extractor_type``
             feature_extractor_class = getattr(config, "feature_extractor_type", None)
             if hasattr(config, "auto_map") and "AutoFeatureExtractor" in config.auto_map:

src/transformers/models/auto/image_processing_auto.py

@@ -27,6 +27,7 @@
     IMAGE_PROCESSOR_NAME,
     PROCESSOR_NAME,
     cached_file,
+    extract_commit_hash,
     is_timm_config_dict,
     is_timm_local_checkpoint,
     is_torchvision_available,
@@ -269,6 +270,7 @@ def get_image_processor_config(
     image_processor.save_pretrained("image-processor-test")
     image_processor_config = get_image_processor_config("image-processor-test")
     ```"""
+    commit_hash = kwargs.get("_commit_hash")
     # Load with a priority given to the nested processor config, if available in repo
     resolved_processor_file = cached_file(
         pretrained_model_name_or_path,
@@ -281,7 +283,9 @@ def get_image_processor_config(
         local_files_only=local_files_only,
         _raise_exceptions_for_gated_repo=False,
         _raise_exceptions_for_missing_entries=False,
+        _commit_hash=commit_hash,
     )
+    commit_hash = extract_commit_hash(resolved_processor_file, commit_hash)
     resolved_image_processor_file = cached_file(
         pretrained_model_name_or_path,
         filename=IMAGE_PROCESSOR_NAME,
@@ -293,6 +297,7 @@ def get_image_processor_config(
         local_files_only=local_files_only,
         _raise_exceptions_for_gated_repo=False,
         _raise_exceptions_for_missing_entries=False,
+        _commit_hash=commit_hash,
     )
 
     # An empty list if none of the possible files is found in the repo
@@ -578,6 +583,8 @@ def from_pretrained(cls, pretrained_model_name_or_path, *inputs, **kwargs):
         backend_kwarg = kwargs.pop("backend", None)
         trust_remote_code = kwargs.pop("trust_remote_code", None)
         kwargs["_from_auto"] = True
+        if kwargs.get("_commit_hash") is None and (commit_hash := getattr(config, "_commit_hash", None)) is not None:
+            kwargs["_commit_hash"] = commit_hash
 
         # Resolve the image processor config filename
         if "image_processor_filename" in kwargs:
@@ -590,15 +597,19 @@ def from_pretrained(cls, pretrained_model_name_or_path, *inputs, **kwargs):
         # Load the image processor config
 
         try:
-            config_dict, _ = ImageProcessingMixin.get_image_processor_dict(
+            config_dict, unused_kwargs = ImageProcessingMixin.get_image_processor_dict(
                 pretrained_model_name_or_path, image_processor_filename=image_processor_filename, **kwargs
             )
+            if (commit_hash := unused_kwargs.pop("_commit_hash", None)) is not None:
+                kwargs["_commit_hash"] = commit_hash
         except Exception as initial_exception:
             # Fallback for Hub TimmWrapper checkpoints (image processing in config.json, not preprocessor_config.json)
             try:
-                config_dict, _ = ImageProcessingMixin.get_image_processor_dict(
+                config_dict, unused_kwargs = ImageProcessingMixin.get_image_processor_dict(
                     pretrained_model_name_or_path, image_processor_filename=CONFIG_NAME, **kwargs
                 )
+                if (commit_hash := unused_kwargs.pop("_commit_hash", None)) is not None:
+                    kwargs["_commit_hash"] = commit_hash
             except Exception:
                 raise initial_exception
 
@@ -626,6 +637,8 @@ def from_pretrained(cls, pretrained_model_name_or_path, *inputs, **kwargs):
                     config = AutoConfig.from_pretrained(
                         pretrained_model_name_or_path, trust_remote_code=trust_remote_code, **kwargs
                     )
+                if (commit_hash := getattr(config, "_commit_hash", None)) is not None:
+                    kwargs["_commit_hash"] = commit_hash
 
                 image_processor_type = getattr(config, "image_processor_type", None)
                 if hasattr(config, "auto_map") and "AutoImageProcessor" in config.auto_map:

src/transformers/models/auto/processing_auto.py

@@ -25,7 +25,14 @@
 from ...image_processing_utils import ImageProcessingMixin
 from ...processing_utils import ProcessorMixin
 from ...tokenization_python import TOKENIZER_CONFIG_FILE
-from ...utils import FEATURE_EXTRACTOR_NAME, PROCESSOR_NAME, VIDEO_PROCESSOR_NAME, cached_file, logging
+from ...utils import (
+    FEATURE_EXTRACTOR_NAME,
+    PROCESSOR_NAME,
+    VIDEO_PROCESSOR_NAME,
+    cached_file,
+    extract_commit_hash,
+    logging,
+)
 from ...video_processing_utils import BaseVideoProcessor
 from .auto_factory import _LazyAutoMapping
 from .auto_mappings import PROCESSOR_MAPPING_NAMES
@@ -194,6 +201,8 @@ def from_pretrained(cls, pretrained_model_name_or_path, **kwargs):
         config = kwargs.pop("config", None)
         trust_remote_code = kwargs.pop("trust_remote_code", None)
         kwargs["_from_auto"] = True
+        if kwargs.get("_commit_hash") is None and (commit_hash := getattr(config, "_commit_hash", None)) is not None:
+            kwargs["_commit_hash"] = commit_hash
 
         processor_class = None
         processor_auto_map = None
@@ -210,6 +219,7 @@ def from_pretrained(cls, pretrained_model_name_or_path, **kwargs):
             "subfolder",
             "repo_type",
             "user_agent",
+            "_commit_hash",
         )
         cached_file_kwargs = {key: kwargs[key] for key in _hub_valid_kwargs if key in kwargs}
         # We don't want to raise
@@ -223,6 +233,9 @@ def from_pretrained(cls, pretrained_model_name_or_path, **kwargs):
 
         # Let's start by checking whether the processor class is saved in a processor config
         processor_config_file = cached_file(pretrained_model_name_or_path, PROCESSOR_NAME, **cached_file_kwargs)
+        commit_hash = extract_commit_hash(processor_config_file, kwargs.get("_commit_hash"))
+        if commit_hash is not None:
+            kwargs["_commit_hash"] = cached_file_kwargs["_commit_hash"] = commit_hash
         if processor_config_file is not None:
             config_dict, _ = ProcessorMixin.get_processor_dict(pretrained_model_name_or_path, **kwargs)
             processor_class = config_dict.get("processor_class")
@@ -234,6 +247,9 @@ def from_pretrained(cls, pretrained_model_name_or_path, **kwargs):
             preprocessor_config_file = cached_file(
                 pretrained_model_name_or_path, FEATURE_EXTRACTOR_NAME, **cached_file_kwargs
             )
+            commit_hash = extract_commit_hash(preprocessor_config_file, commit_hash)
+            if commit_hash is not None:
+                kwargs["_commit_hash"] = cached_file_kwargs["_commit_hash"] = commit_hash
             if preprocessor_config_file is not None:
                 config_dict, _ = ImageProcessingMixin.get_image_processor_dict(pretrained_model_name_or_path, **kwargs)
                 processor_class = config_dict.get("processor_class", None)
@@ -245,6 +261,9 @@ def from_pretrained(cls, pretrained_model_name_or_path, **kwargs):
                 preprocessor_config_file = cached_file(
                     pretrained_model_name_or_path, VIDEO_PROCESSOR_NAME, **cached_file_kwargs
                 )
+                commit_hash = extract_commit_hash(preprocessor_config_file, commit_hash)
+                if commit_hash is not None:
+                    kwargs["_commit_hash"] = cached_file_kwargs["_commit_hash"] = commit_hash
                 if preprocessor_config_file is not None:
                     config_dict, _ = BaseVideoProcessor.get_video_processor_dict(
                         pretrained_model_name_or_path, **kwargs
@@ -257,6 +276,9 @@ def from_pretrained(cls, pretrained_model_name_or_path, **kwargs):
                 preprocessor_config_file = cached_file(
                     pretrained_model_name_or_path, FEATURE_EXTRACTOR_NAME, **cached_file_kwargs
                 )
+                commit_hash = extract_commit_hash(preprocessor_config_file, commit_hash)
+                if commit_hash is not None:
+                    kwargs["_commit_hash"] = cached_file_kwargs["_commit_hash"] = commit_hash
                 if preprocessor_config_file is not None and processor_class is None:
                     config_dict, _ = FeatureExtractionMixin.get_feature_extractor_dict(
                         pretrained_model_name_or_path, **kwargs
@@ -270,6 +292,9 @@ def from_pretrained(cls, pretrained_model_name_or_path, **kwargs):
             tokenizer_config_file = cached_file(
                 pretrained_model_name_or_path, TOKENIZER_CONFIG_FILE, **cached_file_kwargs
             )
+            commit_hash = extract_commit_hash(tokenizer_config_file, commit_hash)
+            if commit_hash is not None:
+                kwargs["_commit_hash"] = cached_file_kwargs["_commit_hash"] = commit_hash
             if tokenizer_config_file is not None:
                 with open(tokenizer_config_file, encoding="utf-8") as reader:
                     config_dict = json.load(reader)
@@ -289,6 +314,8 @@ def from_pretrained(cls, pretrained_model_name_or_path, **kwargs):
                     config = AutoConfig.from_pretrained(
                         pretrained_model_name_or_path, trust_remote_code=trust_remote_code, **kwargs
                     )
+                if (commit_hash := getattr(config, "_commit_hash", None)) is not None:
+                    kwargs["_commit_hash"] = commit_hash
 
                 processor_class = getattr(config, "processor_class", None)
                 if hasattr(config, "auto_map") and "AutoProcessor" in config.auto_map:

src/transformers/models/auto/tokenization_auto.py

@@ -457,7 +457,7 @@ def _has_tekken_tokenizer_file(
         return has_file(
             pretrained_model_name_or_path,
             tekken_filename,
-            revision=kwargs.get("revision"),
+            revision=kwargs.get("_commit_hash") or kwargs.get("revision"),
             token=kwargs.get("token"),
             cache_dir=kwargs.get("cache_dir"),
             local_files_only=kwargs.get("local_files_only", False),
@@ -740,6 +740,7 @@ def from_pretrained(
 
         if gguf_file:
             gguf_path = cached_file(pretrained_model_name_or_path, gguf_file, **kwargs)
+            kwargs["_commit_hash"] = extract_commit_hash(gguf_path, kwargs.get("_commit_hash"))
             config_dict = load_gguf_checkpoint(gguf_path, return_tensors=False)["config"]
             config = AutoConfig.for_model(**config_dict)
         elif config is None:
@@ -750,6 +751,9 @@ def from_pretrained(
             except (ValueError, OSError):
                 config = PreTrainedConfig.from_pretrained(pretrained_model_name_or_path, **kwargs)
 
+        if kwargs.get("_commit_hash") is None and (commit_hash := getattr(config, "_commit_hash", None)) is not None:
+            kwargs["_commit_hash"] = commit_hash
+
         config_model_type = config.model_type
         config_model_name = config.model_name if hasattr(config, "model_name") else None