diff --git a/src/transformers/feature_extraction_utils.py b/src/transformers/feature_extraction_utils.py index 0b03555be9fb..bf26597b47c9 100644 --- a/src/transformers/feature_extraction_utils.py +++ b/src/transformers/feature_extraction_utils.py @@ -40,7 +40,7 @@ requires_backends, safe_load_json_file, ) -from .utils.hub import cached_file, hf_api +from .utils.hub import cached_file, extract_commit_hash, hf_api if TYPE_CHECKING: @@ -377,6 +377,7 @@ def from_pretrained( kwargs["token"] = token feature_extractor_dict, kwargs = cls.get_feature_extractor_dict(pretrained_model_name_or_path, **kwargs) + kwargs.pop("_commit_hash", None) return cls.from_dict(feature_extractor_dict, **kwargs) @@ -450,6 +451,7 @@ def get_feature_extractor_dict( token = kwargs.pop("token", None) local_files_only = kwargs.pop("local_files_only", False) revision = kwargs.pop("revision", None) + commit_hash = kwargs.pop("_commit_hash", None) from_pipeline = kwargs.pop("_from_pipeline", None) from_auto_class = kwargs.pop("_from_auto", False) @@ -486,7 +488,9 @@ def get_feature_extractor_dict( revision=revision, subfolder=subfolder, _raise_exceptions_for_missing_entries=False, + _commit_hash=commit_hash, ) + commit_hash = extract_commit_hash(resolved_processor_file, commit_hash) resolved_feature_extractor_file = cached_file( pretrained_model_name_or_path, filename=feature_extractor_file, @@ -499,7 +503,9 @@ def get_feature_extractor_dict( revision=revision, subfolder=subfolder, _raise_exceptions_for_missing_entries=False, + _commit_hash=commit_hash, ) + commit_hash = extract_commit_hash(resolved_feature_extractor_file, commit_hash) except OSError: # Raise any environment error raise by `cached_file`. It will have a helpful error message adapted to # the original exception. @@ -540,6 +546,8 @@ def get_feature_extractor_dict( f"loading configuration file {feature_extractor_file} from cache at {resolved_feature_extractor_file}" ) + if commit_hash is not None: + kwargs["_commit_hash"] = commit_hash return feature_extractor_dict, kwargs @classmethod diff --git a/src/transformers/image_processing_base.py b/src/transformers/image_processing_base.py index f5d362944ced..f505ef8af02e 100644 --- a/src/transformers/image_processing_base.py +++ b/src/transformers/image_processing_base.py @@ -31,7 +31,7 @@ logging, safe_load_json_file, ) -from .utils.hub import cached_file, hf_api +from .utils.hub import cached_file, extract_commit_hash, hf_api ImageProcessorType = TypeVar("ImageProcessorType", bound="ImageProcessingMixin") @@ -177,6 +177,7 @@ def from_pretrained( kwargs["token"] = token image_processor_dict, kwargs = cls.get_image_processor_dict(pretrained_model_name_or_path, **kwargs) + kwargs.pop("_commit_hash", None) return cls.from_dict(image_processor_dict, **kwargs) @@ -256,6 +257,7 @@ def get_image_processor_dict( revision = kwargs.pop("revision", None) subfolder = kwargs.pop("subfolder", "") image_processor_filename = kwargs.pop("image_processor_filename", IMAGE_PROCESSOR_NAME) + commit_hash = kwargs.pop("_commit_hash", None) from_pipeline = kwargs.pop("_from_pipeline", None) from_auto_class = kwargs.pop("_from_auto", False) @@ -291,7 +293,9 @@ def get_image_processor_dict( revision=revision, subfolder=subfolder, _raise_exceptions_for_missing_entries=False, + _commit_hash=commit_hash, ) + commit_hash = extract_commit_hash(resolved_processor_file, commit_hash) resolved_image_processor_file = cached_file( pretrained_model_name_or_path, filename=image_processor_file, @@ -304,7 +308,9 @@ def get_image_processor_dict( revision=revision, subfolder=subfolder, _raise_exceptions_for_missing_entries=False, + _commit_hash=commit_hash, ) + commit_hash = extract_commit_hash(resolved_image_processor_file, commit_hash) except OSError: # Raise any environment error raise by `cached_file`. It will have a helpful error message adapted to # the original exception. @@ -345,6 +351,8 @@ def get_image_processor_dict( f"loading configuration file {image_processor_file} from cache at {resolved_image_processor_file}" ) + if commit_hash is not None: + kwargs["_commit_hash"] = commit_hash return image_processor_dict, kwargs @classmethod diff --git a/src/transformers/models/auto/feature_extraction_auto.py b/src/transformers/models/auto/feature_extraction_auto.py index eba04a7c799d..06a2c4402972 100644 --- a/src/transformers/models/auto/feature_extraction_auto.py +++ b/src/transformers/models/auto/feature_extraction_auto.py @@ -21,7 +21,15 @@ from ...configuration_utils import PreTrainedConfig from ...dynamic_module_utils import get_class_from_dynamic_module, resolve_trust_remote_code from ...feature_extraction_utils import FeatureExtractionMixin -from ...utils import CONFIG_NAME, FEATURE_EXTRACTOR_NAME, PROCESSOR_NAME, cached_file, logging, safe_load_json_file +from ...utils import ( + CONFIG_NAME, + FEATURE_EXTRACTOR_NAME, + PROCESSOR_NAME, + cached_file, + extract_commit_hash, + logging, + safe_load_json_file, +) from .auto_factory import _LazyAutoMapping from .auto_mappings import FEATURE_EXTRACTOR_MAPPING_NAMES from .configuration_auto import ( @@ -167,6 +175,7 @@ def get_feature_extractor_config( feature_extractor.save_pretrained("feature-extractor-test") feature_extractor_config = get_feature_extractor_config("feature-extractor-test") ```""" + commit_hash = kwargs.get("_commit_hash") # Load with a priority given to the nested processor config, if available in repo resolved_processor_file = cached_file( pretrained_model_name_or_path, @@ -179,7 +188,9 @@ def get_feature_extractor_config( local_files_only=local_files_only, _raise_exceptions_for_gated_repo=False, _raise_exceptions_for_missing_entries=False, + _commit_hash=commit_hash, ) + commit_hash = extract_commit_hash(resolved_processor_file, commit_hash) resolved_feature_extractor_file = cached_file( pretrained_model_name_or_path, filename=FEATURE_EXTRACTOR_NAME, @@ -191,6 +202,7 @@ def get_feature_extractor_config( local_files_only=local_files_only, _raise_exceptions_for_gated_repo=False, _raise_exceptions_for_missing_entries=False, + _commit_hash=commit_hash, ) # An empty list if none of the possible files is found in the repo @@ -299,8 +311,14 @@ def from_pretrained(cls, pretrained_model_name_or_path, **kwargs): config = kwargs.pop("config", None) trust_remote_code = kwargs.pop("trust_remote_code", None) kwargs["_from_auto"] = True + if kwargs.get("_commit_hash") is None and (commit_hash := getattr(config, "_commit_hash", None)) is not None: + kwargs["_commit_hash"] = commit_hash - config_dict, _ = FeatureExtractionMixin.get_feature_extractor_dict(pretrained_model_name_or_path, **kwargs) + config_dict, unused_kwargs = FeatureExtractionMixin.get_feature_extractor_dict( + pretrained_model_name_or_path, **kwargs + ) + if (commit_hash := unused_kwargs.pop("_commit_hash", None)) is not None: + kwargs["_commit_hash"] = commit_hash feature_extractor_class = config_dict.get("feature_extractor_type", None) feature_extractor_auto_map = None if "AutoFeatureExtractor" in config_dict.get("auto_map", {}): @@ -312,6 +330,8 @@ def from_pretrained(cls, pretrained_model_name_or_path, **kwargs): config = AutoConfig.from_pretrained( pretrained_model_name_or_path, trust_remote_code=trust_remote_code, **kwargs ) + if (commit_hash := getattr(config, "_commit_hash", None)) is not None: + kwargs["_commit_hash"] = commit_hash # It could be in `config.feature_extractor_type`` feature_extractor_class = getattr(config, "feature_extractor_type", None) if hasattr(config, "auto_map") and "AutoFeatureExtractor" in config.auto_map: diff --git a/src/transformers/models/auto/image_processing_auto.py b/src/transformers/models/auto/image_processing_auto.py index e96fe67321ad..7333f4920d59 100644 --- a/src/transformers/models/auto/image_processing_auto.py +++ b/src/transformers/models/auto/image_processing_auto.py @@ -27,6 +27,7 @@ IMAGE_PROCESSOR_NAME, PROCESSOR_NAME, cached_file, + extract_commit_hash, is_timm_config_dict, is_timm_local_checkpoint, is_torchvision_available, @@ -269,6 +270,7 @@ def get_image_processor_config( image_processor.save_pretrained("image-processor-test") image_processor_config = get_image_processor_config("image-processor-test") ```""" + commit_hash = kwargs.get("_commit_hash") # Load with a priority given to the nested processor config, if available in repo resolved_processor_file = cached_file( pretrained_model_name_or_path, @@ -281,7 +283,9 @@ def get_image_processor_config( local_files_only=local_files_only, _raise_exceptions_for_gated_repo=False, _raise_exceptions_for_missing_entries=False, + _commit_hash=commit_hash, ) + commit_hash = extract_commit_hash(resolved_processor_file, commit_hash) resolved_image_processor_file = cached_file( pretrained_model_name_or_path, filename=IMAGE_PROCESSOR_NAME, @@ -293,6 +297,7 @@ def get_image_processor_config( local_files_only=local_files_only, _raise_exceptions_for_gated_repo=False, _raise_exceptions_for_missing_entries=False, + _commit_hash=commit_hash, ) # An empty list if none of the possible files is found in the repo @@ -578,6 +583,8 @@ def from_pretrained(cls, pretrained_model_name_or_path, *inputs, **kwargs): backend_kwarg = kwargs.pop("backend", None) trust_remote_code = kwargs.pop("trust_remote_code", None) kwargs["_from_auto"] = True + if kwargs.get("_commit_hash") is None and (commit_hash := getattr(config, "_commit_hash", None)) is not None: + kwargs["_commit_hash"] = commit_hash # Resolve the image processor config filename if "image_processor_filename" in kwargs: @@ -590,15 +597,19 @@ def from_pretrained(cls, pretrained_model_name_or_path, *inputs, **kwargs): # Load the image processor config try: - config_dict, _ = ImageProcessingMixin.get_image_processor_dict( + config_dict, unused_kwargs = ImageProcessingMixin.get_image_processor_dict( pretrained_model_name_or_path, image_processor_filename=image_processor_filename, **kwargs ) + if (commit_hash := unused_kwargs.pop("_commit_hash", None)) is not None: + kwargs["_commit_hash"] = commit_hash except Exception as initial_exception: # Fallback for Hub TimmWrapper checkpoints (image processing in config.json, not preprocessor_config.json) try: - config_dict, _ = ImageProcessingMixin.get_image_processor_dict( + config_dict, unused_kwargs = ImageProcessingMixin.get_image_processor_dict( pretrained_model_name_or_path, image_processor_filename=CONFIG_NAME, **kwargs ) + if (commit_hash := unused_kwargs.pop("_commit_hash", None)) is not None: + kwargs["_commit_hash"] = commit_hash except Exception: raise initial_exception @@ -626,6 +637,8 @@ def from_pretrained(cls, pretrained_model_name_or_path, *inputs, **kwargs): config = AutoConfig.from_pretrained( pretrained_model_name_or_path, trust_remote_code=trust_remote_code, **kwargs ) + if (commit_hash := getattr(config, "_commit_hash", None)) is not None: + kwargs["_commit_hash"] = commit_hash image_processor_type = getattr(config, "image_processor_type", None) if hasattr(config, "auto_map") and "AutoImageProcessor" in config.auto_map: diff --git a/src/transformers/models/auto/processing_auto.py b/src/transformers/models/auto/processing_auto.py index 05642dc47ecd..facf4eacfffc 100644 --- a/src/transformers/models/auto/processing_auto.py +++ b/src/transformers/models/auto/processing_auto.py @@ -25,7 +25,14 @@ from ...image_processing_utils import ImageProcessingMixin from ...processing_utils import ProcessorMixin from ...tokenization_python import TOKENIZER_CONFIG_FILE -from ...utils import FEATURE_EXTRACTOR_NAME, PROCESSOR_NAME, VIDEO_PROCESSOR_NAME, cached_file, logging +from ...utils import ( + FEATURE_EXTRACTOR_NAME, + PROCESSOR_NAME, + VIDEO_PROCESSOR_NAME, + cached_file, + extract_commit_hash, + logging, +) from ...video_processing_utils import BaseVideoProcessor from .auto_factory import _LazyAutoMapping from .auto_mappings import PROCESSOR_MAPPING_NAMES @@ -194,6 +201,8 @@ def from_pretrained(cls, pretrained_model_name_or_path, **kwargs): config = kwargs.pop("config", None) trust_remote_code = kwargs.pop("trust_remote_code", None) kwargs["_from_auto"] = True + if kwargs.get("_commit_hash") is None and (commit_hash := getattr(config, "_commit_hash", None)) is not None: + kwargs["_commit_hash"] = commit_hash processor_class = None processor_auto_map = None @@ -210,6 +219,7 @@ def from_pretrained(cls, pretrained_model_name_or_path, **kwargs): "subfolder", "repo_type", "user_agent", + "_commit_hash", ) cached_file_kwargs = {key: kwargs[key] for key in _hub_valid_kwargs if key in kwargs} # We don't want to raise @@ -223,6 +233,9 @@ def from_pretrained(cls, pretrained_model_name_or_path, **kwargs): # Let's start by checking whether the processor class is saved in a processor config processor_config_file = cached_file(pretrained_model_name_or_path, PROCESSOR_NAME, **cached_file_kwargs) + commit_hash = extract_commit_hash(processor_config_file, kwargs.get("_commit_hash")) + if commit_hash is not None: + kwargs["_commit_hash"] = cached_file_kwargs["_commit_hash"] = commit_hash if processor_config_file is not None: config_dict, _ = ProcessorMixin.get_processor_dict(pretrained_model_name_or_path, **kwargs) processor_class = config_dict.get("processor_class") @@ -234,6 +247,9 @@ def from_pretrained(cls, pretrained_model_name_or_path, **kwargs): preprocessor_config_file = cached_file( pretrained_model_name_or_path, FEATURE_EXTRACTOR_NAME, **cached_file_kwargs ) + commit_hash = extract_commit_hash(preprocessor_config_file, commit_hash) + if commit_hash is not None: + kwargs["_commit_hash"] = cached_file_kwargs["_commit_hash"] = commit_hash if preprocessor_config_file is not None: config_dict, _ = ImageProcessingMixin.get_image_processor_dict(pretrained_model_name_or_path, **kwargs) processor_class = config_dict.get("processor_class", None) @@ -245,6 +261,9 @@ def from_pretrained(cls, pretrained_model_name_or_path, **kwargs): preprocessor_config_file = cached_file( pretrained_model_name_or_path, VIDEO_PROCESSOR_NAME, **cached_file_kwargs ) + commit_hash = extract_commit_hash(preprocessor_config_file, commit_hash) + if commit_hash is not None: + kwargs["_commit_hash"] = cached_file_kwargs["_commit_hash"] = commit_hash if preprocessor_config_file is not None: config_dict, _ = BaseVideoProcessor.get_video_processor_dict( pretrained_model_name_or_path, **kwargs @@ -257,6 +276,9 @@ def from_pretrained(cls, pretrained_model_name_or_path, **kwargs): preprocessor_config_file = cached_file( pretrained_model_name_or_path, FEATURE_EXTRACTOR_NAME, **cached_file_kwargs ) + commit_hash = extract_commit_hash(preprocessor_config_file, commit_hash) + if commit_hash is not None: + kwargs["_commit_hash"] = cached_file_kwargs["_commit_hash"] = commit_hash if preprocessor_config_file is not None and processor_class is None: config_dict, _ = FeatureExtractionMixin.get_feature_extractor_dict( pretrained_model_name_or_path, **kwargs @@ -270,6 +292,9 @@ def from_pretrained(cls, pretrained_model_name_or_path, **kwargs): tokenizer_config_file = cached_file( pretrained_model_name_or_path, TOKENIZER_CONFIG_FILE, **cached_file_kwargs ) + commit_hash = extract_commit_hash(tokenizer_config_file, commit_hash) + if commit_hash is not None: + kwargs["_commit_hash"] = cached_file_kwargs["_commit_hash"] = commit_hash if tokenizer_config_file is not None: with open(tokenizer_config_file, encoding="utf-8") as reader: config_dict = json.load(reader) @@ -289,6 +314,8 @@ def from_pretrained(cls, pretrained_model_name_or_path, **kwargs): config = AutoConfig.from_pretrained( pretrained_model_name_or_path, trust_remote_code=trust_remote_code, **kwargs ) + if (commit_hash := getattr(config, "_commit_hash", None)) is not None: + kwargs["_commit_hash"] = commit_hash processor_class = getattr(config, "processor_class", None) if hasattr(config, "auto_map") and "AutoProcessor" in config.auto_map: diff --git a/src/transformers/models/auto/tokenization_auto.py b/src/transformers/models/auto/tokenization_auto.py index 272445c2128e..f20ff3eec78c 100644 --- a/src/transformers/models/auto/tokenization_auto.py +++ b/src/transformers/models/auto/tokenization_auto.py @@ -457,7 +457,7 @@ def _has_tekken_tokenizer_file( return has_file( pretrained_model_name_or_path, tekken_filename, - revision=kwargs.get("revision"), + revision=kwargs.get("_commit_hash") or kwargs.get("revision"), token=kwargs.get("token"), cache_dir=kwargs.get("cache_dir"), local_files_only=kwargs.get("local_files_only", False), @@ -740,6 +740,7 @@ def from_pretrained( if gguf_file: gguf_path = cached_file(pretrained_model_name_or_path, gguf_file, **kwargs) + kwargs["_commit_hash"] = extract_commit_hash(gguf_path, kwargs.get("_commit_hash")) config_dict = load_gguf_checkpoint(gguf_path, return_tensors=False)["config"] config = AutoConfig.for_model(**config_dict) elif config is None: @@ -750,6 +751,9 @@ def from_pretrained( except (ValueError, OSError): config = PreTrainedConfig.from_pretrained(pretrained_model_name_or_path, **kwargs) + if kwargs.get("_commit_hash") is None and (commit_hash := getattr(config, "_commit_hash", None)) is not None: + kwargs["_commit_hash"] = commit_hash + config_model_type = config.model_type config_model_name = config.model_name if hasattr(config, "model_name") else None diff --git a/src/transformers/models/auto/video_processing_auto.py b/src/transformers/models/auto/video_processing_auto.py index 87bdac748075..a1a866b436f4 100644 --- a/src/transformers/models/auto/video_processing_auto.py +++ b/src/transformers/models/auto/video_processing_auto.py @@ -27,6 +27,7 @@ PROCESSOR_NAME, VIDEO_PROCESSOR_NAME, cached_file, + extract_commit_hash, is_torchvision_available, logging, safe_load_json_file, @@ -170,6 +171,7 @@ def get_video_processor_config( video_processor.save_pretrained("video-processor-test") video_processor = get_video_processor_config("video-processor-test") ```""" + commit_hash = kwargs.get("_commit_hash") # Load with a priority given to the nested processor config, if available in repo resolved_processor_file = cached_file( pretrained_model_name_or_path, @@ -182,27 +184,28 @@ def get_video_processor_config( local_files_only=local_files_only, _raise_exceptions_for_gated_repo=False, _raise_exceptions_for_missing_entries=False, + _commit_hash=commit_hash, ) - resolved_video_processor_files = [ - resolved_file - for filename in [VIDEO_PROCESSOR_NAME, IMAGE_PROCESSOR_NAME] - if ( - resolved_file := cached_file( - pretrained_model_name_or_path, - filename=filename, - cache_dir=cache_dir, - force_download=force_download, - proxies=proxies, - token=token, - revision=revision, - local_files_only=local_files_only, - _raise_exceptions_for_gated_repo=False, - _raise_exceptions_for_missing_entries=False, - _raise_exceptions_for_connection_errors=False, - ) + commit_hash = extract_commit_hash(resolved_processor_file, commit_hash) + resolved_video_processor_files = [] + for filename in [VIDEO_PROCESSOR_NAME, IMAGE_PROCESSOR_NAME]: + resolved_file = cached_file( + pretrained_model_name_or_path, + filename=filename, + cache_dir=cache_dir, + force_download=force_download, + proxies=proxies, + token=token, + revision=revision, + local_files_only=local_files_only, + _raise_exceptions_for_gated_repo=False, + _raise_exceptions_for_missing_entries=False, + _raise_exceptions_for_connection_errors=False, + _commit_hash=commit_hash, ) - is not None - ] + commit_hash = extract_commit_hash(resolved_file, commit_hash) + if resolved_file is not None: + resolved_video_processor_files.append(resolved_file) resolved_video_processor_file = resolved_video_processor_files[0] if resolved_video_processor_files else None # An empty list if none of the possible files is found in the repo @@ -313,8 +316,14 @@ def from_pretrained(cls, pretrained_model_name_or_path, *inputs, **kwargs): config = kwargs.pop("config", None) trust_remote_code = kwargs.pop("trust_remote_code", None) kwargs["_from_auto"] = True + if kwargs.get("_commit_hash") is None and (commit_hash := getattr(config, "_commit_hash", None)) is not None: + kwargs["_commit_hash"] = commit_hash - config_dict, _ = BaseVideoProcessor.get_video_processor_dict(pretrained_model_name_or_path, **kwargs) + config_dict, unused_kwargs = BaseVideoProcessor.get_video_processor_dict( + pretrained_model_name_or_path, **kwargs + ) + if (commit_hash := unused_kwargs.pop("_commit_hash", None)) is not None: + kwargs["_commit_hash"] = commit_hash video_processor_class = config_dict.get("video_processor_type", None) video_processor_auto_map = None if "AutoVideoProcessor" in config_dict.get("auto_map", {}): @@ -342,6 +351,8 @@ def from_pretrained(cls, pretrained_model_name_or_path, *inputs, **kwargs): config = AutoConfig.from_pretrained( pretrained_model_name_or_path, trust_remote_code=trust_remote_code, **kwargs ) + if (commit_hash := getattr(config, "_commit_hash", None)) is not None: + kwargs["_commit_hash"] = commit_hash # It could be in `config.video_processor_type`` video_processor_class = getattr(config, "video_processor_type", None) diff --git a/src/transformers/models/bark/processing_bark.py b/src/transformers/models/bark/processing_bark.py index 9ff3b6dff581..105b24697b88 100644 --- a/src/transformers/models/bark/processing_bark.py +++ b/src/transformers/models/bark/processing_bark.py @@ -24,7 +24,7 @@ from ...processing_utils import ProcessorMixin from ...tokenization_utils_base import BatchEncoding from ...utils import auto_docstring, logging -from ...utils.hub import cached_file +from ...utils.hub import cached_file, extract_commit_hash from ..auto import AutoTokenizer @@ -89,7 +89,10 @@ def from_pretrained( _raise_exceptions_for_gated_repo=False, _raise_exceptions_for_missing_entries=False, _raise_exceptions_for_connection_errors=False, + _commit_hash=kwargs.get("_commit_hash"), ) + if (commit_hash := extract_commit_hash(speaker_embeddings_path, kwargs.get("_commit_hash"))) is not None: + kwargs["_commit_hash"] = commit_hash if speaker_embeddings_path is None: logger.warning( f"""`{os.path.join(pretrained_processor_name_or_path, speaker_embeddings_dict_path)}` does not exists @@ -188,6 +191,13 @@ def _load_voice_preset(self, voice_preset: str | None = None, **kwargs): voice_preset_dict = {} token = kwargs.get("token") repo_or_path = self.speaker_embeddings.get("repo_or_path", "/") + subfolder = kwargs.pop("subfolder", None) + cache_dir = kwargs.pop("cache_dir", None) + force_download = kwargs.pop("force_download", False) + proxies = kwargs.pop("proxies", None) + local_files_only = kwargs.pop("local_files_only", False) + revision = kwargs.pop("revision", None) + commit_hash = kwargs.pop("_commit_hash", None) for key in ["semantic_prompt", "coarse_prompt", "fine_prompt"]: if key not in voice_preset_paths: raise ValueError( @@ -200,17 +210,19 @@ def _load_voice_preset(self, voice_preset: str | None = None, **kwargs): path = cached_file( self.speaker_embeddings.get("repo_or_path", "/"), voice_preset_paths[key], - subfolder=kwargs.pop("subfolder", None), - cache_dir=kwargs.pop("cache_dir", None), - force_download=kwargs.pop("force_download", False), - proxies=kwargs.pop("proxies", None), - local_files_only=kwargs.pop("local_files_only", False), + subfolder=subfolder, + cache_dir=cache_dir, + force_download=force_download, + proxies=proxies, + local_files_only=local_files_only, token=token, - revision=kwargs.pop("revision", None), + revision=revision, _raise_exceptions_for_gated_repo=False, _raise_exceptions_for_missing_entries=False, _raise_exceptions_for_connection_errors=False, + _commit_hash=commit_hash, ) + commit_hash = extract_commit_hash(path, commit_hash) if path is None: raise ValueError( f"""`{os.path.join(self.speaker_embeddings.get("repo_or_path", "/"), voice_preset_paths[key])}` does not exists diff --git a/src/transformers/models/ernie4_5_vl_moe/video_processing_ernie4_5_vl_moe.py b/src/transformers/models/ernie4_5_vl_moe/video_processing_ernie4_5_vl_moe.py index 4959cfc6b767..b149df13da67 100644 --- a/src/transformers/models/ernie4_5_vl_moe/video_processing_ernie4_5_vl_moe.py +++ b/src/transformers/models/ernie4_5_vl_moe/video_processing_ernie4_5_vl_moe.py @@ -44,7 +44,7 @@ logging, safe_load_json_file, ) -from ...utils.hub import cached_file +from ...utils.hub import cached_file, extract_commit_hash from ...utils.import_utils import is_torchvision_available, is_tracing, requires from ...video_processing_utils import BASE_VIDEO_PROCESSOR_DOCSTRING, BaseVideoProcessor from ...video_utils import ( @@ -143,6 +143,7 @@ def get_video_processor_dict( local_files_only = kwargs.pop("local_files_only", False) revision = kwargs.pop("revision", None) subfolder = kwargs.pop("subfolder", "") + commit_hash = kwargs.pop("_commit_hash", None) from_pipeline = kwargs.pop("_from_pipeline", None) from_auto_class = kwargs.pop("_from_auto", False) @@ -178,27 +179,28 @@ def get_video_processor_dict( revision=revision, subfolder=subfolder, _raise_exceptions_for_missing_entries=False, + _commit_hash=commit_hash, ) - resolved_video_processor_files = [ - resolved_file - for filename in [video_processor_file, IMAGE_PROCESSOR_NAME] - if ( - resolved_file := cached_file( - pretrained_model_name_or_path, - filename=filename, - cache_dir=cache_dir, - force_download=force_download, - proxies=proxies, - local_files_only=local_files_only, - token=token, - user_agent=user_agent, - revision=revision, - subfolder=subfolder, - _raise_exceptions_for_missing_entries=False, - ) + commit_hash = extract_commit_hash(resolved_processor_file, commit_hash) + resolved_video_processor_files = [] + for filename in [video_processor_file, IMAGE_PROCESSOR_NAME]: + resolved_file = cached_file( + pretrained_model_name_or_path, + filename=filename, + cache_dir=cache_dir, + force_download=force_download, + proxies=proxies, + local_files_only=local_files_only, + token=token, + user_agent=user_agent, + revision=revision, + subfolder=subfolder, + _raise_exceptions_for_missing_entries=False, + _commit_hash=commit_hash, ) - is not None - ] + commit_hash = extract_commit_hash(resolved_file, commit_hash) + if resolved_file is not None: + resolved_video_processor_files.append(resolved_file) resolved_video_processor_file = ( resolved_video_processor_files[0] if resolved_video_processor_files else None ) @@ -254,6 +256,7 @@ def get_video_processor_dict( revision=revision, subfolder=subfolder, _raise_exceptions_for_missing_entries=False, + _commit_hash=commit_hash, ) try: ImageFont.truetype(video_processor_dict["font"]) @@ -270,6 +273,8 @@ def get_video_processor_dict( f"loading configuration file {video_processor_file} from cache at {resolved_video_processor_file}" ) + if commit_hash is not None: + kwargs["_commit_hash"] = commit_hash return video_processor_dict, kwargs def to_dict(self) -> dict[str, Any]: diff --git a/src/transformers/processing_utils.py b/src/transformers/processing_utils.py index 7f162f4f731c..1aabcb967a68 100644 --- a/src/transformers/processing_utils.py +++ b/src/transformers/processing_utils.py @@ -57,6 +57,7 @@ cached_file, copy_func, direct_transformers_import, + extract_commit_hash, hf_api, is_torch_available, list_repo_templates, @@ -1224,6 +1225,7 @@ def get_processor_dict( """ # holding a copy for optionally loading the audio tokenizer (if available) audio_tokenizer_kwargs = copy.deepcopy(kwargs) + audio_tokenizer_kwargs.pop("_commit_hash", None) cache_dir = kwargs.pop("cache_dir", None) force_download = kwargs.pop("force_download", False) @@ -1232,6 +1234,7 @@ def get_processor_dict( local_files_only = kwargs.pop("local_files_only", False) revision = kwargs.pop("revision", None) subfolder = kwargs.pop("subfolder", "") + commit_hash = kwargs.pop("_commit_hash", None) from_pipeline = kwargs.pop("_from_pipeline", None) from_auto_class = kwargs.pop("_from_auto", False) @@ -1270,7 +1273,7 @@ def get_processor_dict( for template in list_repo_templates( pretrained_model_name_or_path, local_files_only=local_files_only, - revision=revision, + revision=commit_hash or revision, cache_dir=cache_dir, token=token, ): @@ -1294,7 +1297,9 @@ def get_processor_dict( revision=revision, subfolder=subfolder, _raise_exceptions_for_missing_entries=False, + _commit_hash=commit_hash, ) + commit_hash = extract_commit_hash(resolved_processor_file, commit_hash) # chat_template.json is a legacy file used by the processor class # a raw chat_template.jinja is preferred in future @@ -1310,7 +1315,9 @@ def get_processor_dict( revision=revision, subfolder=subfolder, _raise_exceptions_for_missing_entries=False, + _commit_hash=commit_hash, ) + commit_hash = extract_commit_hash(resolved_chat_template_file, commit_hash) resolved_raw_chat_template_file = cached_file( pretrained_model_name_or_path, @@ -1324,10 +1331,12 @@ def get_processor_dict( revision=revision, subfolder=subfolder, _raise_exceptions_for_missing_entries=False, + _commit_hash=commit_hash, ) + commit_hash = extract_commit_hash(resolved_raw_chat_template_file, commit_hash) - resolved_additional_chat_template_files = { - template_name: cached_file( + for template_name, template_file in additional_chat_template_files.items(): + resolved_additional_chat_template_files[template_name] = cached_file( pretrained_model_name_or_path, template_file, cache_dir=cache_dir, @@ -1339,9 +1348,11 @@ def get_processor_dict( revision=revision, subfolder=subfolder, _raise_exceptions_for_missing_entries=False, + _commit_hash=commit_hash, + ) + commit_hash = extract_commit_hash( + resolved_additional_chat_template_files[template_name], commit_hash ) - for template_name, template_file in additional_chat_template_files.items() - } resolved_audio_tokenizer_file = cached_file( pretrained_model_name_or_path, @@ -1355,7 +1366,9 @@ def get_processor_dict( revision=revision, subfolder=subfolder, _raise_exceptions_for_missing_entries=False, + _commit_hash=commit_hash, ) + commit_hash = extract_commit_hash(resolved_audio_tokenizer_file, commit_hash) except OSError: # Raise any environment error raise by `cached_file`. It will have a helpful error message adapted to # the original exception. @@ -1441,6 +1454,8 @@ def get_processor_dict( audio_tokenizer_path, **audio_tokenizer_kwargs ) + if commit_hash is not None: + kwargs["_commit_hash"] = commit_hash return processor_dict, kwargs @classmethod @@ -1719,6 +1734,8 @@ def from_pretrained( # Get processor_dict first so we can use it to instantiate non-tokenizer sub-processors processor_dict, instantiation_kwargs = cls.get_processor_dict(pretrained_model_name_or_path, **kwargs) + if (commit_hash := instantiation_kwargs.pop("_commit_hash", None)) is not None: + kwargs["_commit_hash"] = commit_hash args = cls._get_arguments_from_pretrained(pretrained_model_name_or_path, processor_dict, **kwargs) return cls.from_args_and_dict(args, processor_dict, **instantiation_kwargs) diff --git a/src/transformers/tokenization_utils_base.py b/src/transformers/tokenization_utils_base.py index c3b13fb33294..123f0c91f089 100644 --- a/src/transformers/tokenization_utils_base.py +++ b/src/transformers/tokenization_utils_base.py @@ -1675,7 +1675,7 @@ def from_pretrained( for template in list_repo_templates( pretrained_model_name_or_path, local_files_only=local_files_only, - revision=revision, + revision=commit_hash or revision, cache_dir=cache_dir, token=token, ): @@ -1685,7 +1685,9 @@ def from_pretrained( remote_files = [] if not is_local and not local_files_only: try: - remote_files = hf_api().list_repo_files(pretrained_model_name_or_path, revision=revision) + remote_files = hf_api().list_repo_files( + pretrained_model_name_or_path, revision=commit_hash or revision + ) except Exception: remote_files = [] elif pretrained_model_name_or_path and os.path.isdir(pretrained_model_name_or_path): diff --git a/src/transformers/utils/hub.py b/src/transformers/utils/hub.py index 542f1175d8f0..48d2948ca29f 100644 --- a/src/transformers/utils/hub.py +++ b/src/transformers/utils/hub.py @@ -357,7 +357,8 @@ def cached_files( if False, do not raise an exception for connection errors but return None. _commit_hash (`str`, *optional*): passed when we are chaining several calls to various files (e.g. when loading a tokenizer or - a pipeline). If files are cached for this commit hash, avoid calls to head and get from the cache. + a pipeline). Cached files are loaded directly, and cache misses are resolved against this commit + instead of resolving the requested revision again. @@ -439,7 +440,7 @@ def cached_files( filenames[0], subfolder=None if len(subfolder) == 0 else subfolder, repo_type=repo_type, - revision=revision, + revision=_commit_hash or revision, cache_dir=cache_dir, user_agent=user_agent, force_download=force_download, @@ -453,7 +454,7 @@ def cached_files( path_or_repo_id, allow_patterns=full_filenames, repo_type=repo_type, - revision=revision, + revision=_commit_hash or revision, cache_dir=cache_dir, user_agent=user_agent, force_download=force_download, @@ -489,7 +490,7 @@ def cached_files( # Now we try to recover if we can find all files correctly in the cache resolved_files = [ - _get_cache_file_to_return(path_or_repo_id, filename, cache_dir, revision, repo_type) + _get_cache_file_to_return(path_or_repo_id, filename, cache_dir, _commit_hash or revision, repo_type) for filename in full_filenames ] if all(file is not None for file in resolved_files): @@ -527,7 +528,8 @@ def cached_files( raise e resolved_files = [ - _get_cache_file_to_return(path_or_repo_id, filename, cache_dir, revision) for filename in full_filenames + _get_cache_file_to_return(path_or_repo_id, filename, cache_dir, _commit_hash or revision) + for filename in full_filenames ] # If there are any missing file and the flag is active, raise if any(file is None for file in resolved_files) and _raise_exceptions_for_missing_entries: diff --git a/src/transformers/video_processing_utils.py b/src/transformers/video_processing_utils.py index f1bfbde680a8..f44cbd535e5b 100644 --- a/src/transformers/video_processing_utils.py +++ b/src/transformers/video_processing_utils.py @@ -46,7 +46,7 @@ logging, safe_load_json_file, ) -from .utils.hub import cached_file, hf_api +from .utils.hub import cached_file, extract_commit_hash, hf_api from .utils.import_utils import requires from .video_utils import ( VideoInput, @@ -512,6 +512,7 @@ def from_pretrained( kwargs["token"] = token video_processor_dict, kwargs = cls.get_video_processor_dict(pretrained_model_name_or_path, **kwargs) + kwargs.pop("_commit_hash", None) return cls.from_dict(video_processor_dict, **kwargs) @@ -588,6 +589,7 @@ def get_video_processor_dict( local_files_only = kwargs.pop("local_files_only", False) revision = kwargs.pop("revision", None) subfolder = kwargs.pop("subfolder", "") + commit_hash = kwargs.pop("_commit_hash", None) from_pipeline = kwargs.pop("_from_pipeline", None) from_auto_class = kwargs.pop("_from_auto", False) @@ -623,27 +625,28 @@ def get_video_processor_dict( revision=revision, subfolder=subfolder, _raise_exceptions_for_missing_entries=False, + _commit_hash=commit_hash, ) - resolved_video_processor_files = [ - resolved_file - for filename in [video_processor_file, IMAGE_PROCESSOR_NAME] - if ( - resolved_file := cached_file( - pretrained_model_name_or_path, - filename=filename, - cache_dir=cache_dir, - force_download=force_download, - proxies=proxies, - local_files_only=local_files_only, - token=token, - user_agent=user_agent, - revision=revision, - subfolder=subfolder, - _raise_exceptions_for_missing_entries=False, - ) + commit_hash = extract_commit_hash(resolved_processor_file, commit_hash) + resolved_video_processor_files = [] + for filename in [video_processor_file, IMAGE_PROCESSOR_NAME]: + resolved_file = cached_file( + pretrained_model_name_or_path, + filename=filename, + cache_dir=cache_dir, + force_download=force_download, + proxies=proxies, + local_files_only=local_files_only, + token=token, + user_agent=user_agent, + revision=revision, + subfolder=subfolder, + _raise_exceptions_for_missing_entries=False, + _commit_hash=commit_hash, ) - is not None - ] + commit_hash = extract_commit_hash(resolved_file, commit_hash) + if resolved_file is not None: + resolved_video_processor_files.append(resolved_file) resolved_video_processor_file = ( resolved_video_processor_files[0] if resolved_video_processor_files else None ) @@ -687,6 +690,8 @@ def get_video_processor_dict( f"loading configuration file {video_processor_file} from cache at {resolved_video_processor_file}" ) + if commit_hash is not None: + kwargs["_commit_hash"] = commit_hash return video_processor_dict, kwargs @classmethod