From 65b07161a5db1d01a635c060990d5bf7aeff5412 Mon Sep 17 00:00:00 2001 From: Xin He Date: Mon, 23 Mar 2026 19:41:51 +0800 Subject: [PATCH 1/7] Update ipex backend to be deprecated. Signed-off-by: Xin He --- auto_round/inference/backend.py | 12 ++++++------ auto_round/inference/convert_model.py | 5 +++++ docs/step_by_step.md | 16 ++++++++++------ docs/step_by_step_CN.md | 14 +++++++++----- 4 files changed, 30 insertions(+), 17 deletions(-) diff --git a/auto_round/inference/backend.py b/auto_round/inference/backend.py index 9cf5fe9610..a858f55970 100644 --- a/auto_round/inference/backend.py +++ b/auto_round/inference/backend.py @@ -509,7 +509,7 @@ def fp8_static_scheme_checker( priority=6, checkers=[ark_feature_checker], alias=["ark"], - compute_dtype=["float32", "float16"], + compute_dtype=["float32", "float16", "bfloat16"], data_type=["int"], act_bits=WOQ_DEFAULT_ACT_BITS, requirements=["torch>=2.8.0", "auto-round-lib"], @@ -524,7 +524,7 @@ def fp8_static_scheme_checker( priority=6, checkers=[ark_feature_checker], alias=["ark"], - compute_dtype=["float32", "float16"], + compute_dtype=["float32", "float16", "bfloat16"], data_type=["int"], act_bits=WOQ_DEFAULT_ACT_BITS, requirements=["torch>=2.8.0", "auto-round-lib"], @@ -539,7 +539,7 @@ def fp8_static_scheme_checker( priority=6, checkers=[ark_feature_checker], alias=["ark"], - compute_dtype=["float32", "float16"], + compute_dtype=["float32", "float16", "bfloat16"], data_type=["int"], act_bits=WOQ_DEFAULT_ACT_BITS, requirements=["torch>=2.8.0", "auto-round-lib"], @@ -554,7 +554,7 @@ def fp8_static_scheme_checker( priority=6, checkers=[ark_feature_checker], alias=["ark"], - compute_dtype=["float32", "float16"], + compute_dtype=["float32", "float16", "bfloat16"], data_type=["int"], act_bits=WOQ_DEFAULT_ACT_BITS, requirements=["torch>=2.8.0", "auto-round-lib"], @@ -569,7 +569,7 @@ def fp8_static_scheme_checker( priority=6, checkers=[ark_feature_checker], alias=["ark"], - compute_dtype=["float32", "float16"], + compute_dtype=["float32", "float16", "bfloat16"], data_type=["int"], act_bits=WOQ_DEFAULT_ACT_BITS, requirements=["torch>=2.8.0", "auto-round-lib"], @@ -584,7 +584,7 @@ def fp8_static_scheme_checker( priority=6, checkers=[ark_feature_checker], alias=["ark"], - compute_dtype=["float32", "float16"], + compute_dtype=["float32", "float16", "bfloat16"], data_type=["int"], act_bits=WOQ_DEFAULT_ACT_BITS, requirements=["torch>=2.8.0", "auto-round-lib"], diff --git a/auto_round/inference/convert_model.py b/auto_round/inference/convert_model.py index f55ca7c502..afea71b806 100644 --- a/auto_round/inference/convert_model.py +++ b/auto_round/inference/convert_model.py @@ -528,6 +528,11 @@ def post_init(model: torch.nn.Module, used_backends: list[str]) -> None: need_gptqmodel_init = True elif backend.startswith(("ipex", "auto_round_kernel")): need_ipex_init = True + if backend.startswith("ipex"): + logger.warning_once( + f"Backend '{backend}' is deprecated and will be removed in a future release. " + "Please use the 'ark' backend instead (requires auto-round-lib and torch>=2.8.0)." + ) # AutoGPTQ post-init if need_autogptq_init: diff --git a/docs/step_by_step.md b/docs/step_by_step.md index 8d322c6eb2..d7bfa2049c 100644 --- a/docs/step_by_step.md +++ b/docs/step_by_step.md @@ -629,7 +629,7 @@ AutoRound automatically selects the best available backend based on the installe ### CPU -Supports 2, 4, and 8 bits. We recommend using intel-extension-for-pytorch (IPEX) for 4 bits inference. +Supports 2, 4, and 8 bits. We recommend using auto-round-lib (ark) for inference. ```python from transformers import AutoModelForCausalLM, AutoTokenizer @@ -645,7 +645,7 @@ print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50, do_sample=Fal ### Intel GPU -Supports 4 bits only. We recommend using intel-extension-for-pytorch (IPEX) for inference. +Supports 4,8 bits. We recommend using auto-round-lib (ark) for inference. ```python from transformers import AutoModelForCausalLM, AutoTokenizer @@ -698,13 +698,13 @@ print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50, do_sample=Fal AutoRound automatically selects the backend for each layer based on compatibility. In general, the priority order is Marlin > ExLLaMAV2 > Triton, but the final choice depends on factors such as group size, bit width, packing format, hardware device, and other implementation details. The backend may not always be the most suitable for certain devices. -You can specify your preferred backend such as "ipex" for CPU and Intel GPU, "marlin/exllamav2/triton" for CUDA, according to your needs or hardware compatibility. Please note that additional corresponding libraries may be required. +You can specify your preferred backend such as "ark" for CPU and Intel GPU, "marlin/exllamav2/triton" for CUDA, according to your needs or hardware compatibility. Please note that additional corresponding libraries may be required. ```python from transformers import AutoModelForCausalLM, AutoTokenizer, AutoRoundConfig model_name = "OPEA/Qwen2.5-1.5B-Instruct-int4-sym-inc" -quantization_config = AutoRoundConfig(backend="ipex") +quantization_config = AutoRoundConfig(backend="ark") model = AutoModelForCausalLM.from_pretrained( model_name, device_map="cpu", quantization_config=quantization_config, torch_dtype="auto" ) @@ -715,13 +715,17 @@ print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50, do_sample=Fal ``` | Name | Devices | Bits | Dtypes | Priority | Packing format | Requirements | |----------------------------------------------|--------------|---------|-----------|----------|-----------------|-----------------------------------| -| ipex | cpu/xpu | 4 | BF16/FP16 | 5 | gptq_zp+-1/awq | intel-extension-for-pytorch | +| ark | cpu | 2,4,8 | FP32/FP16/BF16 | 6 | gptq/gptq_zp+-1 | auto-round-lib | +| ark | cpu | 4 | FP32/FP16/BF16 | 6 | awq | auto-round-lib | +| ark | xpu | 4,8 | FP32/FP16/BF16 | 6 | gptq/gptq_zp+-1 | auto-round-lib | +| ark | xpu | 4 | FP32/FP16/BF16 | 6 | awq | auto-round-lib | +| ipex(To be deprecated) | cpu/xpu | 4 | BF16/FP16 | 5 | gptq_zp+-1/awq | intel-extension-for-pytorch | | marlin | cuda | 4,8 | BF16/FP16 | 6 | gptq/gptq_zp+-1 | gptqmodel | | exllamav2 or
gptqmodel:exllamav2 | cuda | 4 | BF16/FP16 | 5 | gptq/gptq_zp+-1 | gptqmodel | | exllamav2 or
gptq:exllamav2 | cuda | 4 | FP16 | 3 | gptq_zp+-1 | auto-gptq
transformers<5.0.0 | | gptq:cuda | cuda | 2,3,4,8 | FP16 | 1 | gptq_zp+-1 | auto-gptq
transformers<5.0.0 | | triton | xpu/cuda | 2,4,8 | BF16/FP16 | 2 | gptq/gptq_zp+-1 | auto-round | -| awq | cuda | 4 | FP16 | 5 | awq | auto-awq
transformers<4.57.0 | +| awq | cuda | 4 | FP16 | 5 | awq | auto-awq
transformers<4.57.0 | | gptqmodel:awq or
gptqmodel:awq_exllamav2 | cuda | 4 | BF16/FP16 | 6 | awq | gptqmodel | | gptqmodel:awq_marlin | cuda | 4,8 | FP16 | 5 | awq | gptqmodel | | gptqmodel:awq_gemm | cuda | 4 | FP16 | 3 | awq | gptqmodel | diff --git a/docs/step_by_step_CN.md b/docs/step_by_step_CN.md index b77ce44d04..299d8fc427 100644 --- a/docs/step_by_step_CN.md +++ b/docs/step_by_step_CN.md @@ -590,7 +590,7 @@ AutoRound 支持十余种推理后端，并会根据已安装的库自动选择 ​**请勿在推理过程中手动将量化后的模型迁移到其他设备**​（例如执行 `model.to('cpu')`），否则可能导致意外错误。 ### CPU -支持 2、4、8 bits 模型，其中 **4-bits 推理推荐搭配 intel-extension-for-pytorch（IPEX）** 使用，示例： +支持 2、4、8 bits 模型，推荐搭配 **auto-round-lib（ark）** 进行推理，示例： ```python from transformers import AutoModelForCausalLM, AutoTokenizer @@ -604,7 +604,7 @@ print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50, do_sample=Fal ``` ### 英特尔 GPU -**仅支持 4-bits 模型**，推荐搭配 **IPEX** 使用，示例： +支持 4、8 bits 模型，推荐搭配 **auto-round-lib（ark）** 进行推理，示例： ```python from transformers import AutoModelForCausalLM, AutoTokenizer @@ -652,7 +652,7 @@ print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50, do_sample=Fal AutoRound 会根据兼容性为每个层自动选择推理后端，默认优先级大致为：Marlin > ExLLaMAV2 > Triton。最终的选择会受到 **group size、bit width、packing format、hardware device** 等多种因素的影响。 默认选择的后端并非在所有设备上都是最优的，你可根据需求或硬件兼容性手动指定后端： -- CPU/英特尔 GPU：推荐`ipex` +- CPU/英特尔 GPU：推荐`ark`（需要 `auto-round-lib` 和 `torch>=2.8.0`） - CUDA：可指定`marlin`/`exllamav2`/`triton` **注意**：手动指定后端的话，可能要安装相关依赖。 @@ -662,7 +662,7 @@ AutoRound 会根据兼容性为每个层自动选择推理后端，默认优先 from transformers import AutoModelForCausalLM, AutoTokenizer, AutoRoundConfig model_name = "OPEA/Qwen2.5-1.5B-Instruct-int4-sym-inc" -quantization_config = AutoRoundConfig(backend="ipex") +quantization_config = AutoRoundConfig(backend="ark") model = AutoModelForCausalLM.from_pretrained( model_name, device_map="cpu", quantization_config=quantization_config, torch_dtype="auto" ) @@ -676,7 +676,11 @@ print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50, do_sample=Fal #### 各推理后端的详细支持说明 | 后端名称 | 支持设备 | 支持位宽 | 支持数据类型 | 优先级 | 打包格式 | 依赖库要求 | |-------------------------|----------------|----------------|--------------|--------|-----------------|--------------------------------| -| ipex | cpu/xpu | 4 | BF16/FP16 | 5 | gptq_zp+-1/awq | intel-extension-for-pytorch | +| ark | cpu | 2、4、8 | FP32/FP16/BF16 | 6 | gptq/gptq_zp+-1 | auto-round-lib | +| ark | cpu | 4 | FP32/FP16/BF16 | 6 | awq | auto-round-lib | +| ark | xpu | 4、8 | FP32/FP16/BF16 | 6 | gptq/gptq_zp+-1 | auto-round-lib | +| ark | xpu | 4 | FP32/FP16/BF16 | 6 | awq | auto-round-lib | +| ipex（即将废弃） | cpu/xpu | 4 | BF16/FP16 | 5 | gptq_zp+-1/awq | intel-extension-for-pytorch | | marlin | cuda | 4、8 | BF16/FP16 | 6 | gptq/gptq_zp+-1 | gptqmodel | | exllamav2/
gptqmodel:exllamav2 | cuda | 4 | BF16/FP16 | 5 | gptq | gptqmodel | | exllamav2/
gptq:exllamav2 | cuda | 4 | FP16 | 5 | gptq_zp+-1 | auto-gptq
transformers<5.0.0 | From cbf5757955bc406a891cee729659c7364ce8e951 Mon Sep 17 00:00:00 2001 From: Xin He Date: Tue, 24 Mar 2026 09:33:26 +0800 Subject: [PATCH 2/7] Update docs/step_by_step.md Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- docs/step_by_step.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/step_by_step.md b/docs/step_by_step.md index d7bfa2049c..19ffa19d07 100644 --- a/docs/step_by_step.md +++ b/docs/step_by_step.md @@ -715,10 +715,10 @@ print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50, do_sample=Fal ``` | Name | Devices | Bits | Dtypes | Priority | Packing format | Requirements | |----------------------------------------------|--------------|---------|-----------|----------|-----------------|-----------------------------------| -| ark | cpu | 2,4,8 | FP32/FP16/BF16 | 6 | gptq/gptq_zp+-1 | auto-round-lib | -| ark | cpu | 4 | FP32/FP16/BF16 | 6 | awq | auto-round-lib | -| ark | xpu | 4,8 | FP32/FP16/BF16 | 6 | gptq/gptq_zp+-1 | auto-round-lib | -| ark | xpu | 4 | FP32/FP16/BF16 | 6 | awq | auto-round-lib | +| ark | cpu | 2,4,8 | FP32/FP16/BF16 | 6 | gptq/gptq_zp+-1 | auto-round-lib
torch>=2.8.0 | +| ark | cpu | 4 | FP32/FP16/BF16 | 6 | awq | auto-round-lib
torch>=2.8.0 | +| ark | xpu | 4,8 | FP32/FP16/BF16 | 6 | gptq/gptq_zp+-1 | auto-round-lib
torch>=2.8.0 | +| ark | xpu | 4 | FP32/FP16/BF16 | 6 | awq | auto-round-lib
torch>=2.8.0 | | ipex(To be deprecated) | cpu/xpu | 4 | BF16/FP16 | 5 | gptq_zp+-1/awq | intel-extension-for-pytorch | | marlin | cuda | 4,8 | BF16/FP16 | 6 | gptq/gptq_zp+-1 | gptqmodel | | exllamav2 or
gptqmodel:exllamav2 | cuda | 4 | BF16/FP16 | 5 | gptq/gptq_zp+-1 | gptqmodel | From 027e57acafb8d46f61ddfb750f84d4b90b750f88 Mon Sep 17 00:00:00 2001 From: Xin He Date: Tue, 24 Mar 2026 09:33:38 +0800 Subject: [PATCH 3/7] Update docs/step_by_step.md Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- docs/step_by_step.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/step_by_step.md b/docs/step_by_step.md index 19ffa19d07..eae553051f 100644 --- a/docs/step_by_step.md +++ b/docs/step_by_step.md @@ -629,7 +629,7 @@ AutoRound automatically selects the best available backend based on the installe ### CPU -Supports 2, 4, and 8 bits. We recommend using auto-round-lib (ark) for inference. +Supports 2, 4, and 8 bits. We recommend using auto-round-lib (ark) for inference. When using the ark backend, ensure that your PyTorch version is >= 2.8.0. ```python from transformers import AutoModelForCausalLM, AutoTokenizer From 2d30693dcdf1a8f140858dd6f230fcda216ac3e6 Mon Sep 17 00:00:00 2001 From: Xin He Date: Tue, 24 Mar 2026 09:33:45 +0800 Subject: [PATCH 4/7] Update docs/step_by_step.md Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- docs/step_by_step.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/step_by_step.md b/docs/step_by_step.md index eae553051f..b8fa54992a 100644 --- a/docs/step_by_step.md +++ b/docs/step_by_step.md @@ -645,7 +645,7 @@ print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50, do_sample=Fal ### Intel GPU -Supports 4,8 bits. We recommend using auto-round-lib (ark) for inference. +Supports 4,8 bits. When using the ark backend, PyTorch (torch) >= 2.8.0 is required. We recommend using auto-round-lib (ark) for inference. ```python from transformers import AutoModelForCausalLM, AutoTokenizer From 24f5e8f9f6e2627b18cbf15d963213b8412c9cbf Mon Sep 17 00:00:00 2001 From: Xin He Date: Tue, 24 Mar 2026 09:33:52 +0800 Subject: [PATCH 5/7] Update docs/step_by_step_CN.md Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- docs/step_by_step_CN.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/step_by_step_CN.md b/docs/step_by_step_CN.md index 299d8fc427..d6452b9de3 100644 --- a/docs/step_by_step_CN.md +++ b/docs/step_by_step_CN.md @@ -590,7 +590,7 @@ AutoRound 支持十余种推理后端，并会根据已安装的库自动选择 ​**请勿在推理过程中手动将量化后的模型迁移到其他设备**​（例如执行 `model.to('cpu')`），否则可能导致意外错误。 ### CPU -支持 2、4、8 bits 模型，推荐搭配 **auto-round-lib（ark）** 进行推理，示例： +支持 2、4、8 bits 模型，推荐搭配 **auto-round-lib（ark）**（需 PyTorch>=2.8.0）进行推理，示例： ```python from transformers import AutoModelForCausalLM, AutoTokenizer From 526b711780b34aacd7a465b74bbed2586b325795 Mon Sep 17 00:00:00 2001 From: Xin He Date: Tue, 24 Mar 2026 09:34:00 +0800 Subject: [PATCH 6/7] Update docs/step_by_step_CN.md Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- docs/step_by_step_CN.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/step_by_step_CN.md b/docs/step_by_step_CN.md index d6452b9de3..040f6938d0 100644 --- a/docs/step_by_step_CN.md +++ b/docs/step_by_step_CN.md @@ -676,10 +676,10 @@ print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50, do_sample=Fal #### 各推理后端的详细支持说明 | 后端名称 | 支持设备 | 支持位宽 | 支持数据类型 | 优先级 | 打包格式 | 依赖库要求 | |-------------------------|----------------|----------------|--------------|--------|-----------------|--------------------------------| -| ark | cpu | 2、4、8 | FP32/FP16/BF16 | 6 | gptq/gptq_zp+-1 | auto-round-lib | -| ark | cpu | 4 | FP32/FP16/BF16 | 6 | awq | auto-round-lib | -| ark | xpu | 4、8 | FP32/FP16/BF16 | 6 | gptq/gptq_zp+-1 | auto-round-lib | -| ark | xpu | 4 | FP32/FP16/BF16 | 6 | awq | auto-round-lib | +| ark | cpu | 2、4、8 | FP32/FP16/BF16 | 6 | gptq/gptq_zp+-1 | auto-round-lib
torch>=2.8.0 | +| ark | cpu | 4 | FP32/FP16/BF16 | 6 | awq | auto-round-lib
torch>=2.8.0 | +| ark | xpu | 4、8 | FP32/FP16/BF16 | 6 | gptq/gptq_zp+-1 | auto-round-lib
torch>=2.8.0 | +| ark | xpu | 4 | FP32/FP16/BF16 | 6 | awq | auto-round-lib
torch>=2.8.0 | | ipex（即将废弃） | cpu/xpu | 4 | BF16/FP16 | 5 | gptq_zp+-1/awq | intel-extension-for-pytorch | | marlin | cuda | 4、8 | BF16/FP16 | 6 | gptq/gptq_zp+-1 | gptqmodel | | exllamav2/
gptqmodel:exllamav2 | cuda | 4 | BF16/FP16 | 5 | gptq | gptqmodel | From 998490def6002b73ce60efed845e7f8f49d405b5 Mon Sep 17 00:00:00 2001 From: Xin He Date: Tue, 24 Mar 2026 09:35:19 +0800 Subject: [PATCH 7/7] Update step_by_step_CN.md --- docs/step_by_step_CN.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/step_by_step_CN.md b/docs/step_by_step_CN.md index 040f6938d0..a3097ddcaa 100644 --- a/docs/step_by_step_CN.md +++ b/docs/step_by_step_CN.md @@ -604,7 +604,7 @@ print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50, do_sample=Fal ``` ### 英特尔 GPU -支持 4、8 bits 模型，推荐搭配 **auto-round-lib（ark）** 进行推理，示例： +支持 4、8 bits 模型，推荐搭配 **auto-round-lib（ark）**（需 PyTorch>=2.8.0）进行推理，示例： ```python from transformers import AutoModelForCausalLM, AutoTokenizer