diff --git a/auto_round/inference/backend.py b/auto_round/inference/backend.py
index 9cf5fe9610..a858f55970 100644
--- a/auto_round/inference/backend.py
+++ b/auto_round/inference/backend.py
@@ -509,7 +509,7 @@ def fp8_static_scheme_checker(
priority=6,
checkers=[ark_feature_checker],
alias=["ark"],
- compute_dtype=["float32", "float16"],
+ compute_dtype=["float32", "float16", "bfloat16"],
data_type=["int"],
act_bits=WOQ_DEFAULT_ACT_BITS,
requirements=["torch>=2.8.0", "auto-round-lib"],
@@ -524,7 +524,7 @@ def fp8_static_scheme_checker(
priority=6,
checkers=[ark_feature_checker],
alias=["ark"],
- compute_dtype=["float32", "float16"],
+ compute_dtype=["float32", "float16", "bfloat16"],
data_type=["int"],
act_bits=WOQ_DEFAULT_ACT_BITS,
requirements=["torch>=2.8.0", "auto-round-lib"],
@@ -539,7 +539,7 @@ def fp8_static_scheme_checker(
priority=6,
checkers=[ark_feature_checker],
alias=["ark"],
- compute_dtype=["float32", "float16"],
+ compute_dtype=["float32", "float16", "bfloat16"],
data_type=["int"],
act_bits=WOQ_DEFAULT_ACT_BITS,
requirements=["torch>=2.8.0", "auto-round-lib"],
@@ -554,7 +554,7 @@ def fp8_static_scheme_checker(
priority=6,
checkers=[ark_feature_checker],
alias=["ark"],
- compute_dtype=["float32", "float16"],
+ compute_dtype=["float32", "float16", "bfloat16"],
data_type=["int"],
act_bits=WOQ_DEFAULT_ACT_BITS,
requirements=["torch>=2.8.0", "auto-round-lib"],
@@ -569,7 +569,7 @@ def fp8_static_scheme_checker(
priority=6,
checkers=[ark_feature_checker],
alias=["ark"],
- compute_dtype=["float32", "float16"],
+ compute_dtype=["float32", "float16", "bfloat16"],
data_type=["int"],
act_bits=WOQ_DEFAULT_ACT_BITS,
requirements=["torch>=2.8.0", "auto-round-lib"],
@@ -584,7 +584,7 @@ def fp8_static_scheme_checker(
priority=6,
checkers=[ark_feature_checker],
alias=["ark"],
- compute_dtype=["float32", "float16"],
+ compute_dtype=["float32", "float16", "bfloat16"],
data_type=["int"],
act_bits=WOQ_DEFAULT_ACT_BITS,
requirements=["torch>=2.8.0", "auto-round-lib"],
diff --git a/auto_round/inference/convert_model.py b/auto_round/inference/convert_model.py
index f55ca7c502..afea71b806 100644
--- a/auto_round/inference/convert_model.py
+++ b/auto_round/inference/convert_model.py
@@ -528,6 +528,11 @@ def post_init(model: torch.nn.Module, used_backends: list[str]) -> None:
need_gptqmodel_init = True
elif backend.startswith(("ipex", "auto_round_kernel")):
need_ipex_init = True
+ if backend.startswith("ipex"):
+ logger.warning_once(
+ f"Backend '{backend}' is deprecated and will be removed in a future release. "
+ "Please use the 'ark' backend instead (requires auto-round-lib and torch>=2.8.0)."
+ )
# AutoGPTQ post-init
if need_autogptq_init:
diff --git a/docs/step_by_step.md b/docs/step_by_step.md
index 8d322c6eb2..b8fa54992a 100644
--- a/docs/step_by_step.md
+++ b/docs/step_by_step.md
@@ -629,7 +629,7 @@ AutoRound automatically selects the best available backend based on the installe
### CPU
-Supports 2, 4, and 8 bits. We recommend using intel-extension-for-pytorch (IPEX) for 4 bits inference.
+Supports 2, 4, and 8 bits. We recommend using auto-round-lib (ark) for inference. When using the ark backend, ensure that your PyTorch version is >= 2.8.0.
```python
from transformers import AutoModelForCausalLM, AutoTokenizer
@@ -645,7 +645,7 @@ print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50, do_sample=Fal
### Intel GPU
-Supports 4 bits only. We recommend using intel-extension-for-pytorch (IPEX) for inference.
+Supports 4,8 bits. When using the ark backend, PyTorch (torch) >= 2.8.0 is required. We recommend using auto-round-lib (ark) for inference.
```python
from transformers import AutoModelForCausalLM, AutoTokenizer
@@ -698,13 +698,13 @@ print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50, do_sample=Fal
AutoRound automatically selects the backend for each layer based on compatibility. In general, the priority order is Marlin > ExLLaMAV2 > Triton, but the final choice depends on factors such as group size, bit width, packing format, hardware device, and other implementation details.
The backend may not always be the most suitable for certain devices.
-You can specify your preferred backend such as "ipex" for CPU and Intel GPU, "marlin/exllamav2/triton" for CUDA, according to your needs or hardware compatibility. Please note that additional corresponding libraries may be required.
+You can specify your preferred backend such as "ark" for CPU and Intel GPU, "marlin/exllamav2/triton" for CUDA, according to your needs or hardware compatibility. Please note that additional corresponding libraries may be required.
```python
from transformers import AutoModelForCausalLM, AutoTokenizer, AutoRoundConfig
model_name = "OPEA/Qwen2.5-1.5B-Instruct-int4-sym-inc"
-quantization_config = AutoRoundConfig(backend="ipex")
+quantization_config = AutoRoundConfig(backend="ark")
model = AutoModelForCausalLM.from_pretrained(
model_name, device_map="cpu", quantization_config=quantization_config, torch_dtype="auto"
)
@@ -715,13 +715,17 @@ print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50, do_sample=Fal
```
| Name | Devices | Bits | Dtypes | Priority | Packing format | Requirements |
|----------------------------------------------|--------------|---------|-----------|----------|-----------------|-----------------------------------|
-| ipex | cpu/xpu | 4 | BF16/FP16 | 5 | gptq_zp+-1/awq | intel-extension-for-pytorch |
+| ark | cpu | 2,4,8 | FP32/FP16/BF16 | 6 | gptq/gptq_zp+-1 | auto-round-lib
torch>=2.8.0 |
+| ark | cpu | 4 | FP32/FP16/BF16 | 6 | awq | auto-round-lib
torch>=2.8.0 |
+| ark | xpu | 4,8 | FP32/FP16/BF16 | 6 | gptq/gptq_zp+-1 | auto-round-lib
torch>=2.8.0 |
+| ark | xpu | 4 | FP32/FP16/BF16 | 6 | awq | auto-round-lib
torch>=2.8.0 |
+| ipex(To be deprecated) | cpu/xpu | 4 | BF16/FP16 | 5 | gptq_zp+-1/awq | intel-extension-for-pytorch |
| marlin | cuda | 4,8 | BF16/FP16 | 6 | gptq/gptq_zp+-1 | gptqmodel |
| exllamav2 or
gptqmodel:exllamav2 | cuda | 4 | BF16/FP16 | 5 | gptq/gptq_zp+-1 | gptqmodel |
| exllamav2 or
gptq:exllamav2 | cuda | 4 | FP16 | 3 | gptq_zp+-1 | auto-gptq
transformers<5.0.0 |
| gptq:cuda | cuda | 2,3,4,8 | FP16 | 1 | gptq_zp+-1 | auto-gptq
transformers<5.0.0 |
| triton | xpu/cuda | 2,4,8 | BF16/FP16 | 2 | gptq/gptq_zp+-1 | auto-round |
-| awq | cuda | 4 | FP16 | 5 | awq | auto-awq
transformers<4.57.0 |
+| awq | cuda | 4 | FP16 | 5 | awq | auto-awq
transformers<4.57.0 |
| gptqmodel:awq or
gptqmodel:awq_exllamav2 | cuda | 4 | BF16/FP16 | 6 | awq | gptqmodel |
| gptqmodel:awq_marlin | cuda | 4,8 | FP16 | 5 | awq | gptqmodel |
| gptqmodel:awq_gemm | cuda | 4 | FP16 | 3 | awq | gptqmodel |
diff --git a/docs/step_by_step_CN.md b/docs/step_by_step_CN.md
index b77ce44d04..a3097ddcaa 100644
--- a/docs/step_by_step_CN.md
+++ b/docs/step_by_step_CN.md
@@ -590,7 +590,7 @@ AutoRound 支持十余种推理后端，并会根据已安装的库自动选择
**请勿在推理过程中手动将量化后的模型迁移到其他设备**（例如执行 `model.to('cpu')`），否则可能导致意外错误。
### CPU
-支持 2、4、8 bits 模型，其中 **4-bits 推理推荐搭配 intel-extension-for-pytorch（IPEX）** 使用，示例：
+支持 2、4、8 bits 模型，推荐搭配 **auto-round-lib（ark）**（需 PyTorch>=2.8.0）进行推理，示例：
```python
from transformers import AutoModelForCausalLM, AutoTokenizer
@@ -604,7 +604,7 @@ print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50, do_sample=Fal
```
### 英特尔 GPU
-**仅支持 4-bits 模型**，推荐搭配 **IPEX** 使用，示例：
+支持 4、8 bits 模型，推荐搭配 **auto-round-lib（ark）**（需 PyTorch>=2.8.0）进行推理，示例：
```python
from transformers import AutoModelForCausalLM, AutoTokenizer
@@ -652,7 +652,7 @@ print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50, do_sample=Fal
AutoRound 会根据兼容性为每个层自动选择推理后端，默认优先级大致为：Marlin > ExLLaMAV2 > Triton。最终的选择会受到 **group size、bit width、packing format、hardware device** 等多种因素的影响。
默认选择的后端并非在所有设备上都是最优的，你可根据需求或硬件兼容性手动指定后端：
-- CPU/英特尔 GPU：推荐`ipex`
+- CPU/英特尔 GPU：推荐`ark`（需要 `auto-round-lib` 和 `torch>=2.8.0`）
- CUDA：可指定`marlin`/`exllamav2`/`triton`
**注意**：手动指定后端的话，可能要安装相关依赖。
@@ -662,7 +662,7 @@ AutoRound 会根据兼容性为每个层自动选择推理后端，默认优先
from transformers import AutoModelForCausalLM, AutoTokenizer, AutoRoundConfig
model_name = "OPEA/Qwen2.5-1.5B-Instruct-int4-sym-inc"
-quantization_config = AutoRoundConfig(backend="ipex")
+quantization_config = AutoRoundConfig(backend="ark")
model = AutoModelForCausalLM.from_pretrained(
model_name, device_map="cpu", quantization_config=quantization_config, torch_dtype="auto"
)
@@ -676,7 +676,11 @@ print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50, do_sample=Fal
#### 各推理后端的详细支持说明
| 后端名称 | 支持设备 | 支持位宽 | 支持数据类型 | 优先级 | 打包格式 | 依赖库要求 |
|-------------------------|----------------|----------------|--------------|--------|-----------------|--------------------------------|
-| ipex | cpu/xpu | 4 | BF16/FP16 | 5 | gptq_zp+-1/awq | intel-extension-for-pytorch |
+| ark | cpu | 2、4、8 | FP32/FP16/BF16 | 6 | gptq/gptq_zp+-1 | auto-round-lib
torch>=2.8.0 |
+| ark | cpu | 4 | FP32/FP16/BF16 | 6 | awq | auto-round-lib
torch>=2.8.0 |
+| ark | xpu | 4、8 | FP32/FP16/BF16 | 6 | gptq/gptq_zp+-1 | auto-round-lib
torch>=2.8.0 |
+| ark | xpu | 4 | FP32/FP16/BF16 | 6 | awq | auto-round-lib
torch>=2.8.0 |
+| ipex（即将废弃） | cpu/xpu | 4 | BF16/FP16 | 5 | gptq_zp+-1/awq | intel-extension-for-pytorch |
| marlin | cuda | 4、8 | BF16/FP16 | 6 | gptq/gptq_zp+-1 | gptqmodel |
| exllamav2/
gptqmodel:exllamav2 | cuda | 4 | BF16/FP16 | 5 | gptq | gptqmodel |
| exllamav2/
gptq:exllamav2 | cuda | 4 | FP16 | 5 | gptq_zp+-1 | auto-gptq
transformers<5.0.0 |