From 1f2c4061a94b25d03f9dedd2e9e1ab14f4885da3 Mon Sep 17 00:00:00 2001 From: gaoziyuan Date: Mon, 18 Aug 2025 22:27:24 +0800 Subject: [PATCH 1/2] fix bias --- fastdeploy/model_executor/layers/linear.py | 5 ++++- .../model_executor/layers/quantization/block_wise_fp8.py | 2 +- .../model_executor/layers/quantization/tensor_wise_fp8.py | 2 +- fastdeploy/model_executor/layers/quantization/w4afp8.py | 2 +- fastdeploy/model_executor/layers/quantization/weight_only.py | 4 ++-- fastdeploy/model_executor/layers/quantization/wfp8afp8.py | 2 +- 6 files changed, 10 insertions(+), 7 deletions(-) diff --git a/fastdeploy/model_executor/layers/linear.py b/fastdeploy/model_executor/layers/linear.py index 54681124177..a6686581189 100644 --- a/fastdeploy/model_executor/layers/linear.py +++ b/fastdeploy/model_executor/layers/linear.py @@ -63,7 +63,7 @@ def process_loaded_weights(self, layer, weights) -> None: def apply(self, layer: nn.Layer, x: paddle.Tensor) -> paddle.Tensor: linear_out = paddle.matmul(x, layer.weight) - if layer.with_bias: + if layer.with_bias and layer.add_bias: linear_out = paddle.add(linear_out, layer.bias) return linear_out @@ -614,6 +614,7 @@ def load_state_dict(self, state_dict: dict): # bias if self.with_bias: + print("gaoziyuan test qkv bias") if self.bias_key in state_dict.keys(): bias_tensor = paddle.to_tensor(get_tensor(state_dict.pop(self.bias_key))) self.bias.set_value(bias_tensor) @@ -717,6 +718,8 @@ def forward_cuda(self, x: paddle.Tensor) -> paddle.Tensor: out = self.quant_method.apply(self, x) else: out = paddle.matmul(x, self.weight) + if self.with_bias and self.add_bias: + out = paddle.add(out, self.bias) if self.reduce_results and self.nranks > 1: tensor_model_parallel_all_reduce(out) diff --git a/fastdeploy/model_executor/layers/quantization/block_wise_fp8.py b/fastdeploy/model_executor/layers/quantization/block_wise_fp8.py index a003e1888e4..165b1647a38 100644 --- a/fastdeploy/model_executor/layers/quantization/block_wise_fp8.py +++ b/fastdeploy/model_executor/layers/quantization/block_wise_fp8.py @@ -133,6 +133,6 @@ def apply(self, layer, x): (layer.weight, layer.weight_scale), linear_out, ) - if layer.with_bias: + if layer.with_bias and layer.add_bias: linear_out = paddle.add(linear_out, layer.bias) return linear_out diff --git a/fastdeploy/model_executor/layers/quantization/tensor_wise_fp8.py b/fastdeploy/model_executor/layers/quantization/tensor_wise_fp8.py index 9576882ec9c..4f6f2e19683 100644 --- a/fastdeploy/model_executor/layers/quantization/tensor_wise_fp8.py +++ b/fastdeploy/model_executor/layers/quantization/tensor_wise_fp8.py @@ -131,7 +131,7 @@ def apply(self, layer, x): layer.weight, transpose_x=False, transpose_y=True, - bias=None, + bias=layer.bias if layer.add_bias and layer.with_bias else None, scale=self.total_scale, output_dtype="bfloat16", activation_type="identity", diff --git a/fastdeploy/model_executor/layers/quantization/w4afp8.py b/fastdeploy/model_executor/layers/quantization/w4afp8.py index 2c0afd3d4b9..804519c7d7e 100644 --- a/fastdeploy/model_executor/layers/quantization/w4afp8.py +++ b/fastdeploy/model_executor/layers/quantization/w4afp8.py @@ -94,7 +94,7 @@ def apply(self, layer, x): layer.weight, layer.weight_scale, zero_points=None, - bias=layer.bias if layer.add_bias else None, + bias=layer.bias if layer.add_bias and layer.with_bias else None, out_scale=self.quant_config.weight_scale_dict.get(layer.prefix + ".weight_scale") / ( self.quant_config.act_scale_dict.get(layer.prefix + ".activation_scale") diff --git a/fastdeploy/model_executor/layers/quantization/weight_only.py b/fastdeploy/model_executor/layers/quantization/weight_only.py index 4825faaf77e..4b9a406250d 100644 --- a/fastdeploy/model_executor/layers/quantization/weight_only.py +++ b/fastdeploy/model_executor/layers/quantization/weight_only.py @@ -210,7 +210,7 @@ def apply(self, layer, x): linear_out = weight_only_linear( x, weight=layer.weight, - bias=layer.bias if layer.add_bias else None, + bias=layer.bias if layer.add_bias and layer.with_bias else None, weight_scale=layer.weight_scale, weight_dtype=("int8" if self.quant_config.name() == "wint8" else "int4"), arch=80, @@ -219,7 +219,7 @@ def apply(self, layer, x): linear_out = weight_only_linear( x, weight=layer.weight, - bias=layer.bias if layer.add_bias else None, + bias=layer.bias if layer.add_bias and layer.with_bias else None, weight_scale=layer.weight_scale, weight_dtype=("int8" if self.quant_config.name() == "wint8" else "int4"), arch=self.quant_config.weight_only_linear_arch, diff --git a/fastdeploy/model_executor/layers/quantization/wfp8afp8.py b/fastdeploy/model_executor/layers/quantization/wfp8afp8.py index f868a9aabf4..95dc355a7d0 100644 --- a/fastdeploy/model_executor/layers/quantization/wfp8afp8.py +++ b/fastdeploy/model_executor/layers/quantization/wfp8afp8.py @@ -118,7 +118,7 @@ def apply(self, layer, x): a_scales, layer.weight_scale, out_type, - layer.bias, + layer.bias if layer.add_bias and layer.with_bias else None, ) else: raise NotImplementedError From c360bd0422b69ecd4d042d2bca66b60570efdead Mon Sep 17 00:00:00 2001 From: gaoziyuan Date: Mon, 18 Aug 2025 22:31:07 +0800 Subject: [PATCH 2/2] fix --- fastdeploy/model_executor/layers/linear.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/fastdeploy/model_executor/layers/linear.py b/fastdeploy/model_executor/layers/linear.py index a6686581189..c56259e374b 100644 --- a/fastdeploy/model_executor/layers/linear.py +++ b/fastdeploy/model_executor/layers/linear.py @@ -614,7 +614,6 @@ def load_state_dict(self, state_dict: dict): # bias if self.with_bias: - print("gaoziyuan test qkv bias") if self.bias_key in state_dict.keys(): bias_tensor = paddle.to_tensor(get_tensor(state_dict.pop(self.bias_key))) self.bias.set_value(bias_tensor) @@ -651,7 +650,7 @@ def __init__( input_size: int = None, output_size: int = None, with_bias: bool = False, - add_bias: bool = False, + add_bias: bool = True, reduce_results: bool = True, skip_quant: bool = False, ):