From 5df32ed11c51c6d5dae3e0edc16a9929d76bc51d Mon Sep 17 00:00:00 2001 From: Meng Zhang Date: Mon, 20 Mar 2023 20:05:20 +0800 Subject: [PATCH] Fix use_gptj_residual in gptneox convert --- converter/huggingface_gptneox_convert.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/converter/huggingface_gptneox_convert.py b/converter/huggingface_gptneox_convert.py index 7852bf1..6e6c8d8 100644 --- a/converter/huggingface_gptneox_convert.py +++ b/converter/huggingface_gptneox_convert.py @@ -192,17 +192,17 @@ def split_and_convert(args): # Post-process biases if use_gptj_residual is True if use_gptj_residual: - for layer_idx in range(hf_config["n_layer"]): + for layer_idx in range(hf_config["num_hidden_layers"]): attn_bias = np.fromfile( saved_dir + f"/model.layers.{layer_idx}.attention.dense.bias.bin", - dtype=np.float32, + dtype=np_weight_data_type, ) mlp_bias = np.fromfile( saved_dir + f"/model.layers.{layer_idx}.mlp.dense_4h_to_h.bias.bin", - dtype=np.float32, + dtype=np_weight_data_type, ) - (attn_bias + mlp_bias).tofile( + (attn_bias + mlp_bias).astype(np_weight_data_type).tofile( saved_dir + f"/model.layers.{layer_idx}.mlp.attention.bias.sum.bin" )