root
/
tabby
mirror of https://gitee.com/zhang_1334717033/tabby.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Fix use_gptj_residual in gptneox convert

add-more-languages
Meng Zhang 2023-03-20 20:05:20 +08:00
parent 8459a23a9c
commit 5df32ed11c
1 changed files with 4 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
converter/huggingface_gptneox_convert.py
View File

@ -192,17 +192,17 @@ def split_and_convert(args):
# Post-process biases if use_gptj_residual is True
if use_gptj_residual:
for layer_idx in range(hf_config["n_layer"]):
for layer_idx in range(hf_config["num_hidden_layers"]):
attn_bias = np.fromfile(
saved_dir + f"/model.layers.{layer_idx}.attention.dense.bias.bin",
dtype=np.float32,
dtype=np_weight_data_type,
)
mlp_bias = np.fromfile(
saved_dir + f"/model.layers.{layer_idx}.mlp.dense_4h_to_h.bias.bin",
dtype=np.float32,
dtype=np_weight_data_type,
)
(attn_bias + mlp_bias).tofile(
(attn_bias + mlp_bias).astype(np_weight_data_type).tofile(
saved_dir + f"/model.layers.{layer_idx}.mlp.attention.bias.sum.bin"
)