Add testdata for GPTNeoX

converter/huggingface_gptneox_convert.py

@@ -2,12 +2,13 @@ import argparse
 import configparser
 import multiprocessing
 import os
+import shutil
 import sys
 from pathlib import Path
 
 import numpy as np
 import torch
-from transformers import GPTNeoXForCausalLM  # 4.21.1
+from transformers import GPTNeoXForCausalLM
 
 
 def get_weight_data_type(data_type):
@@ -19,51 +20,7 @@ def get_weight_data_type(data_type):
         assert False, f"Invalid weight data type {data_type}"
 
 
-def prefix_prompt_convert(args, config, weight_data_type):
+def split_and_convert_process(saved_dir, factor, key, args, config, val):
-
-    saved_dir = args.saved_dir + "/%d-gpu/" % args.infer_gpu_num
-
-    prompt_in_file_list = args.prompt_in_file_list.split(",")
-
-    task_list = []
-    for idx, prompt_in_file in enumerate(prompt_in_file_list):
-        weights = torch.load(prompt_in_file)
-        task_name = prompt_in_file.split("/")[-1].split(".")[-3]
-
-        total_size = weights.nelement()
-        n_layers = config["num_hidden_layers"]
-        n_head = config["num_heads"]
-        size_per_head = config["hidden_size"] // n_head
-        prefix_prompt_len = total_size // (2 * n_layers * n_head * size_per_head)
-
-        task_list.append((task_name, prefix_prompt_len))
-        # GPT NeoX
-        weights = weights.view(
-            prefix_prompt_len, n_layers, 2, n_head, size_per_head
-        )  ## prefix_seq_len, num_layers, 2, num_heads, size_per_head
-        # weights=weights.view(prefix_prompt_len,28,2,16,256) ## prefix_seq_len, num_layers, 2, num_heads, size_per_head
-        weights = weights.permute(
-            1, 2, 3, 0, 4
-        )  ## num_layers, 2, num_heads, perfix_seq_len, size_per_head
-        local_head_num = n_head // args.infer_gpu_num
-        weights_split = torch.split(weights, local_head_num, dim=2)
-        for i in range(args.infer_gpu_num):
-            output_file_path = (
-                saved_dir
-                + "/model.prefix_prompt."
-                + task_name
-                + ".weight."
-                + str(i)
-                + ".bin"
-            )
-            weights_split[i].detach().cpu().numpy().astype(weight_data_type).tofile(
-                output_file_path
-            )
-
-    return task_list
-
-
-def split_and_convert_process(i, saved_dir, factor, key, args, config, val):
 
     if (
         key.find("input_layernorm.weight") != -1
@@ -75,11 +32,8 @@ def split_and_convert_process(i, saved_dir, factor, key, args, config, val):
         or key.find("final_layernorm.weight") != -1
         or key.find("final_layernorm.bias") != -1
     ):
-
+        saved_path = saved_dir + f"/model.{key}.bin"
-        # shared weights, only need to convert the weights of rank 0
+        val.tofile(saved_path)
-        if i == 0:
-            saved_path = saved_dir + "/model." + key + ".bin"
-            val.tofile(saved_path)
 
     elif (
         key.find("attention.dense.weight") != -1
@@ -87,7 +41,7 @@ def split_and_convert_process(i, saved_dir, factor, key, args, config, val):
     ):
         split_vals = np.split(val, factor, axis=0)
         for j in range(factor):
-            saved_path = saved_dir + "/model." + key + ".%d.bin" % (i * factor + j)
+            saved_path = saved_dir + f"/model.{key}.{j}.bin"
             split_vals[j].tofile(saved_path)
 
     elif (
@@ -97,7 +51,7 @@ def split_and_convert_process(i, saved_dir, factor, key, args, config, val):
 
         split_vals = np.split(val, factor, axis=-1)
         for j in range(factor):
-            saved_path = saved_dir + "/model." + key + ".%d.bin" % (i * factor + j)
+            saved_path = saved_dir + f"/model.{key}.{j}.bin"
             split_vals[j].tofile(saved_path)
 
     elif key.find("attention.query_key_value.bias") != -1:
@@ -109,7 +63,7 @@ def split_and_convert_process(i, saved_dir, factor, key, args, config, val):
         split_vals = np.split(val, factor, axis=-1)
 
         for j in range(factor):
-            saved_path = saved_dir + "/model." + key + ".%d.bin" % (i * factor + j)
+            saved_path = saved_dir + f"/model.{key}.{j}.bin"
             split_vals[j].tofile(saved_path)
 
     elif key.find("attention.query_key_value.weight") != -1:
@@ -125,7 +79,7 @@ def split_and_convert_process(i, saved_dir, factor, key, args, config, val):
         split_vals = np.split(val, factor, axis=-1)
 
         for j in range(factor):
-            saved_path = saved_dir + "/model." + key + ".%d.bin" % (i * factor + j)
+            saved_path = saved_dir + f"/model.{key}.{j}.bin"
             split_vals[j].tofile(saved_path)
 
     else:
@@ -149,45 +103,30 @@ def split_and_convert(args):
     # model = torch.load(ckpt_name)
     model = GPTNeoXForCausalLM.from_pretrained(args.in_file)
     hf_config = vars(model.config)
-    if "gpt_j_residual" not in hf_config:
-        hf_config["gpt_j_residual"] = 0
 
     np_weight_data_type = get_weight_data_type(args.weight_data_type)
 
-    task_list = []
-    if args.prompt_in_file_list is not None:
-        task_list = prefix_prompt_convert(args, hf_config, np_weight_data_type)
-
     try:
         model_name = args.model_name
+        n_heads = hf_config["num_attention_heads"]
+        head_size = hf_config["hidden_size"] // n_heads
+        rotary_dim = int(head_size * hf_config["rotary_pct"])
+        use_gptj_residual = int(hf_config["use_parallel_residual"])
+
         config = configparser.ConfigParser()
         config["gptneox"] = {}
         config["gptneox"]["model_name"] = model_name
-        config["gptneox"]["head_num"] = str(hf_config["num_attention_heads"])
+        config["gptneox"]["head_num"] = str(n_heads)
-        n_embd = hf_config["hidden_size"]
+        config["gptneox"]["size_per_head"] = str(head_size)
-        config["gptneox"]["size_per_head"] = str(
+        config["gptneox"]["inter_size"] = str(hf_config["intermediate_size"])
-            n_embd // hf_config["num_attention_heads"]
-        )
-        config["gptneox"]["inter_size"] = str(n_embd * 4)
         config["gptneox"]["num_layer"] = str(hf_config["num_hidden_layers"])
-        if "rotary_dim" in hf_config:
-            rotary_dim = hf_config["rotary_dim"]
-        else:
-            rotary_dim = n_embd // hf_config["num_attention_heads"]
         config["gptneox"]["rotary_embedding"] = str(rotary_dim)
         config["gptneox"]["vocab_size"] = str(hf_config["vocab_size"])
         config["gptneox"]["start_id"] = str(hf_config["bos_token_id"])
         config["gptneox"]["end_id"] = str(hf_config["eos_token_id"])
-        config["gptneox"]["use_gptj_residual"] = str(int(hf_config["gpt_j_residual"]))
+        config["gptneox"]["use_gptj_residual"] = str(use_gptj_residual)
         config["gptneox"]["weight_data_type"] = args.weight_data_type
 
-        if len(task_list) > 0:
-            config["gptneox"]["num_tasks"] = str(len(task_list))
-            config["gptneox"]["prompt_learning_type"] = str(2)
-            for idx, (task_name, prompt_length) in enumerate(task_list):
-                config[f"task_{idx}"] = {}
-                config[f"task_{idx}"]["task_name"] = task_name
-                config[f"task_{idx}"]["prompt_length"] = str(prompt_length)
         with open((Path(saved_dir) / f"config.ini").as_posix(), "w") as configfile:
             config.write(configfile)
     except Exception as e:
@@ -211,24 +150,19 @@ def split_and_convert(args):
     torch.multiprocessing.set_start_method("spawn")
     pool = multiprocessing.Pool(args.processes)
     for name, param in model.named_parameters():
+        array = param.detach().cpu().numpy().astype(np_weight_data_type)
+        # print("input shape", name, array.shape)
         if name.find("weight") == -1 and name.find("bias") == -1:
+            print("skipped", name)
             continue
         elif name == "gpt_neox.embed_in.weight":
-            param.detach().cpu().numpy().astype(np_weight_data_type).tofile(
+            array.tofile(saved_dir + "model.wte.bin")
-                saved_dir + "model.wte.bin"
-            )
         elif name == "gpt_neox.final_layer_norm.bias":
-            param.detach().cpu().numpy().astype(np_weight_data_type).tofile(
+            array.tofile(saved_dir + "model.final_layernorm.bias.bin")
-                saved_dir + "model.final_layernorm.bias.bin"
-            )
         elif name == "gpt_neox.final_layer_norm.weight":
-            param.detach().cpu().numpy().astype(np_weight_data_type).tofile(
+            array.tofile(saved_dir + "model.final_layernorm.weight.bin")
-                saved_dir + "model.final_layernorm.weight.bin"
-            )
         elif name == "embed_out.weight":
-            param.detach().cpu().numpy().astype(np_weight_data_type).tofile(
+            array.tofile(saved_dir + "model.lm_head.weight.bin")
-                saved_dir + "model.lm_head.weight.bin"
-            )
         else:
             processed = False
             for i in range(len(ft_model_name_pattern)):
@@ -238,17 +172,12 @@ def split_and_convert(args):
                         split_and_convert_process,
                         [
                             (
-                                0,
                                 saved_dir,
                                 factor,
                                 new_name,
                                 args,
                                 vars(model.config),
-                                param.detach()
+                                array.T,
-                                .cpu()
-                                .numpy()
-                                .astype(np_weight_data_type)
-                                .T,
                             )
                         ],
                     )
@@ -256,13 +185,13 @@ def split_and_convert(args):
                     break
 
             if not processed:
-                raise Exception("Unused layer", name)
+                print("Unused layer", name)
 
     pool.close()
     pool.join()
 
     # Post-process biases if use_gptj_residual is True
-    if hf_config["gpt_j_residual"]:
+    if use_gptj_residual:
         for layer_idx in range(hf_config["n_layer"]):
             attn_bias = np.fromfile(
                 saved_dir + f"/model.layers.{layer_idx}.attention.dense.bias.bin",
@@ -290,13 +219,6 @@ if __name__ == "__main__":
         help="file name of input checkpoint file",
         required=True,
     )
-    parser.add_argument(
-        "-prompt_in_file_list",
-        "-p_i_list",
-        type=str,
-        help="list of the prompt weight file path,"
-        "separate by (,). e.g. -prompt_in_file_list prefix_prompt.task0.weight,prefix_prompt.task1.weight",
-    )
     parser.add_argument(
         "-trained_gpu_num",
         "-t_g",
@@ -331,4 +253,5 @@ if __name__ == "__main__":
         print("{}: {}".format(key, vars(args)[key]))
     print("========================================")
 
+    shutil.rmtree(args.saved_dir, ignore_errors=True)
     split_and_convert(args)

converter/testdata/tiny-random-GPTNeoXModel/fastertransformer/1/1-gpu/config.ini

@@ -1,12 +0,0 @@
-[gptneox]
-model_name = tiny-random-GPTNeoX
-head_num = 4
-size_per_head = 8
-inter_size = 128
-num_layer = 5
-rotary_embedding = 8
-vocab_size = 1024
-start_id = 0
-end_id = 0
-use_gptj_residual = 0
-weight_data_type = fp32

converter/testdata/tiny-random-GPTNeoXModel/fastertransformer/1/1-gpu/model.final_layernorm.bias.bin

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:38723a2e5e8a17aa7950dc008209944e898f69a7bd10a23c839d341e935fd5ca
-size 128

converter/testdata/tiny-random-GPTNeoXModel/fastertransformer/1/1-gpu/model.final_layernorm.weight.bin

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:b638277a8690e175a9137feff1e43c067f9faf4e2f600caf468fb05b0403b717
-size 128

converter/testdata/tiny-random-GPTNeoXModel/fastertransformer/1/1-gpu/model.layers.0.attention.dense.bias.bin

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:38723a2e5e8a17aa7950dc008209944e898f69a7bd10a23c839d341e935fd5ca
-size 128

converter/testdata/tiny-random-GPTNeoXModel/fastertransformer/1/1-gpu/model.layers.0.attention.dense.weight.0.bin

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:75ec999d1d55bc4af21e7ee8101f7540ff53f73725fc332f175bac14fda1b83a
-size 4096

converter/testdata/tiny-random-GPTNeoXModel/fastertransformer/1/1-gpu/model.layers.0.attention.query_key_value.bias.0.bin

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:a1a4f5721c1c4610af7f71078f3a68c330536d679803b0e0507ee8dc10c5dfca
-size 384

converter/testdata/tiny-random-GPTNeoXModel/fastertransformer/1/1-gpu/model.layers.0.attention.query_key_value.weight.0.bin

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:8d0af572105f74f7711069438049a1b539af19b43e4d341fd314b5c67792ce28
-size 12288

converter/testdata/tiny-random-GPTNeoXModel/fastertransformer/1/1-gpu/model.layers.0.input_layernorm.bias.bin

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:38723a2e5e8a17aa7950dc008209944e898f69a7bd10a23c839d341e935fd5ca
-size 128

converter/testdata/tiny-random-GPTNeoXModel/fastertransformer/1/1-gpu/model.layers.0.input_layernorm.weight.bin

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:b638277a8690e175a9137feff1e43c067f9faf4e2f600caf468fb05b0403b717
-size 128

converter/testdata/tiny-random-GPTNeoXModel/fastertransformer/1/1-gpu/model.layers.0.mlp.dense_4h_to_h.bias.bin

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:38723a2e5e8a17aa7950dc008209944e898f69a7bd10a23c839d341e935fd5ca
-size 128

converter/testdata/tiny-random-GPTNeoXModel/fastertransformer/1/1-gpu/model.layers.0.mlp.dense_4h_to_h.weight.0.bin

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:aca4617a559ee69fe7c96a62087f3b18700da03d5ad974ab8c58c01d32a5a65e
-size 4736

converter/testdata/tiny-random-GPTNeoXModel/fastertransformer/1/1-gpu/model.layers.0.mlp.dense_h_to_4h.bias.0.bin

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:3b18c58c739716e76429634a61375c45b3b5cd470c22ab6d3e14cee23dd992e1
-size 148

converter/testdata/tiny-random-GPTNeoXModel/fastertransformer/1/1-gpu/model.layers.0.mlp.dense_h_to_4h.weight.0.bin

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:8c7eae417acdf83e12125252829446e277269e3aeff2543148576a6b267934d3
-size 4736

converter/testdata/tiny-random-GPTNeoXModel/fastertransformer/1/1-gpu/model.layers.0.post_attention_layernorm.bias.bin

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:38723a2e5e8a17aa7950dc008209944e898f69a7bd10a23c839d341e935fd5ca
-size 128

converter/testdata/tiny-random-GPTNeoXModel/fastertransformer/1/1-gpu/model.layers.0.post_attention_layernorm.weight.bin

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:b638277a8690e175a9137feff1e43c067f9faf4e2f600caf468fb05b0403b717
-size 128

converter/testdata/tiny-random-GPTNeoXModel/fastertransformer/1/1-gpu/model.layers.1.attention.dense.bias.bin

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:38723a2e5e8a17aa7950dc008209944e898f69a7bd10a23c839d341e935fd5ca
-size 128

converter/testdata/tiny-random-GPTNeoXModel/fastertransformer/1/1-gpu/model.layers.1.attention.dense.weight.0.bin

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:b9e123291a9e860163e8e6acd45f4e46ab7f65a3da84767d9c45541ff2e61a27
-size 4096

converter/testdata/tiny-random-GPTNeoXModel/fastertransformer/1/1-gpu/model.layers.1.attention.query_key_value.bias.0.bin

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:a1a4f5721c1c4610af7f71078f3a68c330536d679803b0e0507ee8dc10c5dfca
-size 384

converter/testdata/tiny-random-GPTNeoXModel/fastertransformer/1/1-gpu/model.layers.1.attention.query_key_value.weight.0.bin

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:3aa877cf3a9c6f51414a3773bf2036af613f999dff1d08966c84f5c0164be0bb
-size 12288

converter/testdata/tiny-random-GPTNeoXModel/fastertransformer/1/1-gpu/model.layers.1.input_layernorm.bias.bin

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:38723a2e5e8a17aa7950dc008209944e898f69a7bd10a23c839d341e935fd5ca
-size 128

converter/testdata/tiny-random-GPTNeoXModel/fastertransformer/1/1-gpu/model.layers.1.input_layernorm.weight.bin

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:b638277a8690e175a9137feff1e43c067f9faf4e2f600caf468fb05b0403b717
-size 128

converter/testdata/tiny-random-GPTNeoXModel/fastertransformer/1/1-gpu/model.layers.1.mlp.dense_4h_to_h.bias.bin

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:38723a2e5e8a17aa7950dc008209944e898f69a7bd10a23c839d341e935fd5ca
-size 128

converter/testdata/tiny-random-GPTNeoXModel/fastertransformer/1/1-gpu/model.layers.1.mlp.dense_4h_to_h.weight.0.bin

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:0568057def9087e03b35fdcc8ed89ad88bcf672b9a8d3562d816e95b4de8b10f
-size 4736

converter/testdata/tiny-random-GPTNeoXModel/fastertransformer/1/1-gpu/model.layers.1.mlp.dense_h_to_4h.bias.0.bin

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:3b18c58c739716e76429634a61375c45b3b5cd470c22ab6d3e14cee23dd992e1
-size 148

converter/testdata/tiny-random-GPTNeoXModel/fastertransformer/1/1-gpu/model.layers.1.mlp.dense_h_to_4h.weight.0.bin

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:38d439b2f0ce9e61bdc56aaea018bac52ba20f5eb6a9af39f11b920afa98a74d
-size 4736

converter/testdata/tiny-random-GPTNeoXModel/fastertransformer/1/1-gpu/model.layers.1.post_attention_layernorm.bias.bin

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:38723a2e5e8a17aa7950dc008209944e898f69a7bd10a23c839d341e935fd5ca
-size 128

converter/testdata/tiny-random-GPTNeoXModel/fastertransformer/1/1-gpu/model.layers.1.post_attention_layernorm.weight.bin

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:b638277a8690e175a9137feff1e43c067f9faf4e2f600caf468fb05b0403b717
-size 128

converter/testdata/tiny-random-GPTNeoXModel/fastertransformer/1/1-gpu/model.layers.2.attention.dense.bias.bin

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:38723a2e5e8a17aa7950dc008209944e898f69a7bd10a23c839d341e935fd5ca
-size 128

converter/testdata/tiny-random-GPTNeoXModel/fastertransformer/1/1-gpu/model.layers.2.attention.dense.weight.0.bin

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:439187a6e4a716b263062a0393e015944688d5ade8becc855b18b53799a1b9f4
-size 4096

converter/testdata/tiny-random-GPTNeoXModel/fastertransformer/1/1-gpu/model.layers.2.attention.query_key_value.bias.0.bin

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:a1a4f5721c1c4610af7f71078f3a68c330536d679803b0e0507ee8dc10c5dfca
-size 384

converter/testdata/tiny-random-GPTNeoXModel/fastertransformer/1/1-gpu/model.layers.2.attention.query_key_value.weight.0.bin

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:ff118539b509bc4d35473c1c4d2ecee86276a1b56f8b1c128fb343fbe7126b29
-size 12288

converter/testdata/tiny-random-GPTNeoXModel/fastertransformer/1/1-gpu/model.layers.2.input_layernorm.bias.bin

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:38723a2e5e8a17aa7950dc008209944e898f69a7bd10a23c839d341e935fd5ca
-size 128

converter/testdata/tiny-random-GPTNeoXModel/fastertransformer/1/1-gpu/model.layers.2.input_layernorm.weight.bin

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:b638277a8690e175a9137feff1e43c067f9faf4e2f600caf468fb05b0403b717
-size 128

converter/testdata/tiny-random-GPTNeoXModel/fastertransformer/1/1-gpu/model.layers.2.mlp.dense_4h_to_h.bias.bin

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:38723a2e5e8a17aa7950dc008209944e898f69a7bd10a23c839d341e935fd5ca
-size 128

converter/testdata/tiny-random-GPTNeoXModel/fastertransformer/1/1-gpu/model.layers.2.mlp.dense_4h_to_h.weight.0.bin

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:373d0d807fe8142dc107c53ba616cc7735e391ccc99947143e9490abf56ab807
-size 4736

converter/testdata/tiny-random-GPTNeoXModel/fastertransformer/1/1-gpu/model.layers.2.mlp.dense_h_to_4h.bias.0.bin

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:3b18c58c739716e76429634a61375c45b3b5cd470c22ab6d3e14cee23dd992e1
-size 148

converter/testdata/tiny-random-GPTNeoXModel/fastertransformer/1/1-gpu/model.layers.2.mlp.dense_h_to_4h.weight.0.bin

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:0954135db648aa7945baaa5f861b1b8012188dd199b9b8bc7c8343757ded04fc
-size 4736

converter/testdata/tiny-random-GPTNeoXModel/fastertransformer/1/1-gpu/model.layers.2.post_attention_layernorm.bias.bin

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:38723a2e5e8a17aa7950dc008209944e898f69a7bd10a23c839d341e935fd5ca
-size 128

converter/testdata/tiny-random-GPTNeoXModel/fastertransformer/1/1-gpu/model.layers.2.post_attention_layernorm.weight.bin

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:b638277a8690e175a9137feff1e43c067f9faf4e2f600caf468fb05b0403b717
-size 128

converter/testdata/tiny-random-GPTNeoXModel/fastertransformer/1/1-gpu/model.layers.3.attention.dense.bias.bin

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:38723a2e5e8a17aa7950dc008209944e898f69a7bd10a23c839d341e935fd5ca
-size 128

converter/testdata/tiny-random-GPTNeoXModel/fastertransformer/1/1-gpu/model.layers.3.attention.dense.weight.0.bin

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:8318d93f17e38918736e07b5b70f5148d4b28f8096190902477118700b0a762e
-size 4096

converter/testdata/tiny-random-GPTNeoXModel/fastertransformer/1/1-gpu/model.layers.3.attention.query_key_value.bias.0.bin

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:a1a4f5721c1c4610af7f71078f3a68c330536d679803b0e0507ee8dc10c5dfca
-size 384

converter/testdata/tiny-random-GPTNeoXModel/fastertransformer/1/1-gpu/model.layers.3.attention.query_key_value.weight.0.bin

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:1d5b8deb13e270dea2c77ea5a9f3d65d375bb008d8820128bcfec3c6efb4a454
-size 12288

converter/testdata/tiny-random-GPTNeoXModel/fastertransformer/1/1-gpu/model.layers.3.input_layernorm.bias.bin

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:38723a2e5e8a17aa7950dc008209944e898f69a7bd10a23c839d341e935fd5ca
-size 128

converter/testdata/tiny-random-GPTNeoXModel/fastertransformer/1/1-gpu/model.layers.3.input_layernorm.weight.bin

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:b638277a8690e175a9137feff1e43c067f9faf4e2f600caf468fb05b0403b717
-size 128

converter/testdata/tiny-random-GPTNeoXModel/fastertransformer/1/1-gpu/model.layers.3.mlp.dense_4h_to_h.bias.bin

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:38723a2e5e8a17aa7950dc008209944e898f69a7bd10a23c839d341e935fd5ca
-size 128

converter/testdata/tiny-random-GPTNeoXModel/fastertransformer/1/1-gpu/model.layers.3.mlp.dense_4h_to_h.weight.0.bin

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:d1a725384598dc3cf2889e659a0b136abd98def073ec1c632f36db3803a987a2
-size 4736

converter/testdata/tiny-random-GPTNeoXModel/fastertransformer/1/1-gpu/model.layers.3.mlp.dense_h_to_4h.bias.0.bin

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:3b18c58c739716e76429634a61375c45b3b5cd470c22ab6d3e14cee23dd992e1
-size 148

converter/testdata/tiny-random-GPTNeoXModel/fastertransformer/1/1-gpu/model.layers.3.mlp.dense_h_to_4h.weight.0.bin

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:3dccd82f3a3e32ea7619e48b972c77ff9def1b3fee58a2df924092a707e30152
-size 4736

converter/testdata/tiny-random-GPTNeoXModel/fastertransformer/1/1-gpu/model.layers.3.post_attention_layernorm.bias.bin

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:38723a2e5e8a17aa7950dc008209944e898f69a7bd10a23c839d341e935fd5ca
-size 128

converter/testdata/tiny-random-GPTNeoXModel/fastertransformer/1/1-gpu/model.layers.3.post_attention_layernorm.weight.bin

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:b638277a8690e175a9137feff1e43c067f9faf4e2f600caf468fb05b0403b717
-size 128

converter/testdata/tiny-random-GPTNeoXModel/fastertransformer/1/1-gpu/model.layers.4.attention.dense.bias.bin

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:38723a2e5e8a17aa7950dc008209944e898f69a7bd10a23c839d341e935fd5ca
-size 128

converter/testdata/tiny-random-GPTNeoXModel/fastertransformer/1/1-gpu/model.layers.4.attention.dense.weight.0.bin

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:43e013e9d9ddfc3f604562e7136e02ef97bcc40ecd42f94a236945b6e05e014a
-size 4096

converter/testdata/tiny-random-GPTNeoXModel/fastertransformer/1/1-gpu/model.layers.4.attention.query_key_value.bias.0.bin

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:a1a4f5721c1c4610af7f71078f3a68c330536d679803b0e0507ee8dc10c5dfca
-size 384

converter/testdata/tiny-random-GPTNeoXModel/fastertransformer/1/1-gpu/model.layers.4.attention.query_key_value.weight.0.bin

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:202256fec152abace918bd2da29d3eb9a9213920622756bee56f11162903f043
-size 12288

converter/testdata/tiny-random-GPTNeoXModel/fastertransformer/1/1-gpu/model.layers.4.input_layernorm.bias.bin

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:38723a2e5e8a17aa7950dc008209944e898f69a7bd10a23c839d341e935fd5ca
-size 128

converter/testdata/tiny-random-GPTNeoXModel/fastertransformer/1/1-gpu/model.layers.4.input_layernorm.weight.bin

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:b638277a8690e175a9137feff1e43c067f9faf4e2f600caf468fb05b0403b717
-size 128

converter/testdata/tiny-random-GPTNeoXModel/fastertransformer/1/1-gpu/model.layers.4.mlp.dense_4h_to_h.bias.bin

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:38723a2e5e8a17aa7950dc008209944e898f69a7bd10a23c839d341e935fd5ca
-size 128

converter/testdata/tiny-random-GPTNeoXModel/fastertransformer/1/1-gpu/model.layers.4.mlp.dense_4h_to_h.weight.0.bin

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:1715bf310c65c00750acacf5247186e3422f4c85fde6f056ba21b380a8097b80
-size 4736

converter/testdata/tiny-random-GPTNeoXModel/fastertransformer/1/1-gpu/model.layers.4.mlp.dense_h_to_4h.bias.0.bin

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:3b18c58c739716e76429634a61375c45b3b5cd470c22ab6d3e14cee23dd992e1
-size 148

converter/testdata/tiny-random-GPTNeoXModel/fastertransformer/1/1-gpu/model.layers.4.mlp.dense_h_to_4h.weight.0.bin

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:482a3307809bfb1eb7a34fc3780ed76e2dc4ba51536eef4d9d616d846db729e6
-size 4736

converter/testdata/tiny-random-GPTNeoXModel/fastertransformer/1/1-gpu/model.layers.4.post_attention_layernorm.bias.bin

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:38723a2e5e8a17aa7950dc008209944e898f69a7bd10a23c839d341e935fd5ca
-size 128

converter/testdata/tiny-random-GPTNeoXModel/fastertransformer/1/1-gpu/model.layers.4.post_attention_layernorm.weight.bin

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:b638277a8690e175a9137feff1e43c067f9faf4e2f600caf468fb05b0403b717
-size 128

converter/testdata/tiny-random-GPTNeoXModel/fastertransformer/1/1-gpu/model.lm_head.weight.bin

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:b2d2ae19f874eb46f4c94c8c58930a9df564bdb9205aedcc47f0daadc14ae9a5
-size 131072

converter/testdata/tiny-random-GPTNeoXModel/fastertransformer/1/1-gpu/model.wte.bin

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:51a808360ffa74213dcbfac40776866c50b1bc18fc3d6993e856ae2ffa92e0d0
-size 131072

converter/tests/gptneox/docker-compose.yaml

@@ -0,0 +1,17 @@
+version: '3.3'
+services:
+  triton:
+    image: ghcr.io/tabbyml/fastertransformer_backend:main
+    command: mpirun -n 1 --allow-run-as-root /opt/tritonserver/bin/tritonserver --model-repository=/model
+    shm_size: 1gb
+    ports:
+      - "8001:8001"
+    volumes:
+      - ./testdata:/model
+    deploy:
+      resources:
+        reservations:
+          devices:
+            - driver: nvidia
+              count: all
+              capabilities: [gpu]

converter/tests/gptneox/testdata/fastertransformer/1/1-gpu/config.ini

@@ -0,0 +1,12 @@
+[gptneox]
+model_name = gptneox
+head_num = 12
+size_per_head = 64
+inter_size = 3072
+num_layer = 12
+rotary_embedding = 64
+vocab_size = 50304
+start_id = 0
+end_id = 0
+use_gptj_residual = 0
+weight_data_type = fp32

converter/tests/gptneox/testdata/fastertransformer/1/1-gpu/model.final_layernorm.bias.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c8e8e6b531dabfeff987975cc6c937428139beb7f5699b8b938545716d8eeca7
+size 3072

converter/tests/gptneox/testdata/fastertransformer/1/1-gpu/model.final_layernorm.weight.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a20104b8a794c5dfa02c2847d25a2697ae60971ea296192dc88a9eaaa479f606
+size 3072

converter/tests/gptneox/testdata/fastertransformer/1/1-gpu/model.layers.0.attention.dense.bias.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3df9dbc92f1617b447db1aa855da24d07660a12c41b22dee002696813b3ebbba
+size 3072

converter/tests/gptneox/testdata/fastertransformer/1/1-gpu/model.layers.0.attention.dense.weight.0.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5eba16ee6fe2391cf2efcb7162e4662dd905a58cb8610ff32d07813e7575d6fc
+size 2359296

converter/tests/gptneox/testdata/fastertransformer/1/1-gpu/model.layers.0.attention.query_key_value.bias.0.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:93fb9545a9c7ba88f8927bb1929a3d1d74962d18909ee21225e6b9756c77c332
+size 9216

converter/tests/gptneox/testdata/fastertransformer/1/1-gpu/model.layers.0.attention.query_key_value.weight.0.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:35a3c434d0af69813352edd06e29e671e8e160de1221a90abe6f308cd6b624ab
+size 7077888

converter/tests/gptneox/testdata/fastertransformer/1/1-gpu/model.layers.0.input_layernorm.bias.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f324b52b78c5e245f0b68559ac1b40c7c54aa1d48005ca2d12887cb53dbbc9b2
+size 3072

converter/tests/gptneox/testdata/fastertransformer/1/1-gpu/model.layers.0.input_layernorm.weight.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1c632d0534e7e7e4eab3d843a0a1f0d0f3843aa9b1aefd6b706ad71cf3de08ed
+size 3072

converter/tests/gptneox/testdata/fastertransformer/1/1-gpu/model.layers.0.mlp.dense_4h_to_h.bias.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7e2b220335cce206b3928380f069259c01e382937189f1d86d40641bea319a27
+size 3072

converter/tests/gptneox/testdata/fastertransformer/1/1-gpu/model.layers.0.mlp.dense_4h_to_h.weight.0.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2d59fe9d26e5a6b5bf67a2d631bf012cb67e30a8aee34d5a8b5589a1a3b767d0
+size 9437184

converter/tests/gptneox/testdata/fastertransformer/1/1-gpu/model.layers.0.mlp.dense_h_to_4h.bias.0.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1fb0620b893fc6c19a0d99f7b0b19fcbc81262b7de48532a0cfd74a6c2ceb11e
+size 12288

converter/tests/gptneox/testdata/fastertransformer/1/1-gpu/model.layers.0.mlp.dense_h_to_4h.weight.0.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ea4b2ed74e487155f748f4ad057070db5a1ef9bbe18f155caf05f1889a7db0e4
+size 9437184

converter/tests/gptneox/testdata/fastertransformer/1/1-gpu/model.layers.0.post_attention_layernorm.bias.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0e949431800d44dc72725fb2a14c1b6257df33a087c89a5363504b2796f033ef
+size 3072

converter/tests/gptneox/testdata/fastertransformer/1/1-gpu/model.layers.0.post_attention_layernorm.weight.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e63fbed352f174592299da93e92c5652b45d4d439aea92867c30bc292d2d8891
+size 3072

converter/tests/gptneox/testdata/fastertransformer/1/1-gpu/model.layers.1.attention.dense.bias.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ca3de5c5d037abd4ba67b56fca3fd35e11f2d3355313684066188424579295f0
+size 3072

converter/tests/gptneox/testdata/fastertransformer/1/1-gpu/model.layers.1.attention.dense.weight.0.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3bfab993d3877ea0dcdbb35f1bfbe371cf03dee47e23a124e1a3004c9abde182
+size 2359296

converter/tests/gptneox/testdata/fastertransformer/1/1-gpu/model.layers.1.attention.query_key_value.bias.0.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ef2961811f9b09add00d1acf68bbb0d983b2a266415bcf4167c108cf0af2b162
+size 9216

converter/tests/gptneox/testdata/fastertransformer/1/1-gpu/model.layers.1.attention.query_key_value.weight.0.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:678d7bdfff7a580935c141a3c8d56121925ad5688ca4839150f7c38aa07b3816
+size 7077888

converter/tests/gptneox/testdata/fastertransformer/1/1-gpu/model.layers.1.input_layernorm.bias.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:22ee83ada70710e37e74558e042db460e1264b2373ad2272891f8681dd32992f
+size 3072

converter/tests/gptneox/testdata/fastertransformer/1/1-gpu/model.layers.1.input_layernorm.weight.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ea7f1c9586a26d8b4c7db214c5e036691d24a6aaa0700434e34fb89789e8413d
+size 3072

converter/tests/gptneox/testdata/fastertransformer/1/1-gpu/model.layers.1.mlp.dense_4h_to_h.bias.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ea6d9660941b2258411fa5a95bc2efae46f9b37b2c2677dbfd15fc4bfa97b7d8
+size 3072

converter/tests/gptneox/testdata/fastertransformer/1/1-gpu/model.layers.1.mlp.dense_4h_to_h.weight.0.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:df394855b5b83e6cef0b009b66d75bc6ad9536f63ea81c118821269ddb389811
+size 9437184

converter/tests/gptneox/testdata/fastertransformer/1/1-gpu/model.layers.1.mlp.dense_h_to_4h.bias.0.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:970ccc870e06e091619acffcd71eec44e61ea4e38490094850dea049a55ee4d4
+size 12288

converter/tests/gptneox/testdata/fastertransformer/1/1-gpu/model.layers.1.mlp.dense_h_to_4h.weight.0.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:669ab6458f06db05c0310bf04a4c6681b00faa5ad498d144552ddc8b0ebcad1c
+size 9437184

converter/tests/gptneox/testdata/fastertransformer/1/1-gpu/model.layers.1.post_attention_layernorm.bias.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d77f19d5997955ea3d639595ca6d0a1d94f16eb0a54e6ed9256d151e84a32160
+size 3072

converter/tests/gptneox/testdata/fastertransformer/1/1-gpu/model.layers.1.post_attention_layernorm.weight.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a208323db32e45ec21cb6fa92754862a2d8db9dc32a148ede730567e1b53ea6d
+size 3072

converter/tests/gptneox/testdata/fastertransformer/1/1-gpu/model.layers.10.attention.dense.bias.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9a991d95e2e3a0c716990227c3fca9617a3762963404bd1bbb1649d6031476d7
+size 3072

converter/tests/gptneox/testdata/fastertransformer/1/1-gpu/model.layers.10.attention.dense.weight.0.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0948b6eb35b6e388e7ad4cdf315abdf943960d7952dfebbebb4897023cc9a278
+size 2359296

converter/tests/gptneox/testdata/fastertransformer/1/1-gpu/model.layers.10.attention.query_key_value.bias.0.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4755cff515205a9072c5f00b4e592939e5359f7b535d447bc1f7da44830ce011
+size 9216

converter/tests/gptneox/testdata/fastertransformer/1/1-gpu/model.layers.10.attention.query_key_value.weight.0.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4b52cdfe8cd07b6f604cd3d848c7ab44efaa5a260441b7da057c42488992b3d6
+size 7077888

converter/tests/gptneox/testdata/fastertransformer/1/1-gpu/model.layers.10.input_layernorm.bias.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:21b95c5b59d5a99b5f833825f620ddde882036901fda433046d6047a2ad90b51
+size 3072

converter/tests/gptneox/testdata/fastertransformer/1/1-gpu/model.layers.10.input_layernorm.weight.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2e927894a15e7a284be51655cb46338d98d4ca77378d9f865dc00707ae62758b
+size 3072