[gptneox]
model_name = tiny-random-GPTNeoX
head_num = 4
size_per_head = 8
inter_size = 128
num_layer = 5
rotary_embedding = 8
vocab_size = 1024
start_id = 0
end_id = 0
use_gptj_residual = 0
weight_data_type = fp32