Fix tokenizer with \n\n (#15)

2023-03-26 19:47:32 +08:00 · 2023-03-26 19:47:32 +08:00 · d76ed403c3
parent ce666962e7
commit d76ed403c3
1 changed files with 5 additions and 0 deletions
--- a/tabby/server/triton.py
+++ b/tabby/server/triton.py
@ -117,6 +117,11 @@ def to_word_list_format(word_dict, tokenizer):
            item_flat_ids += ids
            item_offsets.append(len(ids))

+            if word == "\n\n":
+                ids = tokenizer.encode("\n") * 2
+                item_flat_ids += ids
+                item_offsets.append(len(ids))
+
        flat_ids.append(np.array(item_flat_ids))
        offsets.append(np.cumsum(np.array(item_offsets)))