From 981133d6c80cfd2a0b5eb0fe3010476eb6a4704b Mon Sep 17 00:00:00 2001
From: Meng Zhang <meng@tabbyml.com>
Date: Tue, 17 Oct 2023 15:25:38 -0700
Subject: [PATCH] fix: remove empty tokens for similarity calculation (#581)

---
 crates/tabby/src/serve/completions/prompt.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/crates/tabby/src/serve/completions/prompt.rs b/crates/tabby/src/serve/completions/prompt.rs
index e9ef202..4bb2e42 100644
--- a/crates/tabby/src/serve/completions/prompt.rs
+++ b/crates/tabby/src/serve/completions/prompt.rs
@@ -169,7 +169,7 @@ lazy_static! {
 fn tokenize_text(text: &str) -> Vec<&str> {
     TOKENIZER
         .split(text)
-        .filter(|s| *s != "AND" && *s != "OR" && *s != "NOT")
+        .filter(|s| *s != "AND" && *s != "OR" && *s != "NOT" && !s.is_empty())
         .collect()
 }