fix: remove empty tokens for similarity calculation (#581)

2023-10-17 15:25:38 -07:00 · 2023-10-17 15:25:38 -07:00 · 981133d6c8
parent 23a105e4cb
commit 981133d6c8
1 changed files with 1 additions and 1 deletions
--- a/crates/tabby/src/serve/completions/prompt.rs
+++ b/crates/tabby/src/serve/completions/prompt.rs
@ -169,7 +169,7 @@ lazy_static! {
 fn tokenize_text(text: &str) -> Vec<&str> {
    TOKENIZER
        .split(text)
-        .filter(|s| *s != "AND" && *s != "OR" && *s != "NOT")
+        .filter(|s| *s != "AND" && *s != "OR" && *s != "NOT" && !s.is_empty())
        .collect()
 }