root
/
tabby
mirror of https://gitee.com/zhang_1334717033/tabby.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

fix: remove empty tokens for similarity calculation (#581)

add-dagster-data-pipeline
Meng Zhang 2023-10-17 15:25:38 -07:00 committed by GitHub
parent 23a105e4cb
commit 981133d6c8
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 1 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
crates/tabby/src/serve/completions/prompt.rs
View File

@ -169,7 +169,7 @@ lazy_static! {
fn tokenize_text(text: &str) -> Vec<&str> { fn tokenize_text(text: &str) -> Vec<&str> {
TOKENIZER TOKENIZER
.split(text) .split(text)
.filter(|s| *s != "AND" && *s != "OR" && *s != "NOT") .filter(|s| *s != "AND" && *s != "OR" && *s != "NOT" && !s.is_empty())
.collect() .collect()
} }