From 9ca1f7e5f1850d2264e9b997f9544f5e82147d46 Mon Sep 17 00:00:00 2001
From: Meng Zhang <meng@tabbyml.com>
Date: Sat, 24 Jun 2023 18:15:52 -0700
Subject: [PATCH] =?UTF-8?q?fix:=20add=20additional=20whitespace=20to=20mat?=
 =?UTF-8?q?ch=20tokens=20that=20combining=20space=20and=20li=E2=80=A6=20(#?=
 =?UTF-8?q?270)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* fix: add additional whitespace to match tokens that combining space and line break

* fix lint
---
 crates/tabby-scheduler/src/lib.rs             |  1 -
 .../tabby/src/serve/completions/languages.rs  | 39 +++++++++++++++++--
 2 files changed, 35 insertions(+), 5 deletions(-)

diff --git a/crates/tabby-scheduler/src/lib.rs b/crates/tabby-scheduler/src/lib.rs
index df60758..423392f 100644
--- a/crates/tabby-scheduler/src/lib.rs
+++ b/crates/tabby-scheduler/src/lib.rs
@@ -23,7 +23,6 @@ pub async fn scheduler(now: bool) -> Result<()> {
         let ret = dataset::create_dataset(&config);
         if let Err(err) = ret {
             error!("Failed to build dataset, err: '{}'", err);
-            return;
         }
     };
 
diff --git a/crates/tabby/src/serve/completions/languages.rs b/crates/tabby/src/serve/completions/languages.rs
index b9a5713..eed3325 100644
--- a/crates/tabby/src/serve/completions/languages.rs
+++ b/crates/tabby/src/serve/completions/languages.rs
@@ -3,18 +3,37 @@ use std::collections::HashMap;
 use lazy_static::lazy_static;
 
 lazy_static! {
-    static ref DEFAULT: Vec<&'static str> = vec!("\n\n");
+    static ref DEFAULT: Vec<&'static str> = vec![
+        "\n\n",
+        "\n\n  ",
+        "\n\n    ",
+        "\n\n      ",
+        "\n\n        ",
+        "\n\n          ",
+        "\n\n            ",
+        "\n\n              ",
+        "\n\n",
+        "\n\n\t",
+        "\n\n\t\t",
+        "\n\n\t\t\t",
+        "\n\n\t\t\t\t",
+        "\n\n\t\t\t\t\t",
+        "\n\n\t\t\t\t\t\t",
+        "\n\n\t\t\t\t\t\t\t",
+    ];
     static ref LANGUAGES: HashMap<&'static str, Vec<&'static str>> = {
         let mut map = HashMap::new();
-        map.insert("python", vec!["\n\n", "\ndef", "\n#", "\nfrom", "\nclass"]);
+        map.insert(
+            "python",
+            vec!["\ndef", "\n#", "\nfrom", "\nclass"].with_default(),
+        );
         map.insert(
             "javascript",
-            vec!["\n\n", "\nfunction", "\n//", "\nimport", "\nclass"],
+            vec!["\nfunction", "\n//", "\nimport", "\nclass"],
         );
         map.insert(
             "typescript",
             vec![
-                "\n\n",
                 "\nfunction",
                 "\n//",
                 "\nimport",
@@ -27,6 +46,18 @@ lazy_static! {
     };
 }
 
+trait WithDefault {
+    fn with_default(self) -> Self;
+}
+
+impl WithDefault for Vec<&'static str> {
+    fn with_default(mut self) -> Self {
+        let mut x = DEFAULT.clone();
+        self.append(&mut x);
+        self
+    }
+}
+
 pub fn get_stop_words(language: &str) -> &'static Vec<&'static str> {
     LANGUAGES.get(language).unwrap_or(&DEFAULT)
 }