feat(agent): add experimental postprocess limit scope by syntax. (#758)

* feat(agent): add tree-sitter parser.

* feat(agent): make parser updating tree cache optional.

* feat(agent): add experimental limit scopy by syntax.

* test(agent): update unit test for limitScopeBySyntax.

.github/workflows/intellij-test.yml

@@ -18,6 +18,8 @@ jobs:
     steps:
       - name: Checkout
         uses: actions/checkout@v3
+        with:
+          lfs: true
       - name: Setup JDK
         uses: actions/setup-java@v3
         with:

.github/workflows/tabby-agent-test.yml

@@ -22,6 +22,8 @@ jobs:
     steps:
       - name: Checkout
         uses: actions/checkout@v3
+        with:
+          lfs: true
       - name: Setup Node.js
         uses: actions/setup-node@v3
         with:

.github/workflows/vscode-test.yml

@@ -24,6 +24,8 @@ jobs:
     steps:
       - name: Checkout
         uses: actions/checkout@v3
+        with:
+          lfs: true
       - name: Setup Node.js
         uses: actions/setup-node@v3
         with:

clients/tabby-agent/.gitattributes

@@ -0,0 +1 @@
+*.wasm filter=lfs diff=lfs merge=lfs -text

clients/tabby-agent/package.json

@@ -23,11 +23,11 @@
     "@types/node": "^18.12.0",
     "chai": "^4.3.7",
     "dedent": "^0.7.0",
+    "esbuild-plugin-copy": "^2.1.1",
     "esbuild-plugin-polyfill-node": "^0.3.0",
     "mocha": "^10.2.0",
     "openapi-typescript": "^6.6.1",
     "prettier": "^3.0.0",
-    "rimraf": "^5.0.1",
     "ts-node": "^10.9.1",
     "tsup": "^7.1.0",
     "typescript": "^5.0.3"
@@ -47,6 +47,7 @@
     "rotating-file-stream": "^3.1.0",
     "stats-logscale": "^1.0.7",
     "toml": "^3.0.0",
-    "uuid": "^9.0.0"
+    "uuid": "^9.0.0",
+    "web-tree-sitter": "^0.20.8"
   }
 }

clients/tabby-agent/src/AgentConfig.ts

@@ -21,11 +21,15 @@ export type AgentConfig = {
     timeout: number;
   };
   postprocess: {
-    limitScopeByIndentation: {
+    limitScope: {
-      // When completion is continuing the current line, limit the scope to:
+      // Prefer to use syntax parser than indentation
-      // false(default): the line scope, meaning use the next indent level as the limit.
+      experimentalSyntax: boolean;
-      // true: the block scope, meaning use the current indent level as the limit.
+      indentation: {
-      experimentalKeepBlockScopeWhenCompletingLine: boolean;
+        // When completion is continuing the current line, limit the scope to:
+        // false(default): the line scope, meaning use the next indent level as the limit.
+        // true: the block scope, meaning use the current indent level as the limit.
+        experimentalKeepBlockScopeWhenCompletingLine: boolean;
+      };
     };
   };
   logs: {
@@ -66,8 +70,11 @@ export const defaultAgentConfig: AgentConfig = {
     timeout: 4000, // ms
   },
   postprocess: {
-    limitScopeByIndentation: {
+    limitScope: {
-      experimentalKeepBlockScopeWhenCompletingLine: false,
+      experimentalSyntax: false,
+      indentation: {
+        experimentalKeepBlockScopeWhenCompletingLine: false,
+      },
     },
   },
   logs: {

clients/tabby-agent/src/TabbyAgent.ts

@@ -529,7 +529,7 @@ export class TabbyAgent extends EventEmitter implements Agent {
             throw options.signal.reason;
           }
           // Build cache
-          this.completionCache.buildCache(context, completionResponse);
+          this.completionCache.buildCache(context, JSON.parse(JSON.stringify(completionResponse)));
         }
       }
       // Postprocess (post-cache)

clients/tabby-agent/src/postprocess/index.ts

@@ -4,7 +4,7 @@ import { applyFilter } from "./base";
 import { removeRepetitiveBlocks } from "./removeRepetitiveBlocks";
 import { removeRepetitiveLines } from "./removeRepetitiveLines";
 import { removeLineEndsWithRepetition } from "./removeLineEndsWithRepetition";
-import { limitScopeByIndentation } from "./limitScopeByIndentation";
+import { limitScope } from "./limitScope";
 import { trimSpace } from "./trimSpace";
 import { dropDuplicated } from "./dropDuplicated";
 import { dropBlank } from "./dropBlank";
@@ -30,7 +30,7 @@ export async function postCacheProcess(
   return Promise.resolve(response)
     .then(applyFilter(removeRepetitiveBlocks(context), context))
     .then(applyFilter(removeRepetitiveLines(context), context))
-    .then(applyFilter(limitScopeByIndentation(context, config["limitScopeByIndentation"]), context))
+    .then(applyFilter(limitScope(context, config["limitScope"]), context))
     .then(applyFilter(dropDuplicated(context), context))
     .then(applyFilter(trimSpace(context), context))
     .then(applyFilter(dropBlank(), context));

clients/tabby-agent/src/postprocess/limitScope.ts

@@ -0,0 +1,24 @@
+import { CompletionContext } from "../Agent";
+import { AgentConfig } from "../AgentConfig";
+import { isBrowser } from "../env";
+import { PostprocessFilter } from "./base";
+import { limitScopeByIndentation } from "./limitScopeByIndentation";
+import { limitScopeBySyntax, supportedLanguages } from "./limitScopeBySyntax";
+
+export function limitScope(
+  context: CompletionContext,
+  config: AgentConfig["postprocess"]["limitScope"],
+): PostprocessFilter {
+  return isBrowser
+    ? (input) => {
+        // syntax parser is not supported in browser yet
+        return limitScopeByIndentation(context, config["indentation"])(input);
+      }
+    : (input) => {
+        if (config.experimentalSyntax && supportedLanguages.indexOf(context.language) >= 0) {
+          return limitScopeBySyntax(context)(input);
+        } else {
+          return limitScopeByIndentation(context, config["indentation"])(input);
+        }
+      };
+}

clients/tabby-agent/src/postprocess/limitScopeByIndentation.ts

@@ -18,7 +18,7 @@ function processContext(
   lines: string[],
   prefixLines: string[],
   suffixLines: string[],
-  config: AgentConfig["postprocess"]["limitScopeByIndentation"],
+  config: AgentConfig["postprocess"]["limitScope"]["indentation"],
 ): { indentLevelLimit: number; allowClosingLine: (closingLine: string) => boolean } {
   let allowClosingLine = false;
   let result = { indentLevelLimit: 0, allowClosingLine: (closingLine: string) => allowClosingLine };
@@ -103,14 +103,14 @@ function processContext(
 
 export function limitScopeByIndentation(
   context: CompletionContext,
-  config: AgentConfig["postprocess"]["limitScopeByIndentation"],
+  config: AgentConfig["postprocess"]["limitScope"]["indentation"],
 ): PostprocessFilter {
   return (input) => {
-    const { prefix, suffix, prefixLines, suffixLines } = context;
+    const { prefixLines, suffixLines } = context;
     const inputLines = splitLines(input);
     if (context.mode === "fill-in-line") {
       if (inputLines.length > 1) {
-        logger.debug({ input, prefix, suffix }, "Drop content with multiple lines");
+        logger.debug({ inputLines, prefixLines, suffixLines }, "Drop content with multiple lines");
         return null;
       }
     }
@@ -139,7 +139,10 @@ export function limitScopeByIndentation(
       }
     }
     if (index < inputLines.length) {
-      logger.debug({ input, prefix, suffix, scopeEndAt: index }, "Remove content out of scope");
+      logger.debug(
+        { inputLines, prefixLines, suffixLines, scopeLineCount: index },
+        "Remove content out of indent scope",
+      );
       return inputLines.slice(0, index).join("").trimEnd();
     }
     return input;

clients/tabby-agent/src/postprocess/limitScopeBySyntax.test.ts

@@ -0,0 +1,409 @@
+import { expect } from "chai";
+import { documentContext, inline } from "./testUtils";
+import { limitScopeBySyntax } from "./limitScopeBySyntax";
+
+describe("postprocess", () => {
+  describe("limitScopeBySyntax javascript", () => {
+    it("should limit scope at function_declaration.", async () => {
+      const context = {
+        ...documentContext`
+        function findMax(arr) {║}
+        `,
+        language: "javascript",
+      };
+      const completion = inline`
+                               ├
+          let max = arr[0];
+          for (let i = 1; i < arr.length; i++) {
+            if (arr[i] > max) {
+              max = arr[i];
+            }
+          }
+          return max;
+        }
+        console.log(findMax([1, 2, 3, 4, 5]));┤
+      `;
+      const expected = inline`
+                               ├
+          let max = arr[0];
+          for (let i = 1; i < arr.length; i++) {
+            if (arr[i] > max) {
+              max = arr[i];
+            }
+          }
+          return max;
+        }┤
+      `;
+      expect(await limitScopeBySyntax(context)(completion)).to.eq(expected);
+    });
+
+    it("should limit scope at function_declaration", async () => {
+      const context = {
+        ...documentContext`
+        function findMax(arr) {
+          let max = arr[0];║
+        }
+        `,
+        language: "javascript",
+      };
+      const completion = inline`
+                           ├
+          for (let i = 1; i < arr.length; i++) {
+            if (arr[i] > max) {
+              max = arr[i];
+            }
+          }
+          return max;
+        }┤
+      `;
+      expect(await limitScopeBySyntax(context)(completion)).to.eq(completion);
+    });
+
+    it("should limit scope at function_declaration", async () => {
+      const context = {
+        ...documentContext`
+        function findMax(arr) {
+          let max = arr[0];
+          for (let i = 1; i < arr.length; i++) {
+            if (arr[i] > max) {
+              max = arr[i];
+            }
+          }║
+        }
+        `,
+        language: "javascript",
+      };
+      const completion = inline`
+           ├
+          return max;
+        }┤
+      `;
+      expect(await limitScopeBySyntax(context)(completion)).to.eq(completion);
+    });
+
+    it("should limit scope at for_statement.", async () => {
+      const context = {
+        ...documentContext`
+        function findMax(arr) {
+          let max = arr[0];
+          for║
+        }
+        `,
+        language: "javascript",
+      };
+      const completion = inline`
+             ├ (let i = 1; i < arr.length; i++) {
+            if (arr[i] > max) {
+              max = arr[i];
+            }
+          }
+          return max;
+        }
+        console.log(findMax([1, 2, 3, 4, 5]));┤
+      `;
+      const expected = inline`
+             ├ (let i = 1; i < arr.length; i++) {
+            if (arr[i] > max) {
+              max = arr[i];
+            }
+          }┤
+        ┴┴
+      `;
+      expect(await limitScopeBySyntax(context)(completion)).to.eq(expected);
+    });
+
+    it("should limit scope at current node if no parent scope found.", async () => {
+      const context = {
+        ...documentContext`
+        let a =║
+        `,
+        language: "javascript",
+      };
+      const completion = inline`
+               ├ 1;
+        let b = 2;┤
+      `;
+      const expected = inline`
+               ├ 1;┤
+      `;
+      expect(await limitScopeBySyntax(context)(completion)).to.eq(expected);
+    });
+
+    it("should handle the bad case of limitScopeByIndentation", async () => {
+      const context = {
+        ...documentContext`
+        function sortWords(input) {
+          const output = input.trim()
+            .split("\n")
+            .map((line) => line.split(" "))
+            ║
+        }
+        `,
+        language: "javascript",
+      };
+      const completion = inline`
+            ├.flat()
+            .sort()
+            .join(" ");
+          console.log(output);
+          return output;
+        }
+        sortWord("world hello");┤
+      `;
+      const expected = inline`
+            ├.flat()
+            .sort()
+            .join(" ");
+          console.log(output);
+          return output;
+        };┤
+      `;
+      expect(await limitScopeBySyntax(context)(completion)).not.to.eq(expected);
+    });
+  });
+
+  describe("limitScopeBySyntax python", () => {
+    it("should limit scope at function_definition.", async () => {
+      const context = {
+        ...documentContext`
+        def find_min(arr):║
+        `,
+        language: "python",
+      };
+      const completion = inline`
+                          ├
+          min = arr[0]
+          for i in range(1, len(arr)):
+            if arr[i] < min:
+              min = arr[i]
+          return min
+        print(find_min([1, 2, 3, 4, 5]))┤
+      `;
+      const expected = inline`
+                          ├
+          min = arr[0]
+          for i in range(1, len(arr)):
+            if arr[i] < min:
+              min = arr[i]
+          return min┤
+        ┴┴
+      `;
+      expect(await limitScopeBySyntax(context)(completion)).to.eq(expected);
+    });
+
+    it("should limit scope at function_definition.", async () => {
+      const context = {
+        ...documentContext`
+        def find_min(arr):
+          min = arr[0]║
+        `,
+        language: "python",
+      };
+      const completion = inline`
+                      ├
+          for i in range(1, len(arr)):
+            if arr[i] < min:
+              min = arr[i]
+          return min
+        print(find_min([1, 2, 3, 4, 5]))┤
+      `;
+      const expected = inline`
+                      ├
+          for i in range(1, len(arr)):
+            if arr[i] < min:
+              min = arr[i]
+          return min┤
+        ┴┴
+      `;
+      expect(await limitScopeBySyntax(context)(completion)).to.eq(expected);
+    });
+
+    it("should limit scope at function_definition.", async () => {
+      const context = {
+        ...documentContext`
+        def find_min(arr):
+          min = arr[0]
+          for i in range(1, len(arr)):
+            if arr[i] < min:
+              min = arr[i]║
+        `,
+        language: "python",
+      };
+      const completion = inline`
+                          ├
+          return min┤
+      `;
+      expect(await limitScopeBySyntax(context)(completion)).to.eq(completion);
+    });
+
+    it("should limit scope at for_statement.", async () => {
+      const context = {
+        ...documentContext`
+        def find_min(arr):
+          max = arr[0]
+          for║
+        `,
+        language: "python",
+      };
+      const completion = inline`
+             ├ i in range(1, len(arr)):
+            if arr[i] < min:
+              min = arr[i]
+          return min
+        ┤
+      `;
+      const expected = inline`
+             ├ i in range(1, len(arr)):
+            if arr[i] < min:
+              min = arr[i]┤
+        ┴┴┴┴
+      `;
+      expect(await limitScopeBySyntax(context)(completion)).to.eq(expected);
+    });
+
+    it("should handle the bad case of limitScopeByIndentation", async () => {
+      const context = {
+        ...documentContext`
+        def findMax(arr):
+          ║
+        `,
+        language: "python",
+      };
+      const completion = inline`
+          ├max = arr[0]
+          for i in range(1, len(arr)):
+            if arr[i] > max:
+              max = arr[i]
+          return max
+        findMax([1, 2, 3, 4, 5])┤
+      `;
+      const expected = inline`
+          ├max = arr[0]
+          for i in range(1, len(arr)):
+            if arr[i] > max:
+              max = arr[i]
+          return max┤
+      `;
+      expect(await limitScopeBySyntax(context)(completion)).not.to.eq(expected);
+    });
+  });
+
+  describe("limitScopeBySyntax go", () => {
+    it("should limit scope at function_declaration.", async () => {
+      const context = {
+        ...documentContext`
+        func findMin(arr []int) int {║}
+        `,
+        language: "go",
+      };
+      const completion = inline`
+                                     ├
+          min := math.MaxInt64
+          for _, v := range arr {
+            if v < min {
+              min = v
+            }
+          }
+          return min
+        }
+        
+        func main() {
+          arr := []int{5, 2, 9, 8, 1, 3}
+          fmt.Println(findMin(arr)) // Output: 1
+        }┤
+      `;
+      const expected = inline`
+                                     ├
+          min := math.MaxInt64
+          for _, v := range arr {
+            if v < min {
+              min = v
+            }
+          }
+          return min
+        }┤
+      `;
+      expect(await limitScopeBySyntax(context)(completion)).to.eq(expected);
+    });
+
+    it("should limit scope at for_statement.", async () => {
+      const context = {
+        ...documentContext`
+        func findMin(arr []int) int {
+          min := math.MaxInt64
+          for║
+        `,
+        language: "go",
+      };
+      const completion = inline`
+             ├ _, v := range arr {
+            if v < min {
+              min = v
+            }
+          }
+          return min
+        }┤
+      `;
+      const expected = inline`
+             ├ _, v := range arr {
+            if v < min {
+              min = v
+            }
+          }┤
+        ┴┴
+      `;
+      expect(await limitScopeBySyntax(context)(completion)).to.eq(expected);
+    });
+  });
+
+  describe("limitScopeBySyntax rust", () => {
+    it("should limit scope at function_item.", async () => {
+      const context = {
+        ...documentContext`
+        fn find_min(arr: &[i32]) -> i32 {║}
+        `,
+        language: "rust",
+      };
+      const completion = inline`
+                                         ├
+          *arr.iter().min().unwrap()
+        }
+        fn main() {
+          let arr = vec![5, 2, 9, 8, 1, 3];
+          println!("{}", find_min(&arr)); // Output: 1
+        }┤
+      `;
+      const expected = inline`
+                                         ├
+          *arr.iter().min().unwrap()
+        }┤
+      `;
+      expect(await limitScopeBySyntax(context)(completion)).to.eq(expected);
+    });
+  });
+
+  describe("limitScopeBySyntax ruby", () => {
+    it("should limit scope at for.", async () => {
+      const context = {
+        ...documentContext`
+        def fibonacci(n)║
+        `,
+        language: "ruby",
+      };
+      const completion = inline`
+                        ├
+          return n if n <= 1
+          fibonacci(n - 1) + fibonacci(n - 2)
+        end
+        puts fibonacci(10)┤
+      `;
+      const expected = inline`
+                        ├
+          return n if n <= 1
+          fibonacci(n - 1) + fibonacci(n - 2)
+        end┤
+      `;
+      expect(await limitScopeBySyntax(context)(completion)).to.eq(expected);
+    });
+  });
+});

clients/tabby-agent/src/postprocess/limitScopeBySyntax.ts

@@ -0,0 +1,86 @@
+import type TreeSitterParser from "web-tree-sitter";
+import { getParser, languagesConfigs } from "../syntax/parser";
+import { typeList } from "../syntax/typeList";
+import { CompletionContext } from "../Agent";
+import { PostprocessFilter, logger } from "./base";
+
+export const supportedLanguages = Object.keys(languagesConfigs);
+
+function findLineBegin(text: string, position: number): number {
+  let lastNonBlankCharPos = position - 1;
+  while (lastNonBlankCharPos >= 0 && text[lastNonBlankCharPos].match(/\s/)) {
+    lastNonBlankCharPos--;
+  }
+  if (lastNonBlankCharPos < 0) {
+    return 0;
+  }
+  const lineBegin = text.lastIndexOf("\n", lastNonBlankCharPos);
+  if (lineBegin < 0) {
+    return 0;
+  }
+  const line = text.slice(lineBegin + 1, position);
+  const indentation = line.search(/\S/);
+  return lineBegin + 1 + indentation;
+}
+
+function findLineEnd(text: string, position: number): number {
+  let firstNonBlankCharPos = position;
+  while (firstNonBlankCharPos < text.length && text[firstNonBlankCharPos].match(/\s/)) {
+    firstNonBlankCharPos++;
+  }
+  if (firstNonBlankCharPos >= text.length) {
+    return text.length;
+  }
+  const lineEnd = text.indexOf("\n", firstNonBlankCharPos);
+  if (lineEnd < 0) {
+    return text.length;
+  }
+  return lineEnd;
+}
+
+function findScope(node: TreeSitterParser.SyntaxNode, typeList: string[][]): TreeSitterParser.SyntaxNode {
+  for (const types of typeList) {
+    let scope = node;
+    while (scope) {
+      if (types.indexOf(scope.type) >= 0) {
+        return scope;
+      }
+      scope = scope.parent;
+    }
+  }
+  return node;
+}
+
+export function limitScopeBySyntax(context: CompletionContext): PostprocessFilter {
+  return async (input) => {
+    const { position, text, language, prefix, suffix } = context;
+    if (supportedLanguages.indexOf(language) < 0) {
+      return input;
+    }
+    const languageConfig = languagesConfigs[language];
+    const parser = await getParser(languageConfig);
+
+    const updatedText = prefix + input + suffix;
+    const updatedTree = parser.parse(updatedText);
+    const lineBegin = findLineBegin(updatedText, position);
+    const lineEnd = findLineEnd(updatedText, position);
+    const scope = findScope(updatedTree.rootNode.namedDescendantForIndex(lineBegin, lineEnd), typeList[languageConfig]);
+
+    if (scope.endIndex < position + input.length) {
+      logger.debug(
+        {
+          languageConfig,
+          text,
+          updatedText,
+          position,
+          lineBegin,
+          lineEnd,
+          scope: { type: scope.type, start: scope.startIndex, end: scope.endIndex },
+        },
+        "Remove content out of syntax scope",
+      );
+      return input.slice(0, scope.endIndex - position);
+    }
+    return input;
+  };
+}

clients/tabby-agent/src/postprocess/testUtils.ts

@@ -1,12 +1,13 @@
 import dedent from "dedent";
+import { v4 as uuid } from "uuid";
 import { CompletionContext } from "../CompletionContext";
 
 // `║` is the cursor position
 export function documentContext(strings): CompletionContext {
   const doc = dedent(strings);
   return new CompletionContext({
-    filepath: null,
+    filepath: uuid(),
-    language: null,
+    language: "",
     text: doc.replace(/║/, ""),
     position: doc.indexOf("║"),
   });

clients/tabby-agent/src/syntax/parser.ts

@@ -0,0 +1,45 @@
+import TreeSitterParser from "web-tree-sitter";
+import { isTest } from "../env";
+
+// https://code.visualstudio.com/docs/languages/identifiers
+export const languagesConfigs: Record<string, string> = {
+  javascript: "tsx",
+  typescript: "tsx",
+  javascriptreact: "tsx",
+  typescriptreact: "tsx",
+  python: "python",
+  go: "go",
+  rust: "rust",
+  ruby: "ruby",
+};
+
+var treeSitterInitialized = false;
+
+async function createParser(languageConfig: string): Promise<TreeSitterParser> {
+  if (!treeSitterInitialized) {
+    await TreeSitterParser.init({
+      locateFile(scriptName: string, scriptDirectory: string) {
+        const paths = isTest ? [scriptDirectory, scriptName] : [scriptDirectory, "wasm", scriptName];
+        return require("path").join(...paths);
+      },
+    });
+    treeSitterInitialized = true;
+  }
+  const parser = new TreeSitterParser();
+  const langWasmPaths = isTest
+    ? [process.cwd(), "wasm", `tree-sitter-${languageConfig}.wasm`]
+    : [__dirname, "wasm", `tree-sitter-${languageConfig}.wasm`];
+  parser.setLanguage(await TreeSitterParser.Language.load(require("path").join(...langWasmPaths)));
+  return parser;
+}
+
+const parsers = new Map<string, TreeSitterParser>();
+
+export async function getParser(languageConfig: string): Promise<TreeSitterParser> {
+  let parser = parsers.get(languageConfig);
+  if (!parser) {
+    parser = await createParser(languageConfig);
+    parsers.set(languageConfig, parser);
+  }
+  return parser;
+}

clients/tabby-agent/src/syntax/typeList.ts

@@ -0,0 +1,116 @@
+// https://github.com/tree-sitter/tree-sitter-typescript/blob/master/src/node-types.json
+export const typeList = {
+  tsx: [
+    [
+      "jsx_element",
+      "jsx_self_closing_element",
+
+      // exclude sentence level nodes for now
+      // "expression_statement",
+      // "lexical_declaration",
+
+      "for_statement",
+      "for_in_statement",
+      "if_statement",
+      "while_statement",
+      "do_statement",
+      "switch_statement",
+      "try_statement",
+      "with_statement",
+      "labeled_statement",
+
+      "class_declaration",
+      "abstract_class_declaration",
+      "interface_declaration",
+      "enum_declaration",
+      "type_alias_declaration",
+      "function_declaration",
+      "generator_function_declaration",
+      "ambient_declaration",
+
+      "method_definition",
+
+      "import_statement",
+      "export_statement",
+      "module",
+    ],
+  ],
+
+  // https://github.com/tree-sitter/tree-sitter-python/blob/master/src/node-types.json
+  python: [
+    [
+      "for_statement",
+      "if_statement",
+      "while_statement",
+      "match_statement",
+      "try_statement",
+      "with_statement",
+
+      "function_definition",
+      "decorated_definition",
+      "class_definition",
+
+      "import_statement",
+      "import_from_statement",
+    ],
+  ],
+
+  // https://github.com/tree-sitter/tree-sitter-go/blob/master/src/node-types.json
+  go: [
+    [
+      "for_statement",
+      "if_statement",
+      "expression_switch_statement",
+      "type_switch_statement",
+      "select_statement",
+      "labeled_statement",
+
+      "function_declaration",
+      "method_declaration",
+      "type_declaration",
+
+      "import_declaration",
+      "package_clause",
+    ],
+  ],
+
+  // https://github.com/tree-sitter/tree-sitter-rust/blob/master/src/node-types.json
+  rust: [
+    [
+      "for_expression",
+      "if_expression",
+      "while_expression",
+      "loop_expression",
+      "match_expression",
+      "try_expression",
+
+      "function_item",
+      "type_item",
+      "enum_item",
+      "struct_item",
+      "union_item",
+      "trait_item",
+      "impl_item",
+
+      "use_declaration",
+    ],
+  ],
+
+  // https://github.com/tree-sitter/tree-sitter-ruby/blob/master/src/node-types.json
+  ruby: [
+    [
+      "for",
+      "if",
+      "unless",
+      "while",
+      "until",
+      "case",
+
+      "class",
+      "singleton_class",
+      "method",
+      "singleton_method",
+      "module",
+    ],
+  ],
+};

clients/tabby-agent/tests/golden.test.ts

@@ -81,7 +81,10 @@ describe("agent golden test", () => {
       timeout: 4000,
     },
     postprocess: {
-      limitScopeByIndentation: { experimentalKeepBlockScopeWhenCompletingLine: false },
+      limitScope: {
+        experimentalSyntax: false,
+        indentation: { experimentalKeepBlockScopeWhenCompletingLine: false },
+      },
     },
     logs: { level: "debug" },
     anonymousUsageTracking: { disable: true },
@@ -111,7 +114,7 @@ describe("agent golden test", () => {
       absolutePath: path.join(__dirname, "golden", file),
     };
   });
-  goldenFiles.forEach((goldenFile, index) => {
+  goldenFiles.forEach((goldenFile) => {
     it(goldenFile.path, async () => {
       const test = await createGoldenTest(goldenFile.absolutePath);
       requestId++;
@@ -130,7 +133,7 @@ describe("agent golden test", () => {
       absolutePath: path.join(__dirname, "bad_cases", file),
     };
   });
-  badCasesFiles.forEach((goldenFile, index) => {
+  badCasesFiles.forEach((goldenFile) => {
     it(goldenFile.path, async () => {
       const test = await createGoldenTest(goldenFile.absolutePath);
       requestId++;
@@ -143,6 +146,35 @@ describe("agent golden test", () => {
     });
   });
 
+  it("updateConfig experimental", async () => {
+    requestId++;
+    const updateConfigRequest = [
+      requestId,
+      {
+        func: "updateConfig",
+        args: ["postprocess.limitScope.experimentalSyntax", true],
+      },
+    ];
+    agent.stdin.write(JSON.stringify(updateConfigRequest) + "\n");
+    await waitForResponse(requestId);
+    const expectedConfig = { ...config };
+    expectedConfig.postprocess.limitScope.experimentalSyntax = true;
+    expect(output.shift()).to.deep.equal([0, { event: "configUpdated", config: expectedConfig }]);
+    expect(output.shift()).to.deep.equal([requestId, true]);
+  });
+  badCasesFiles.forEach((goldenFile) => {
+    it("experimental: " + goldenFile.path, async () => {
+      const test = await createGoldenTest(goldenFile.absolutePath);
+      requestId++;
+      const request = [requestId, { func: "provideCompletions", args: [test.request] }];
+      agent.stdin.write(JSON.stringify(request) + "\n");
+      await waitForResponse(requestId);
+      const response = output.shift();
+      expect(response[0]).to.equal(requestId);
+      expect(response[1].choices).to.deep.equal(test.expected.choices);
+    });
+  });
+
   after(() => {
     agent.kill();
   });

clients/tabby-agent/tsup.config.ts

@@ -1,4 +1,5 @@
 import { defineConfig } from "tsup";
+import { copy } from "esbuild-plugin-copy";
 import { polyfillNode } from "esbuild-plugin-polyfill-node";
 import { dependencies } from "./package.json";
 
@@ -16,6 +17,7 @@ export default async () => [
     name: "node-cjs",
     entry: ["src/index.ts"],
     platform: "node",
+    target: "node18",
     format: ["cjs"],
     sourcemap: true,
     esbuildOptions(options) {
@@ -71,10 +73,21 @@ export default async () => [
     name: "cli",
     entry: ["src/cli.ts"],
     platform: "node",
+    target: "node18",
     noExternal: Object.keys(dependencies),
     treeshake: "smallest",
     minify: true,
     sourcemap: true,
+    esbuildPlugins: [
+      copy({
+        assets: [
+          {
+            from: "./wasm/*",
+            to: "./wasm",
+          },
+        ],
+      }),
+    ],
     esbuildOptions(options) {
       defineEnvs(options, { browser: false });
     },

clients/tabby-agent/wasm/tree-sitter-go.wasm

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1b69c5af834fd23053238e484c7fe9ed2f121d5b1fe32242af78576d67e49f1e
+size 240169

clients/tabby-agent/wasm/tree-sitter-python.wasm

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:72d0f97ba6c3134d7873ec5c9d0fd3c1f5137f4eac4dda0709993d92809e62b6
+size 474189

clients/tabby-agent/wasm/tree-sitter-ruby.wasm

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1190cddd839b78c2aec737573399a71c23fe9a546d3543f86304c4c68ca73852
+size 990787

clients/tabby-agent/wasm/tree-sitter-rust.wasm

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:273f9ce6f2c595ad4e63b3195513b61974ae1ec513efcce39da1afa90574ef38
+size 844087

clients/tabby-agent/wasm/tree-sitter-tsx.wasm

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:060422a330f9c819a10e7310788d336dbcb53cc6a4be0e91d40f644564080f97
+size 1182114

clients/tabby-agent/wasm/tree-sitter.wasm

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:17382e1a69bd628107e8dfe37d31d57f7ba948e5f2da77e56a8aa010488dc5ae
+size 186526

clients/vscode/package.json

@@ -208,6 +208,7 @@
     "@vscode/test-web": "^0.0.44",
     "@vscode/vsce": "^2.15.0",
     "assert": "^2.0.0",
+    "esbuild-plugin-copy": "^2.1.1",
     "esbuild-plugin-polyfill-node": "^0.3.0",
     "eslint": "^8.20.0",
     "glob": "^8.0.3",

clients/vscode/tsup.config.ts

@@ -1,4 +1,5 @@
 import { defineConfig } from "tsup";
+import { copy } from "esbuild-plugin-copy";
 import { polyfillNode } from "esbuild-plugin-polyfill-node";
 import { dependencies } from "./package.json";
 
@@ -8,8 +9,19 @@ export default () => [
     entry: ["src/extension.ts"],
     outDir: "dist/node",
     platform: "node",
+    target: "node18",
     external: ["vscode"],
     noExternal: Object.keys(dependencies),
+    esbuildPlugins: [
+      copy({
+        assets: [
+          {
+            from: "../tabby-agent/dist/wasm/*.wasm",
+            to: "./wasm",
+          },
+        ],
+      }),
+    ],
     clean: true,
   }),
   defineConfig({

yarn.lock

@@ -900,7 +900,7 @@ chalk@^2.4.2:
     escape-string-regexp "^1.0.5"
     supports-color "^5.3.0"
 
-chalk@^4.0.0, chalk@^4.1.0:
+chalk@^4.0.0, chalk@^4.1.0, chalk@^4.1.2:
   version "4.1.2"
   resolved "https://registry.yarnpkg.com/chalk/-/chalk-4.1.2.tgz#aac4e2b7734a740867aeb16bf02aad556a1e7a01"
   integrity sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==
@@ -1390,6 +1390,16 @@ es-get-iterator@^1.1.3:
     isarray "^2.0.5"
     stop-iteration-iterator "^1.0.0"
 
+esbuild-plugin-copy@^2.1.1:
+  version "2.1.1"
+  resolved "https://registry.yarnpkg.com/esbuild-plugin-copy/-/esbuild-plugin-copy-2.1.1.tgz#638308ecfd679e4c7c76b71c62f7dd9a4cc7f901"
+  integrity sha512-Bk66jpevTcV8KMFzZI1P7MZKZ+uDcrZm2G2egZ2jNIvVnivDpodZI+/KnpL3Jnap0PBdIHU7HwFGB8r+vV5CVw==
+  dependencies:
+    chalk "^4.1.2"
+    chokidar "^3.5.3"
+    fs-extra "^10.0.1"
+    globby "^11.0.3"
+
 esbuild-plugin-polyfill-node@^0.3.0:
   version "0.3.0"
   resolved "https://registry.yarnpkg.com/esbuild-plugin-polyfill-node/-/esbuild-plugin-polyfill-node-0.3.0.tgz#e7e3804b8272df51ae4f8ebfb7445a03712504cb"
@@ -1706,6 +1716,15 @@ fs-constants@^1.0.0:
   resolved "https://registry.yarnpkg.com/fs-constants/-/fs-constants-1.0.0.tgz#6be0de9be998ce16af8afc24497b9ee9b7ccd9ad"
   integrity sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==
 
+fs-extra@^10.0.1:
+  version "10.1.0"
+  resolved "https://registry.yarnpkg.com/fs-extra/-/fs-extra-10.1.0.tgz#02873cfbc4084dde127eaa5f9905eef2325d1abf"
+  integrity sha512-oRXApq54ETRj4eMiFzGnHWGy+zo5raudjuxN0b8H7s/RU2oW0Wvsx9O0ACRN/kRq9E8Vu/ReskGB5o3ji+FzHQ==
+  dependencies:
+    graceful-fs "^4.2.0"
+    jsonfile "^6.0.1"
+    universalify "^2.0.0"
+
 fs-extra@^11.1.1:
   version "11.1.1"
   resolved "https://registry.yarnpkg.com/fs-extra/-/fs-extra-11.1.1.tgz#da69f7c39f3b002378b0954bb6ae7efdc0876e2d"
@@ -4117,6 +4136,11 @@ vscode-uri@^3.0.7:
   resolved "https://registry.yarnpkg.com/vscode-uri/-/vscode-uri-3.0.7.tgz#6d19fef387ee6b46c479e5fb00870e15e58c1eb8"
   integrity sha512-eOpPHogvorZRobNqJGhapa0JdwaxpjVvyBp0QIUMRMSf8ZAlqOdEquKuRmw9Qwu0qXtJIWqFtMkmvJjUZmMjVA==
 
+web-tree-sitter@^0.20.8:
+  version "0.20.8"
+  resolved "https://registry.yarnpkg.com/web-tree-sitter/-/web-tree-sitter-0.20.8.tgz#1e371cb577584789cadd75cb49c7ddfbc99d04c8"
+  integrity sha512-weOVgZ3aAARgdnb220GqYuh7+rZU0Ka9k9yfKtGAzEYMa6GgiCzW9JjQRJyCJakvibQW+dfjJdihjInKuuCAUQ==
+
 webidl-conversions@^4.0.2:
   version "4.0.2"
   resolved "https://registry.yarnpkg.com/webidl-conversions/-/webidl-conversions-4.0.2.tgz#a855980b1f0b6b359ba1d5d9fb39ae941faa63ad"