diff --git a/.github/workflows/intellij-test.yml b/.github/workflows/intellij-test.yml index d67e31c..fe64b89 100644 --- a/.github/workflows/intellij-test.yml +++ b/.github/workflows/intellij-test.yml @@ -18,6 +18,8 @@ jobs: steps: - name: Checkout uses: actions/checkout@v3 + with: + lfs: true - name: Setup JDK uses: actions/setup-java@v3 with: diff --git a/.github/workflows/tabby-agent-test.yml b/.github/workflows/tabby-agent-test.yml index fc47d85..9f62fb1 100644 --- a/.github/workflows/tabby-agent-test.yml +++ b/.github/workflows/tabby-agent-test.yml @@ -22,6 +22,8 @@ jobs: steps: - name: Checkout uses: actions/checkout@v3 + with: + lfs: true - name: Setup Node.js uses: actions/setup-node@v3 with: diff --git a/.github/workflows/vscode-test.yml b/.github/workflows/vscode-test.yml index ffa952e..f581d0e 100644 --- a/.github/workflows/vscode-test.yml +++ b/.github/workflows/vscode-test.yml @@ -24,6 +24,8 @@ jobs: steps: - name: Checkout uses: actions/checkout@v3 + with: + lfs: true - name: Setup Node.js uses: actions/setup-node@v3 with: diff --git a/clients/tabby-agent/.gitattributes b/clients/tabby-agent/.gitattributes new file mode 100644 index 0000000..5f2fd03 --- /dev/null +++ b/clients/tabby-agent/.gitattributes @@ -0,0 +1 @@ +*.wasm filter=lfs diff=lfs merge=lfs -text \ No newline at end of file diff --git a/clients/tabby-agent/package.json b/clients/tabby-agent/package.json index 6664789..9a1d153 100644 --- a/clients/tabby-agent/package.json +++ b/clients/tabby-agent/package.json @@ -23,11 +23,11 @@ "@types/node": "^18.12.0", "chai": "^4.3.7", "dedent": "^0.7.0", + "esbuild-plugin-copy": "^2.1.1", "esbuild-plugin-polyfill-node": "^0.3.0", "mocha": "^10.2.0", "openapi-typescript": "^6.6.1", "prettier": "^3.0.0", - "rimraf": "^5.0.1", "ts-node": "^10.9.1", "tsup": "^7.1.0", "typescript": "^5.0.3" @@ -47,6 +47,7 @@ "rotating-file-stream": "^3.1.0", "stats-logscale": "^1.0.7", "toml": "^3.0.0", - "uuid": "^9.0.0" + "uuid": "^9.0.0", + "web-tree-sitter": "^0.20.8" } } diff --git a/clients/tabby-agent/src/AgentConfig.ts b/clients/tabby-agent/src/AgentConfig.ts index 7ca09d4..ecc2796 100644 --- a/clients/tabby-agent/src/AgentConfig.ts +++ b/clients/tabby-agent/src/AgentConfig.ts @@ -21,11 +21,15 @@ export type AgentConfig = { timeout: number; }; postprocess: { - limitScopeByIndentation: { - // When completion is continuing the current line, limit the scope to: - // false(default): the line scope, meaning use the next indent level as the limit. - // true: the block scope, meaning use the current indent level as the limit. - experimentalKeepBlockScopeWhenCompletingLine: boolean; + limitScope: { + // Prefer to use syntax parser than indentation + experimentalSyntax: boolean; + indentation: { + // When completion is continuing the current line, limit the scope to: + // false(default): the line scope, meaning use the next indent level as the limit. + // true: the block scope, meaning use the current indent level as the limit. + experimentalKeepBlockScopeWhenCompletingLine: boolean; + }; }; }; logs: { @@ -66,8 +70,11 @@ export const defaultAgentConfig: AgentConfig = { timeout: 4000, // ms }, postprocess: { - limitScopeByIndentation: { - experimentalKeepBlockScopeWhenCompletingLine: false, + limitScope: { + experimentalSyntax: false, + indentation: { + experimentalKeepBlockScopeWhenCompletingLine: false, + }, }, }, logs: { diff --git a/clients/tabby-agent/src/TabbyAgent.ts b/clients/tabby-agent/src/TabbyAgent.ts index b9338e8..ef0e935 100644 --- a/clients/tabby-agent/src/TabbyAgent.ts +++ b/clients/tabby-agent/src/TabbyAgent.ts @@ -529,7 +529,7 @@ export class TabbyAgent extends EventEmitter implements Agent { throw options.signal.reason; } // Build cache - this.completionCache.buildCache(context, completionResponse); + this.completionCache.buildCache(context, JSON.parse(JSON.stringify(completionResponse))); } } // Postprocess (post-cache) diff --git a/clients/tabby-agent/src/postprocess/index.ts b/clients/tabby-agent/src/postprocess/index.ts index ba0ba3b..e558909 100644 --- a/clients/tabby-agent/src/postprocess/index.ts +++ b/clients/tabby-agent/src/postprocess/index.ts @@ -4,7 +4,7 @@ import { applyFilter } from "./base"; import { removeRepetitiveBlocks } from "./removeRepetitiveBlocks"; import { removeRepetitiveLines } from "./removeRepetitiveLines"; import { removeLineEndsWithRepetition } from "./removeLineEndsWithRepetition"; -import { limitScopeByIndentation } from "./limitScopeByIndentation"; +import { limitScope } from "./limitScope"; import { trimSpace } from "./trimSpace"; import { dropDuplicated } from "./dropDuplicated"; import { dropBlank } from "./dropBlank"; @@ -30,7 +30,7 @@ export async function postCacheProcess( return Promise.resolve(response) .then(applyFilter(removeRepetitiveBlocks(context), context)) .then(applyFilter(removeRepetitiveLines(context), context)) - .then(applyFilter(limitScopeByIndentation(context, config["limitScopeByIndentation"]), context)) + .then(applyFilter(limitScope(context, config["limitScope"]), context)) .then(applyFilter(dropDuplicated(context), context)) .then(applyFilter(trimSpace(context), context)) .then(applyFilter(dropBlank(), context)); diff --git a/clients/tabby-agent/src/postprocess/limitScope.ts b/clients/tabby-agent/src/postprocess/limitScope.ts new file mode 100644 index 0000000..4720382 --- /dev/null +++ b/clients/tabby-agent/src/postprocess/limitScope.ts @@ -0,0 +1,24 @@ +import { CompletionContext } from "../Agent"; +import { AgentConfig } from "../AgentConfig"; +import { isBrowser } from "../env"; +import { PostprocessFilter } from "./base"; +import { limitScopeByIndentation } from "./limitScopeByIndentation"; +import { limitScopeBySyntax, supportedLanguages } from "./limitScopeBySyntax"; + +export function limitScope( + context: CompletionContext, + config: AgentConfig["postprocess"]["limitScope"], +): PostprocessFilter { + return isBrowser + ? (input) => { + // syntax parser is not supported in browser yet + return limitScopeByIndentation(context, config["indentation"])(input); + } + : (input) => { + if (config.experimentalSyntax && supportedLanguages.indexOf(context.language) >= 0) { + return limitScopeBySyntax(context)(input); + } else { + return limitScopeByIndentation(context, config["indentation"])(input); + } + }; +} diff --git a/clients/tabby-agent/src/postprocess/limitScopeByIndentation.ts b/clients/tabby-agent/src/postprocess/limitScopeByIndentation.ts index 4e4c8f2..42c32a6 100644 --- a/clients/tabby-agent/src/postprocess/limitScopeByIndentation.ts +++ b/clients/tabby-agent/src/postprocess/limitScopeByIndentation.ts @@ -18,7 +18,7 @@ function processContext( lines: string[], prefixLines: string[], suffixLines: string[], - config: AgentConfig["postprocess"]["limitScopeByIndentation"], + config: AgentConfig["postprocess"]["limitScope"]["indentation"], ): { indentLevelLimit: number; allowClosingLine: (closingLine: string) => boolean } { let allowClosingLine = false; let result = { indentLevelLimit: 0, allowClosingLine: (closingLine: string) => allowClosingLine }; @@ -103,14 +103,14 @@ function processContext( export function limitScopeByIndentation( context: CompletionContext, - config: AgentConfig["postprocess"]["limitScopeByIndentation"], + config: AgentConfig["postprocess"]["limitScope"]["indentation"], ): PostprocessFilter { return (input) => { - const { prefix, suffix, prefixLines, suffixLines } = context; + const { prefixLines, suffixLines } = context; const inputLines = splitLines(input); if (context.mode === "fill-in-line") { if (inputLines.length > 1) { - logger.debug({ input, prefix, suffix }, "Drop content with multiple lines"); + logger.debug({ inputLines, prefixLines, suffixLines }, "Drop content with multiple lines"); return null; } } @@ -139,7 +139,10 @@ export function limitScopeByIndentation( } } if (index < inputLines.length) { - logger.debug({ input, prefix, suffix, scopeEndAt: index }, "Remove content out of scope"); + logger.debug( + { inputLines, prefixLines, suffixLines, scopeLineCount: index }, + "Remove content out of indent scope", + ); return inputLines.slice(0, index).join("").trimEnd(); } return input; diff --git a/clients/tabby-agent/src/postprocess/limitScopeBySyntax.test.ts b/clients/tabby-agent/src/postprocess/limitScopeBySyntax.test.ts new file mode 100644 index 0000000..066c1a5 --- /dev/null +++ b/clients/tabby-agent/src/postprocess/limitScopeBySyntax.test.ts @@ -0,0 +1,409 @@ +import { expect } from "chai"; +import { documentContext, inline } from "./testUtils"; +import { limitScopeBySyntax } from "./limitScopeBySyntax"; + +describe("postprocess", () => { + describe("limitScopeBySyntax javascript", () => { + it("should limit scope at function_declaration.", async () => { + const context = { + ...documentContext` + function findMax(arr) {║} + `, + language: "javascript", + }; + const completion = inline` + ├ + let max = arr[0]; + for (let i = 1; i < arr.length; i++) { + if (arr[i] > max) { + max = arr[i]; + } + } + return max; + } + console.log(findMax([1, 2, 3, 4, 5]));┤ + `; + const expected = inline` + ├ + let max = arr[0]; + for (let i = 1; i < arr.length; i++) { + if (arr[i] > max) { + max = arr[i]; + } + } + return max; + }┤ + `; + expect(await limitScopeBySyntax(context)(completion)).to.eq(expected); + }); + + it("should limit scope at function_declaration", async () => { + const context = { + ...documentContext` + function findMax(arr) { + let max = arr[0];║ + } + `, + language: "javascript", + }; + const completion = inline` + ├ + for (let i = 1; i < arr.length; i++) { + if (arr[i] > max) { + max = arr[i]; + } + } + return max; + }┤ + `; + expect(await limitScopeBySyntax(context)(completion)).to.eq(completion); + }); + + it("should limit scope at function_declaration", async () => { + const context = { + ...documentContext` + function findMax(arr) { + let max = arr[0]; + for (let i = 1; i < arr.length; i++) { + if (arr[i] > max) { + max = arr[i]; + } + }║ + } + `, + language: "javascript", + }; + const completion = inline` + ├ + return max; + }┤ + `; + expect(await limitScopeBySyntax(context)(completion)).to.eq(completion); + }); + + it("should limit scope at for_statement.", async () => { + const context = { + ...documentContext` + function findMax(arr) { + let max = arr[0]; + for║ + } + `, + language: "javascript", + }; + const completion = inline` + ├ (let i = 1; i < arr.length; i++) { + if (arr[i] > max) { + max = arr[i]; + } + } + return max; + } + console.log(findMax([1, 2, 3, 4, 5]));┤ + `; + const expected = inline` + ├ (let i = 1; i < arr.length; i++) { + if (arr[i] > max) { + max = arr[i]; + } + }┤ + ┴┴ + `; + expect(await limitScopeBySyntax(context)(completion)).to.eq(expected); + }); + + it("should limit scope at current node if no parent scope found.", async () => { + const context = { + ...documentContext` + let a =║ + `, + language: "javascript", + }; + const completion = inline` + ├ 1; + let b = 2;┤ + `; + const expected = inline` + ├ 1;┤ + `; + expect(await limitScopeBySyntax(context)(completion)).to.eq(expected); + }); + + it("should handle the bad case of limitScopeByIndentation", async () => { + const context = { + ...documentContext` + function sortWords(input) { + const output = input.trim() + .split("\n") + .map((line) => line.split(" ")) + ║ + } + `, + language: "javascript", + }; + const completion = inline` + ├.flat() + .sort() + .join(" "); + console.log(output); + return output; + } + sortWord("world hello");┤ + `; + const expected = inline` + ├.flat() + .sort() + .join(" "); + console.log(output); + return output; + };┤ + `; + expect(await limitScopeBySyntax(context)(completion)).not.to.eq(expected); + }); + }); + + describe("limitScopeBySyntax python", () => { + it("should limit scope at function_definition.", async () => { + const context = { + ...documentContext` + def find_min(arr):║ + `, + language: "python", + }; + const completion = inline` + ├ + min = arr[0] + for i in range(1, len(arr)): + if arr[i] < min: + min = arr[i] + return min + print(find_min([1, 2, 3, 4, 5]))┤ + `; + const expected = inline` + ├ + min = arr[0] + for i in range(1, len(arr)): + if arr[i] < min: + min = arr[i] + return min┤ + ┴┴ + `; + expect(await limitScopeBySyntax(context)(completion)).to.eq(expected); + }); + + it("should limit scope at function_definition.", async () => { + const context = { + ...documentContext` + def find_min(arr): + min = arr[0]║ + `, + language: "python", + }; + const completion = inline` + ├ + for i in range(1, len(arr)): + if arr[i] < min: + min = arr[i] + return min + print(find_min([1, 2, 3, 4, 5]))┤ + `; + const expected = inline` + ├ + for i in range(1, len(arr)): + if arr[i] < min: + min = arr[i] + return min┤ + ┴┴ + `; + expect(await limitScopeBySyntax(context)(completion)).to.eq(expected); + }); + + it("should limit scope at function_definition.", async () => { + const context = { + ...documentContext` + def find_min(arr): + min = arr[0] + for i in range(1, len(arr)): + if arr[i] < min: + min = arr[i]║ + `, + language: "python", + }; + const completion = inline` + ├ + return min┤ + `; + expect(await limitScopeBySyntax(context)(completion)).to.eq(completion); + }); + + it("should limit scope at for_statement.", async () => { + const context = { + ...documentContext` + def find_min(arr): + max = arr[0] + for║ + `, + language: "python", + }; + const completion = inline` + ├ i in range(1, len(arr)): + if arr[i] < min: + min = arr[i] + return min + ┤ + `; + const expected = inline` + ├ i in range(1, len(arr)): + if arr[i] < min: + min = arr[i]┤ + ┴┴┴┴ + `; + expect(await limitScopeBySyntax(context)(completion)).to.eq(expected); + }); + + it("should handle the bad case of limitScopeByIndentation", async () => { + const context = { + ...documentContext` + def findMax(arr): + ║ + `, + language: "python", + }; + const completion = inline` + ├max = arr[0] + for i in range(1, len(arr)): + if arr[i] > max: + max = arr[i] + return max + findMax([1, 2, 3, 4, 5])┤ + `; + const expected = inline` + ├max = arr[0] + for i in range(1, len(arr)): + if arr[i] > max: + max = arr[i] + return max┤ + `; + expect(await limitScopeBySyntax(context)(completion)).not.to.eq(expected); + }); + }); + + describe("limitScopeBySyntax go", () => { + it("should limit scope at function_declaration.", async () => { + const context = { + ...documentContext` + func findMin(arr []int) int {║} + `, + language: "go", + }; + const completion = inline` + ├ + min := math.MaxInt64 + for _, v := range arr { + if v < min { + min = v + } + } + return min + } + + func main() { + arr := []int{5, 2, 9, 8, 1, 3} + fmt.Println(findMin(arr)) // Output: 1 + }┤ + `; + const expected = inline` + ├ + min := math.MaxInt64 + for _, v := range arr { + if v < min { + min = v + } + } + return min + }┤ + `; + expect(await limitScopeBySyntax(context)(completion)).to.eq(expected); + }); + + it("should limit scope at for_statement.", async () => { + const context = { + ...documentContext` + func findMin(arr []int) int { + min := math.MaxInt64 + for║ + `, + language: "go", + }; + const completion = inline` + ├ _, v := range arr { + if v < min { + min = v + } + } + return min + }┤ + `; + const expected = inline` + ├ _, v := range arr { + if v < min { + min = v + } + }┤ + ┴┴ + `; + expect(await limitScopeBySyntax(context)(completion)).to.eq(expected); + }); + }); + + describe("limitScopeBySyntax rust", () => { + it("should limit scope at function_item.", async () => { + const context = { + ...documentContext` + fn find_min(arr: &[i32]) -> i32 {║} + `, + language: "rust", + }; + const completion = inline` + ├ + *arr.iter().min().unwrap() + } + fn main() { + let arr = vec![5, 2, 9, 8, 1, 3]; + println!("{}", find_min(&arr)); // Output: 1 + }┤ + `; + const expected = inline` + ├ + *arr.iter().min().unwrap() + }┤ + `; + expect(await limitScopeBySyntax(context)(completion)).to.eq(expected); + }); + }); + + describe("limitScopeBySyntax ruby", () => { + it("should limit scope at for.", async () => { + const context = { + ...documentContext` + def fibonacci(n)║ + `, + language: "ruby", + }; + const completion = inline` + ├ + return n if n <= 1 + fibonacci(n - 1) + fibonacci(n - 2) + end + puts fibonacci(10)┤ + `; + const expected = inline` + ├ + return n if n <= 1 + fibonacci(n - 1) + fibonacci(n - 2) + end┤ + `; + expect(await limitScopeBySyntax(context)(completion)).to.eq(expected); + }); + }); +}); diff --git a/clients/tabby-agent/src/postprocess/limitScopeBySyntax.ts b/clients/tabby-agent/src/postprocess/limitScopeBySyntax.ts new file mode 100644 index 0000000..ebb8139 --- /dev/null +++ b/clients/tabby-agent/src/postprocess/limitScopeBySyntax.ts @@ -0,0 +1,86 @@ +import type TreeSitterParser from "web-tree-sitter"; +import { getParser, languagesConfigs } from "../syntax/parser"; +import { typeList } from "../syntax/typeList"; +import { CompletionContext } from "../Agent"; +import { PostprocessFilter, logger } from "./base"; + +export const supportedLanguages = Object.keys(languagesConfigs); + +function findLineBegin(text: string, position: number): number { + let lastNonBlankCharPos = position - 1; + while (lastNonBlankCharPos >= 0 && text[lastNonBlankCharPos].match(/\s/)) { + lastNonBlankCharPos--; + } + if (lastNonBlankCharPos < 0) { + return 0; + } + const lineBegin = text.lastIndexOf("\n", lastNonBlankCharPos); + if (lineBegin < 0) { + return 0; + } + const line = text.slice(lineBegin + 1, position); + const indentation = line.search(/\S/); + return lineBegin + 1 + indentation; +} + +function findLineEnd(text: string, position: number): number { + let firstNonBlankCharPos = position; + while (firstNonBlankCharPos < text.length && text[firstNonBlankCharPos].match(/\s/)) { + firstNonBlankCharPos++; + } + if (firstNonBlankCharPos >= text.length) { + return text.length; + } + const lineEnd = text.indexOf("\n", firstNonBlankCharPos); + if (lineEnd < 0) { + return text.length; + } + return lineEnd; +} + +function findScope(node: TreeSitterParser.SyntaxNode, typeList: string[][]): TreeSitterParser.SyntaxNode { + for (const types of typeList) { + let scope = node; + while (scope) { + if (types.indexOf(scope.type) >= 0) { + return scope; + } + scope = scope.parent; + } + } + return node; +} + +export function limitScopeBySyntax(context: CompletionContext): PostprocessFilter { + return async (input) => { + const { position, text, language, prefix, suffix } = context; + if (supportedLanguages.indexOf(language) < 0) { + return input; + } + const languageConfig = languagesConfigs[language]; + const parser = await getParser(languageConfig); + + const updatedText = prefix + input + suffix; + const updatedTree = parser.parse(updatedText); + const lineBegin = findLineBegin(updatedText, position); + const lineEnd = findLineEnd(updatedText, position); + const scope = findScope(updatedTree.rootNode.namedDescendantForIndex(lineBegin, lineEnd), typeList[languageConfig]); + + if (scope.endIndex < position + input.length) { + logger.debug( + { + languageConfig, + text, + updatedText, + position, + lineBegin, + lineEnd, + scope: { type: scope.type, start: scope.startIndex, end: scope.endIndex }, + }, + "Remove content out of syntax scope", + ); + return input.slice(0, scope.endIndex - position); + } + return input; + }; +} diff --git a/clients/tabby-agent/src/postprocess/testUtils.ts b/clients/tabby-agent/src/postprocess/testUtils.ts index a0252af..ffd122b 100644 --- a/clients/tabby-agent/src/postprocess/testUtils.ts +++ b/clients/tabby-agent/src/postprocess/testUtils.ts @@ -1,12 +1,13 @@ import dedent from "dedent"; +import { v4 as uuid } from "uuid"; import { CompletionContext } from "../CompletionContext"; // `║` is the cursor position export function documentContext(strings): CompletionContext { const doc = dedent(strings); return new CompletionContext({ - filepath: null, - language: null, + filepath: uuid(), + language: "", text: doc.replace(/║/, ""), position: doc.indexOf("║"), }); diff --git a/clients/tabby-agent/src/syntax/parser.ts b/clients/tabby-agent/src/syntax/parser.ts new file mode 100644 index 0000000..52f3c84 --- /dev/null +++ b/clients/tabby-agent/src/syntax/parser.ts @@ -0,0 +1,45 @@ +import TreeSitterParser from "web-tree-sitter"; +import { isTest } from "../env"; + +// https://code.visualstudio.com/docs/languages/identifiers +export const languagesConfigs: Record = { + javascript: "tsx", + typescript: "tsx", + javascriptreact: "tsx", + typescriptreact: "tsx", + python: "python", + go: "go", + rust: "rust", + ruby: "ruby", +}; + +var treeSitterInitialized = false; + +async function createParser(languageConfig: string): Promise { + if (!treeSitterInitialized) { + await TreeSitterParser.init({ + locateFile(scriptName: string, scriptDirectory: string) { + const paths = isTest ? [scriptDirectory, scriptName] : [scriptDirectory, "wasm", scriptName]; + return require("path").join(...paths); + }, + }); + treeSitterInitialized = true; + } + const parser = new TreeSitterParser(); + const langWasmPaths = isTest + ? [process.cwd(), "wasm", `tree-sitter-${languageConfig}.wasm`] + : [__dirname, "wasm", `tree-sitter-${languageConfig}.wasm`]; + parser.setLanguage(await TreeSitterParser.Language.load(require("path").join(...langWasmPaths))); + return parser; +} + +const parsers = new Map(); + +export async function getParser(languageConfig: string): Promise { + let parser = parsers.get(languageConfig); + if (!parser) { + parser = await createParser(languageConfig); + parsers.set(languageConfig, parser); + } + return parser; +} diff --git a/clients/tabby-agent/src/syntax/typeList.ts b/clients/tabby-agent/src/syntax/typeList.ts new file mode 100644 index 0000000..4934183 --- /dev/null +++ b/clients/tabby-agent/src/syntax/typeList.ts @@ -0,0 +1,116 @@ +// https://github.com/tree-sitter/tree-sitter-typescript/blob/master/src/node-types.json +export const typeList = { + tsx: [ + [ + "jsx_element", + "jsx_self_closing_element", + + // exclude sentence level nodes for now + // "expression_statement", + // "lexical_declaration", + + "for_statement", + "for_in_statement", + "if_statement", + "while_statement", + "do_statement", + "switch_statement", + "try_statement", + "with_statement", + "labeled_statement", + + "class_declaration", + "abstract_class_declaration", + "interface_declaration", + "enum_declaration", + "type_alias_declaration", + "function_declaration", + "generator_function_declaration", + "ambient_declaration", + + "method_definition", + + "import_statement", + "export_statement", + "module", + ], + ], + + // https://github.com/tree-sitter/tree-sitter-python/blob/master/src/node-types.json + python: [ + [ + "for_statement", + "if_statement", + "while_statement", + "match_statement", + "try_statement", + "with_statement", + + "function_definition", + "decorated_definition", + "class_definition", + + "import_statement", + "import_from_statement", + ], + ], + + // https://github.com/tree-sitter/tree-sitter-go/blob/master/src/node-types.json + go: [ + [ + "for_statement", + "if_statement", + "expression_switch_statement", + "type_switch_statement", + "select_statement", + "labeled_statement", + + "function_declaration", + "method_declaration", + "type_declaration", + + "import_declaration", + "package_clause", + ], + ], + + // https://github.com/tree-sitter/tree-sitter-rust/blob/master/src/node-types.json + rust: [ + [ + "for_expression", + "if_expression", + "while_expression", + "loop_expression", + "match_expression", + "try_expression", + + "function_item", + "type_item", + "enum_item", + "struct_item", + "union_item", + "trait_item", + "impl_item", + + "use_declaration", + ], + ], + + // https://github.com/tree-sitter/tree-sitter-ruby/blob/master/src/node-types.json + ruby: [ + [ + "for", + "if", + "unless", + "while", + "until", + "case", + + "class", + "singleton_class", + "method", + "singleton_method", + "module", + ], + ], +}; diff --git a/clients/tabby-agent/tests/golden.test.ts b/clients/tabby-agent/tests/golden.test.ts index 0d8443b..75526c7 100644 --- a/clients/tabby-agent/tests/golden.test.ts +++ b/clients/tabby-agent/tests/golden.test.ts @@ -81,7 +81,10 @@ describe("agent golden test", () => { timeout: 4000, }, postprocess: { - limitScopeByIndentation: { experimentalKeepBlockScopeWhenCompletingLine: false }, + limitScope: { + experimentalSyntax: false, + indentation: { experimentalKeepBlockScopeWhenCompletingLine: false }, + }, }, logs: { level: "debug" }, anonymousUsageTracking: { disable: true }, @@ -111,7 +114,7 @@ describe("agent golden test", () => { absolutePath: path.join(__dirname, "golden", file), }; }); - goldenFiles.forEach((goldenFile, index) => { + goldenFiles.forEach((goldenFile) => { it(goldenFile.path, async () => { const test = await createGoldenTest(goldenFile.absolutePath); requestId++; @@ -130,7 +133,7 @@ describe("agent golden test", () => { absolutePath: path.join(__dirname, "bad_cases", file), }; }); - badCasesFiles.forEach((goldenFile, index) => { + badCasesFiles.forEach((goldenFile) => { it(goldenFile.path, async () => { const test = await createGoldenTest(goldenFile.absolutePath); requestId++; @@ -143,6 +146,35 @@ describe("agent golden test", () => { }); }); + it("updateConfig experimental", async () => { + requestId++; + const updateConfigRequest = [ + requestId, + { + func: "updateConfig", + args: ["postprocess.limitScope.experimentalSyntax", true], + }, + ]; + agent.stdin.write(JSON.stringify(updateConfigRequest) + "\n"); + await waitForResponse(requestId); + const expectedConfig = { ...config }; + expectedConfig.postprocess.limitScope.experimentalSyntax = true; + expect(output.shift()).to.deep.equal([0, { event: "configUpdated", config: expectedConfig }]); + expect(output.shift()).to.deep.equal([requestId, true]); + }); + badCasesFiles.forEach((goldenFile) => { + it("experimental: " + goldenFile.path, async () => { + const test = await createGoldenTest(goldenFile.absolutePath); + requestId++; + const request = [requestId, { func: "provideCompletions", args: [test.request] }]; + agent.stdin.write(JSON.stringify(request) + "\n"); + await waitForResponse(requestId); + const response = output.shift(); + expect(response[0]).to.equal(requestId); + expect(response[1].choices).to.deep.equal(test.expected.choices); + }); + }); + after(() => { agent.kill(); }); diff --git a/clients/tabby-agent/tsup.config.ts b/clients/tabby-agent/tsup.config.ts index fbc817e..9cbb153 100644 --- a/clients/tabby-agent/tsup.config.ts +++ b/clients/tabby-agent/tsup.config.ts @@ -1,4 +1,5 @@ import { defineConfig } from "tsup"; +import { copy } from "esbuild-plugin-copy"; import { polyfillNode } from "esbuild-plugin-polyfill-node"; import { dependencies } from "./package.json"; @@ -16,6 +17,7 @@ export default async () => [ name: "node-cjs", entry: ["src/index.ts"], platform: "node", + target: "node18", format: ["cjs"], sourcemap: true, esbuildOptions(options) { @@ -71,10 +73,21 @@ export default async () => [ name: "cli", entry: ["src/cli.ts"], platform: "node", + target: "node18", noExternal: Object.keys(dependencies), treeshake: "smallest", minify: true, sourcemap: true, + esbuildPlugins: [ + copy({ + assets: [ + { + from: "./wasm/*", + to: "./wasm", + }, + ], + }), + ], esbuildOptions(options) { defineEnvs(options, { browser: false }); }, diff --git a/clients/tabby-agent/wasm/tree-sitter-go.wasm b/clients/tabby-agent/wasm/tree-sitter-go.wasm new file mode 100644 index 0000000..719ebf6 --- /dev/null +++ b/clients/tabby-agent/wasm/tree-sitter-go.wasm @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1b69c5af834fd23053238e484c7fe9ed2f121d5b1fe32242af78576d67e49f1e +size 240169 diff --git a/clients/tabby-agent/wasm/tree-sitter-python.wasm b/clients/tabby-agent/wasm/tree-sitter-python.wasm new file mode 100644 index 0000000..ae2cce7 --- /dev/null +++ b/clients/tabby-agent/wasm/tree-sitter-python.wasm @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:72d0f97ba6c3134d7873ec5c9d0fd3c1f5137f4eac4dda0709993d92809e62b6 +size 474189 diff --git a/clients/tabby-agent/wasm/tree-sitter-ruby.wasm b/clients/tabby-agent/wasm/tree-sitter-ruby.wasm new file mode 100644 index 0000000..f0e02da --- /dev/null +++ b/clients/tabby-agent/wasm/tree-sitter-ruby.wasm @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1190cddd839b78c2aec737573399a71c23fe9a546d3543f86304c4c68ca73852 +size 990787 diff --git a/clients/tabby-agent/wasm/tree-sitter-rust.wasm b/clients/tabby-agent/wasm/tree-sitter-rust.wasm new file mode 100644 index 0000000..d608d6c --- /dev/null +++ b/clients/tabby-agent/wasm/tree-sitter-rust.wasm @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:273f9ce6f2c595ad4e63b3195513b61974ae1ec513efcce39da1afa90574ef38 +size 844087 diff --git a/clients/tabby-agent/wasm/tree-sitter-tsx.wasm b/clients/tabby-agent/wasm/tree-sitter-tsx.wasm new file mode 100644 index 0000000..76509d0 --- /dev/null +++ b/clients/tabby-agent/wasm/tree-sitter-tsx.wasm @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:060422a330f9c819a10e7310788d336dbcb53cc6a4be0e91d40f644564080f97 +size 1182114 diff --git a/clients/tabby-agent/wasm/tree-sitter.wasm b/clients/tabby-agent/wasm/tree-sitter.wasm new file mode 100644 index 0000000..81dd158 --- /dev/null +++ b/clients/tabby-agent/wasm/tree-sitter.wasm @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:17382e1a69bd628107e8dfe37d31d57f7ba948e5f2da77e56a8aa010488dc5ae +size 186526 diff --git a/clients/vscode/package.json b/clients/vscode/package.json index d5277eb..be8a3a1 100644 --- a/clients/vscode/package.json +++ b/clients/vscode/package.json @@ -208,6 +208,7 @@ "@vscode/test-web": "^0.0.44", "@vscode/vsce": "^2.15.0", "assert": "^2.0.0", + "esbuild-plugin-copy": "^2.1.1", "esbuild-plugin-polyfill-node": "^0.3.0", "eslint": "^8.20.0", "glob": "^8.0.3", diff --git a/clients/vscode/tsup.config.ts b/clients/vscode/tsup.config.ts index 10c9c79..c89b1c7 100644 --- a/clients/vscode/tsup.config.ts +++ b/clients/vscode/tsup.config.ts @@ -1,4 +1,5 @@ import { defineConfig } from "tsup"; +import { copy } from "esbuild-plugin-copy"; import { polyfillNode } from "esbuild-plugin-polyfill-node"; import { dependencies } from "./package.json"; @@ -8,8 +9,19 @@ export default () => [ entry: ["src/extension.ts"], outDir: "dist/node", platform: "node", + target: "node18", external: ["vscode"], noExternal: Object.keys(dependencies), + esbuildPlugins: [ + copy({ + assets: [ + { + from: "../tabby-agent/dist/wasm/*.wasm", + to: "./wasm", + }, + ], + }), + ], clean: true, }), defineConfig({ diff --git a/yarn.lock b/yarn.lock index f9ca8b3..e8bce74 100644 --- a/yarn.lock +++ b/yarn.lock @@ -900,7 +900,7 @@ chalk@^2.4.2: escape-string-regexp "^1.0.5" supports-color "^5.3.0" -chalk@^4.0.0, chalk@^4.1.0: +chalk@^4.0.0, chalk@^4.1.0, chalk@^4.1.2: version "4.1.2" resolved "https://registry.yarnpkg.com/chalk/-/chalk-4.1.2.tgz#aac4e2b7734a740867aeb16bf02aad556a1e7a01" integrity sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA== @@ -1390,6 +1390,16 @@ es-get-iterator@^1.1.3: isarray "^2.0.5" stop-iteration-iterator "^1.0.0" +esbuild-plugin-copy@^2.1.1: + version "2.1.1" + resolved "https://registry.yarnpkg.com/esbuild-plugin-copy/-/esbuild-plugin-copy-2.1.1.tgz#638308ecfd679e4c7c76b71c62f7dd9a4cc7f901" + integrity sha512-Bk66jpevTcV8KMFzZI1P7MZKZ+uDcrZm2G2egZ2jNIvVnivDpodZI+/KnpL3Jnap0PBdIHU7HwFGB8r+vV5CVw== + dependencies: + chalk "^4.1.2" + chokidar "^3.5.3" + fs-extra "^10.0.1" + globby "^11.0.3" + esbuild-plugin-polyfill-node@^0.3.0: version "0.3.0" resolved "https://registry.yarnpkg.com/esbuild-plugin-polyfill-node/-/esbuild-plugin-polyfill-node-0.3.0.tgz#e7e3804b8272df51ae4f8ebfb7445a03712504cb" @@ -1706,6 +1716,15 @@ fs-constants@^1.0.0: resolved "https://registry.yarnpkg.com/fs-constants/-/fs-constants-1.0.0.tgz#6be0de9be998ce16af8afc24497b9ee9b7ccd9ad" integrity sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow== +fs-extra@^10.0.1: + version "10.1.0" + resolved "https://registry.yarnpkg.com/fs-extra/-/fs-extra-10.1.0.tgz#02873cfbc4084dde127eaa5f9905eef2325d1abf" + integrity sha512-oRXApq54ETRj4eMiFzGnHWGy+zo5raudjuxN0b8H7s/RU2oW0Wvsx9O0ACRN/kRq9E8Vu/ReskGB5o3ji+FzHQ== + dependencies: + graceful-fs "^4.2.0" + jsonfile "^6.0.1" + universalify "^2.0.0" + fs-extra@^11.1.1: version "11.1.1" resolved "https://registry.yarnpkg.com/fs-extra/-/fs-extra-11.1.1.tgz#da69f7c39f3b002378b0954bb6ae7efdc0876e2d" @@ -4117,6 +4136,11 @@ vscode-uri@^3.0.7: resolved "https://registry.yarnpkg.com/vscode-uri/-/vscode-uri-3.0.7.tgz#6d19fef387ee6b46c479e5fb00870e15e58c1eb8" integrity sha512-eOpPHogvorZRobNqJGhapa0JdwaxpjVvyBp0QIUMRMSf8ZAlqOdEquKuRmw9Qwu0qXtJIWqFtMkmvJjUZmMjVA== +web-tree-sitter@^0.20.8: + version "0.20.8" + resolved "https://registry.yarnpkg.com/web-tree-sitter/-/web-tree-sitter-0.20.8.tgz#1e371cb577584789cadd75cb49c7ddfbc99d04c8" + integrity sha512-weOVgZ3aAARgdnb220GqYuh7+rZU0Ka9k9yfKtGAzEYMa6GgiCzW9JjQRJyCJakvibQW+dfjJdihjInKuuCAUQ== + webidl-conversions@^4.0.2: version "4.0.2" resolved "https://registry.yarnpkg.com/webidl-conversions/-/webidl-conversions-4.0.2.tgz#a855980b1f0b6b359ba1d5d9fb39ae941faa63ad"