feat(agent): add experimental postprocess limit scope by syntax. (#758)
* feat(agent): add tree-sitter parser. * feat(agent): make parser updating tree cache optional. * feat(agent): add experimental limit scopy by syntax. * test(agent): update unit test for limitScopeBySyntax.extract-routes
Zhiming Ma
GitHub
parent
d7180ec7b9
commit
568b7b41a5
|
|
@ -18,6 +18,8 @@ jobs:
|
|||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v3
|
||||
with:
|
||||
lfs: true
|
||||
- name: Setup JDK
|
||||
uses: actions/setup-java@v3
|
||||
with:
|
||||
|
|
|
|||
|
|
@ -22,6 +22,8 @@ jobs:
|
|||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v3
|
||||
with:
|
||||
lfs: true
|
||||
- name: Setup Node.js
|
||||
uses: actions/setup-node@v3
|
||||
with:
|
||||
|
|
|
|||
|
|
@ -24,6 +24,8 @@ jobs:
|
|||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v3
|
||||
with:
|
||||
lfs: true
|
||||
- name: Setup Node.js
|
||||
uses: actions/setup-node@v3
|
||||
with:
|
||||
|
|
|
|||
|
|
@ -0,0 +1 @@
|
|||
*.wasm filter=lfs diff=lfs merge=lfs -text
|
||||
|
|
@ -23,11 +23,11 @@
|
|||
"@types/node": "^18.12.0",
|
||||
"chai": "^4.3.7",
|
||||
"dedent": "^0.7.0",
|
||||
"esbuild-plugin-copy": "^2.1.1",
|
||||
"esbuild-plugin-polyfill-node": "^0.3.0",
|
||||
"mocha": "^10.2.0",
|
||||
"openapi-typescript": "^6.6.1",
|
||||
"prettier": "^3.0.0",
|
||||
"rimraf": "^5.0.1",
|
||||
"ts-node": "^10.9.1",
|
||||
"tsup": "^7.1.0",
|
||||
"typescript": "^5.0.3"
|
||||
|
|
@ -47,6 +47,7 @@
|
|||
"rotating-file-stream": "^3.1.0",
|
||||
"stats-logscale": "^1.0.7",
|
||||
"toml": "^3.0.0",
|
||||
"uuid": "^9.0.0"
|
||||
"uuid": "^9.0.0",
|
||||
"web-tree-sitter": "^0.20.8"
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -21,11 +21,15 @@ export type AgentConfig = {
|
|||
timeout: number;
|
||||
};
|
||||
postprocess: {
|
||||
limitScopeByIndentation: {
|
||||
// When completion is continuing the current line, limit the scope to:
|
||||
// false(default): the line scope, meaning use the next indent level as the limit.
|
||||
// true: the block scope, meaning use the current indent level as the limit.
|
||||
experimentalKeepBlockScopeWhenCompletingLine: boolean;
|
||||
limitScope: {
|
||||
// Prefer to use syntax parser than indentation
|
||||
experimentalSyntax: boolean;
|
||||
indentation: {
|
||||
// When completion is continuing the current line, limit the scope to:
|
||||
// false(default): the line scope, meaning use the next indent level as the limit.
|
||||
// true: the block scope, meaning use the current indent level as the limit.
|
||||
experimentalKeepBlockScopeWhenCompletingLine: boolean;
|
||||
};
|
||||
};
|
||||
};
|
||||
logs: {
|
||||
|
|
@ -66,8 +70,11 @@ export const defaultAgentConfig: AgentConfig = {
|
|||
timeout: 4000, // ms
|
||||
},
|
||||
postprocess: {
|
||||
limitScopeByIndentation: {
|
||||
experimentalKeepBlockScopeWhenCompletingLine: false,
|
||||
limitScope: {
|
||||
experimentalSyntax: false,
|
||||
indentation: {
|
||||
experimentalKeepBlockScopeWhenCompletingLine: false,
|
||||
},
|
||||
},
|
||||
},
|
||||
logs: {
|
||||
|
|
|
|||
|
|
@ -529,7 +529,7 @@ export class TabbyAgent extends EventEmitter implements Agent {
|
|||
throw options.signal.reason;
|
||||
}
|
||||
// Build cache
|
||||
this.completionCache.buildCache(context, completionResponse);
|
||||
this.completionCache.buildCache(context, JSON.parse(JSON.stringify(completionResponse)));
|
||||
}
|
||||
}
|
||||
// Postprocess (post-cache)
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ import { applyFilter } from "./base";
|
|||
import { removeRepetitiveBlocks } from "./removeRepetitiveBlocks";
|
||||
import { removeRepetitiveLines } from "./removeRepetitiveLines";
|
||||
import { removeLineEndsWithRepetition } from "./removeLineEndsWithRepetition";
|
||||
import { limitScopeByIndentation } from "./limitScopeByIndentation";
|
||||
import { limitScope } from "./limitScope";
|
||||
import { trimSpace } from "./trimSpace";
|
||||
import { dropDuplicated } from "./dropDuplicated";
|
||||
import { dropBlank } from "./dropBlank";
|
||||
|
|
@ -30,7 +30,7 @@ export async function postCacheProcess(
|
|||
return Promise.resolve(response)
|
||||
.then(applyFilter(removeRepetitiveBlocks(context), context))
|
||||
.then(applyFilter(removeRepetitiveLines(context), context))
|
||||
.then(applyFilter(limitScopeByIndentation(context, config["limitScopeByIndentation"]), context))
|
||||
.then(applyFilter(limitScope(context, config["limitScope"]), context))
|
||||
.then(applyFilter(dropDuplicated(context), context))
|
||||
.then(applyFilter(trimSpace(context), context))
|
||||
.then(applyFilter(dropBlank(), context));
|
||||
|
|
|
|||
|
|
@ -0,0 +1,24 @@
|
|||
import { CompletionContext } from "../Agent";
|
||||
import { AgentConfig } from "../AgentConfig";
|
||||
import { isBrowser } from "../env";
|
||||
import { PostprocessFilter } from "./base";
|
||||
import { limitScopeByIndentation } from "./limitScopeByIndentation";
|
||||
import { limitScopeBySyntax, supportedLanguages } from "./limitScopeBySyntax";
|
||||
|
||||
export function limitScope(
|
||||
context: CompletionContext,
|
||||
config: AgentConfig["postprocess"]["limitScope"],
|
||||
): PostprocessFilter {
|
||||
return isBrowser
|
||||
? (input) => {
|
||||
// syntax parser is not supported in browser yet
|
||||
return limitScopeByIndentation(context, config["indentation"])(input);
|
||||
}
|
||||
: (input) => {
|
||||
if (config.experimentalSyntax && supportedLanguages.indexOf(context.language) >= 0) {
|
||||
return limitScopeBySyntax(context)(input);
|
||||
} else {
|
||||
return limitScopeByIndentation(context, config["indentation"])(input);
|
||||
}
|
||||
};
|
||||
}
|
||||
|
|
@ -18,7 +18,7 @@ function processContext(
|
|||
lines: string[],
|
||||
prefixLines: string[],
|
||||
suffixLines: string[],
|
||||
config: AgentConfig["postprocess"]["limitScopeByIndentation"],
|
||||
config: AgentConfig["postprocess"]["limitScope"]["indentation"],
|
||||
): { indentLevelLimit: number; allowClosingLine: (closingLine: string) => boolean } {
|
||||
let allowClosingLine = false;
|
||||
let result = { indentLevelLimit: 0, allowClosingLine: (closingLine: string) => allowClosingLine };
|
||||
|
|
@ -103,14 +103,14 @@ function processContext(
|
|||
|
||||
export function limitScopeByIndentation(
|
||||
context: CompletionContext,
|
||||
config: AgentConfig["postprocess"]["limitScopeByIndentation"],
|
||||
config: AgentConfig["postprocess"]["limitScope"]["indentation"],
|
||||
): PostprocessFilter {
|
||||
return (input) => {
|
||||
const { prefix, suffix, prefixLines, suffixLines } = context;
|
||||
const { prefixLines, suffixLines } = context;
|
||||
const inputLines = splitLines(input);
|
||||
if (context.mode === "fill-in-line") {
|
||||
if (inputLines.length > 1) {
|
||||
logger.debug({ input, prefix, suffix }, "Drop content with multiple lines");
|
||||
logger.debug({ inputLines, prefixLines, suffixLines }, "Drop content with multiple lines");
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
|
@ -139,7 +139,10 @@ export function limitScopeByIndentation(
|
|||
}
|
||||
}
|
||||
if (index < inputLines.length) {
|
||||
logger.debug({ input, prefix, suffix, scopeEndAt: index }, "Remove content out of scope");
|
||||
logger.debug(
|
||||
{ inputLines, prefixLines, suffixLines, scopeLineCount: index },
|
||||
"Remove content out of indent scope",
|
||||
);
|
||||
return inputLines.slice(0, index).join("").trimEnd();
|
||||
}
|
||||
return input;
|
||||
|
|
|
|||
|
|
@ -0,0 +1,409 @@
|
|||
import { expect } from "chai";
|
||||
import { documentContext, inline } from "./testUtils";
|
||||
import { limitScopeBySyntax } from "./limitScopeBySyntax";
|
||||
|
||||
describe("postprocess", () => {
|
||||
describe("limitScopeBySyntax javascript", () => {
|
||||
it("should limit scope at function_declaration.", async () => {
|
||||
const context = {
|
||||
...documentContext`
|
||||
function findMax(arr) {║}
|
||||
`,
|
||||
language: "javascript",
|
||||
};
|
||||
const completion = inline`
|
||||
├
|
||||
let max = arr[0];
|
||||
for (let i = 1; i < arr.length; i++) {
|
||||
if (arr[i] > max) {
|
||||
max = arr[i];
|
||||
}
|
||||
}
|
||||
return max;
|
||||
}
|
||||
console.log(findMax([1, 2, 3, 4, 5]));┤
|
||||
`;
|
||||
const expected = inline`
|
||||
├
|
||||
let max = arr[0];
|
||||
for (let i = 1; i < arr.length; i++) {
|
||||
if (arr[i] > max) {
|
||||
max = arr[i];
|
||||
}
|
||||
}
|
||||
return max;
|
||||
}┤
|
||||
`;
|
||||
expect(await limitScopeBySyntax(context)(completion)).to.eq(expected);
|
||||
});
|
||||
|
||||
it("should limit scope at function_declaration", async () => {
|
||||
const context = {
|
||||
...documentContext`
|
||||
function findMax(arr) {
|
||||
let max = arr[0];║
|
||||
}
|
||||
`,
|
||||
language: "javascript",
|
||||
};
|
||||
const completion = inline`
|
||||
├
|
||||
for (let i = 1; i < arr.length; i++) {
|
||||
if (arr[i] > max) {
|
||||
max = arr[i];
|
||||
}
|
||||
}
|
||||
return max;
|
||||
}┤
|
||||
`;
|
||||
expect(await limitScopeBySyntax(context)(completion)).to.eq(completion);
|
||||
});
|
||||
|
||||
it("should limit scope at function_declaration", async () => {
|
||||
const context = {
|
||||
...documentContext`
|
||||
function findMax(arr) {
|
||||
let max = arr[0];
|
||||
for (let i = 1; i < arr.length; i++) {
|
||||
if (arr[i] > max) {
|
||||
max = arr[i];
|
||||
}
|
||||
}║
|
||||
}
|
||||
`,
|
||||
language: "javascript",
|
||||
};
|
||||
const completion = inline`
|
||||
├
|
||||
return max;
|
||||
}┤
|
||||
`;
|
||||
expect(await limitScopeBySyntax(context)(completion)).to.eq(completion);
|
||||
});
|
||||
|
||||
it("should limit scope at for_statement.", async () => {
|
||||
const context = {
|
||||
...documentContext`
|
||||
function findMax(arr) {
|
||||
let max = arr[0];
|
||||
for║
|
||||
}
|
||||
`,
|
||||
language: "javascript",
|
||||
};
|
||||
const completion = inline`
|
||||
├ (let i = 1; i < arr.length; i++) {
|
||||
if (arr[i] > max) {
|
||||
max = arr[i];
|
||||
}
|
||||
}
|
||||
return max;
|
||||
}
|
||||
console.log(findMax([1, 2, 3, 4, 5]));┤
|
||||
`;
|
||||
const expected = inline`
|
||||
├ (let i = 1; i < arr.length; i++) {
|
||||
if (arr[i] > max) {
|
||||
max = arr[i];
|
||||
}
|
||||
}┤
|
||||
┴┴
|
||||
`;
|
||||
expect(await limitScopeBySyntax(context)(completion)).to.eq(expected);
|
||||
});
|
||||
|
||||
it("should limit scope at current node if no parent scope found.", async () => {
|
||||
const context = {
|
||||
...documentContext`
|
||||
let a =║
|
||||
`,
|
||||
language: "javascript",
|
||||
};
|
||||
const completion = inline`
|
||||
├ 1;
|
||||
let b = 2;┤
|
||||
`;
|
||||
const expected = inline`
|
||||
├ 1;┤
|
||||
`;
|
||||
expect(await limitScopeBySyntax(context)(completion)).to.eq(expected);
|
||||
});
|
||||
|
||||
it("should handle the bad case of limitScopeByIndentation", async () => {
|
||||
const context = {
|
||||
...documentContext`
|
||||
function sortWords(input) {
|
||||
const output = input.trim()
|
||||
.split("\n")
|
||||
.map((line) => line.split(" "))
|
||||
║
|
||||
}
|
||||
`,
|
||||
language: "javascript",
|
||||
};
|
||||
const completion = inline`
|
||||
├.flat()
|
||||
.sort()
|
||||
.join(" ");
|
||||
console.log(output);
|
||||
return output;
|
||||
}
|
||||
sortWord("world hello");┤
|
||||
`;
|
||||
const expected = inline`
|
||||
├.flat()
|
||||
.sort()
|
||||
.join(" ");
|
||||
console.log(output);
|
||||
return output;
|
||||
};┤
|
||||
`;
|
||||
expect(await limitScopeBySyntax(context)(completion)).not.to.eq(expected);
|
||||
});
|
||||
});
|
||||
|
||||
describe("limitScopeBySyntax python", () => {
|
||||
it("should limit scope at function_definition.", async () => {
|
||||
const context = {
|
||||
...documentContext`
|
||||
def find_min(arr):║
|
||||
`,
|
||||
language: "python",
|
||||
};
|
||||
const completion = inline`
|
||||
├
|
||||
min = arr[0]
|
||||
for i in range(1, len(arr)):
|
||||
if arr[i] < min:
|
||||
min = arr[i]
|
||||
return min
|
||||
print(find_min([1, 2, 3, 4, 5]))┤
|
||||
`;
|
||||
const expected = inline`
|
||||
├
|
||||
min = arr[0]
|
||||
for i in range(1, len(arr)):
|
||||
if arr[i] < min:
|
||||
min = arr[i]
|
||||
return min┤
|
||||
┴┴
|
||||
`;
|
||||
expect(await limitScopeBySyntax(context)(completion)).to.eq(expected);
|
||||
});
|
||||
|
||||
it("should limit scope at function_definition.", async () => {
|
||||
const context = {
|
||||
...documentContext`
|
||||
def find_min(arr):
|
||||
min = arr[0]║
|
||||
`,
|
||||
language: "python",
|
||||
};
|
||||
const completion = inline`
|
||||
├
|
||||
for i in range(1, len(arr)):
|
||||
if arr[i] < min:
|
||||
min = arr[i]
|
||||
return min
|
||||
print(find_min([1, 2, 3, 4, 5]))┤
|
||||
`;
|
||||
const expected = inline`
|
||||
├
|
||||
for i in range(1, len(arr)):
|
||||
if arr[i] < min:
|
||||
min = arr[i]
|
||||
return min┤
|
||||
┴┴
|
||||
`;
|
||||
expect(await limitScopeBySyntax(context)(completion)).to.eq(expected);
|
||||
});
|
||||
|
||||
it("should limit scope at function_definition.", async () => {
|
||||
const context = {
|
||||
...documentContext`
|
||||
def find_min(arr):
|
||||
min = arr[0]
|
||||
for i in range(1, len(arr)):
|
||||
if arr[i] < min:
|
||||
min = arr[i]║
|
||||
`,
|
||||
language: "python",
|
||||
};
|
||||
const completion = inline`
|
||||
├
|
||||
return min┤
|
||||
`;
|
||||
expect(await limitScopeBySyntax(context)(completion)).to.eq(completion);
|
||||
});
|
||||
|
||||
it("should limit scope at for_statement.", async () => {
|
||||
const context = {
|
||||
...documentContext`
|
||||
def find_min(arr):
|
||||
max = arr[0]
|
||||
for║
|
||||
`,
|
||||
language: "python",
|
||||
};
|
||||
const completion = inline`
|
||||
├ i in range(1, len(arr)):
|
||||
if arr[i] < min:
|
||||
min = arr[i]
|
||||
return min
|
||||
┤
|
||||
`;
|
||||
const expected = inline`
|
||||
├ i in range(1, len(arr)):
|
||||
if arr[i] < min:
|
||||
min = arr[i]┤
|
||||
┴┴┴┴
|
||||
`;
|
||||
expect(await limitScopeBySyntax(context)(completion)).to.eq(expected);
|
||||
});
|
||||
|
||||
it("should handle the bad case of limitScopeByIndentation", async () => {
|
||||
const context = {
|
||||
...documentContext`
|
||||
def findMax(arr):
|
||||
║
|
||||
`,
|
||||
language: "python",
|
||||
};
|
||||
const completion = inline`
|
||||
├max = arr[0]
|
||||
for i in range(1, len(arr)):
|
||||
if arr[i] > max:
|
||||
max = arr[i]
|
||||
return max
|
||||
findMax([1, 2, 3, 4, 5])┤
|
||||
`;
|
||||
const expected = inline`
|
||||
├max = arr[0]
|
||||
for i in range(1, len(arr)):
|
||||
if arr[i] > max:
|
||||
max = arr[i]
|
||||
return max┤
|
||||
`;
|
||||
expect(await limitScopeBySyntax(context)(completion)).not.to.eq(expected);
|
||||
});
|
||||
});
|
||||
|
||||
describe("limitScopeBySyntax go", () => {
|
||||
it("should limit scope at function_declaration.", async () => {
|
||||
const context = {
|
||||
...documentContext`
|
||||
func findMin(arr []int) int {║}
|
||||
`,
|
||||
language: "go",
|
||||
};
|
||||
const completion = inline`
|
||||
├
|
||||
min := math.MaxInt64
|
||||
for _, v := range arr {
|
||||
if v < min {
|
||||
min = v
|
||||
}
|
||||
}
|
||||
return min
|
||||
}
|
||||
|
||||
func main() {
|
||||
arr := []int{5, 2, 9, 8, 1, 3}
|
||||
fmt.Println(findMin(arr)) // Output: 1
|
||||
}┤
|
||||
`;
|
||||
const expected = inline`
|
||||
├
|
||||
min := math.MaxInt64
|
||||
for _, v := range arr {
|
||||
if v < min {
|
||||
min = v
|
||||
}
|
||||
}
|
||||
return min
|
||||
}┤
|
||||
`;
|
||||
expect(await limitScopeBySyntax(context)(completion)).to.eq(expected);
|
||||
});
|
||||
|
||||
it("should limit scope at for_statement.", async () => {
|
||||
const context = {
|
||||
...documentContext`
|
||||
func findMin(arr []int) int {
|
||||
min := math.MaxInt64
|
||||
for║
|
||||
`,
|
||||
language: "go",
|
||||
};
|
||||
const completion = inline`
|
||||
├ _, v := range arr {
|
||||
if v < min {
|
||||
min = v
|
||||
}
|
||||
}
|
||||
return min
|
||||
}┤
|
||||
`;
|
||||
const expected = inline`
|
||||
├ _, v := range arr {
|
||||
if v < min {
|
||||
min = v
|
||||
}
|
||||
}┤
|
||||
┴┴
|
||||
`;
|
||||
expect(await limitScopeBySyntax(context)(completion)).to.eq(expected);
|
||||
});
|
||||
});
|
||||
|
||||
describe("limitScopeBySyntax rust", () => {
|
||||
it("should limit scope at function_item.", async () => {
|
||||
const context = {
|
||||
...documentContext`
|
||||
fn find_min(arr: &[i32]) -> i32 {║}
|
||||
`,
|
||||
language: "rust",
|
||||
};
|
||||
const completion = inline`
|
||||
├
|
||||
*arr.iter().min().unwrap()
|
||||
}
|
||||
fn main() {
|
||||
let arr = vec![5, 2, 9, 8, 1, 3];
|
||||
println!("{}", find_min(&arr)); // Output: 1
|
||||
}┤
|
||||
`;
|
||||
const expected = inline`
|
||||
├
|
||||
*arr.iter().min().unwrap()
|
||||
}┤
|
||||
`;
|
||||
expect(await limitScopeBySyntax(context)(completion)).to.eq(expected);
|
||||
});
|
||||
});
|
||||
|
||||
describe("limitScopeBySyntax ruby", () => {
|
||||
it("should limit scope at for.", async () => {
|
||||
const context = {
|
||||
...documentContext`
|
||||
def fibonacci(n)║
|
||||
`,
|
||||
language: "ruby",
|
||||
};
|
||||
const completion = inline`
|
||||
├
|
||||
return n if n <= 1
|
||||
fibonacci(n - 1) + fibonacci(n - 2)
|
||||
end
|
||||
puts fibonacci(10)┤
|
||||
`;
|
||||
const expected = inline`
|
||||
├
|
||||
return n if n <= 1
|
||||
fibonacci(n - 1) + fibonacci(n - 2)
|
||||
end┤
|
||||
`;
|
||||
expect(await limitScopeBySyntax(context)(completion)).to.eq(expected);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,86 @@
|
|||
import type TreeSitterParser from "web-tree-sitter";
|
||||
import { getParser, languagesConfigs } from "../syntax/parser";
|
||||
import { typeList } from "../syntax/typeList";
|
||||
import { CompletionContext } from "../Agent";
|
||||
import { PostprocessFilter, logger } from "./base";
|
||||
|
||||
export const supportedLanguages = Object.keys(languagesConfigs);
|
||||
|
||||
function findLineBegin(text: string, position: number): number {
|
||||
let lastNonBlankCharPos = position - 1;
|
||||
while (lastNonBlankCharPos >= 0 && text[lastNonBlankCharPos].match(/\s/)) {
|
||||
lastNonBlankCharPos--;
|
||||
}
|
||||
if (lastNonBlankCharPos < 0) {
|
||||
return 0;
|
||||
}
|
||||
const lineBegin = text.lastIndexOf("\n", lastNonBlankCharPos);
|
||||
if (lineBegin < 0) {
|
||||
return 0;
|
||||
}
|
||||
const line = text.slice(lineBegin + 1, position);
|
||||
const indentation = line.search(/\S/);
|
||||
return lineBegin + 1 + indentation;
|
||||
}
|
||||
|
||||
function findLineEnd(text: string, position: number): number {
|
||||
let firstNonBlankCharPos = position;
|
||||
while (firstNonBlankCharPos < text.length && text[firstNonBlankCharPos].match(/\s/)) {
|
||||
firstNonBlankCharPos++;
|
||||
}
|
||||
if (firstNonBlankCharPos >= text.length) {
|
||||
return text.length;
|
||||
}
|
||||
const lineEnd = text.indexOf("\n", firstNonBlankCharPos);
|
||||
if (lineEnd < 0) {
|
||||
return text.length;
|
||||
}
|
||||
return lineEnd;
|
||||
}
|
||||
|
||||
function findScope(node: TreeSitterParser.SyntaxNode, typeList: string[][]): TreeSitterParser.SyntaxNode {
|
||||
for (const types of typeList) {
|
||||
let scope = node;
|
||||
while (scope) {
|
||||
if (types.indexOf(scope.type) >= 0) {
|
||||
return scope;
|
||||
}
|
||||
scope = scope.parent;
|
||||
}
|
||||
}
|
||||
return node;
|
||||
}
|
||||
|
||||
export function limitScopeBySyntax(context: CompletionContext): PostprocessFilter {
|
||||
return async (input) => {
|
||||
const { position, text, language, prefix, suffix } = context;
|
||||
if (supportedLanguages.indexOf(language) < 0) {
|
||||
return input;
|
||||
}
|
||||
const languageConfig = languagesConfigs[language];
|
||||
const parser = await getParser(languageConfig);
|
||||
|
||||
const updatedText = prefix + input + suffix;
|
||||
const updatedTree = parser.parse(updatedText);
|
||||
const lineBegin = findLineBegin(updatedText, position);
|
||||
const lineEnd = findLineEnd(updatedText, position);
|
||||
const scope = findScope(updatedTree.rootNode.namedDescendantForIndex(lineBegin, lineEnd), typeList[languageConfig]);
|
||||
|
||||
if (scope.endIndex < position + input.length) {
|
||||
logger.debug(
|
||||
{
|
||||
languageConfig,
|
||||
text,
|
||||
updatedText,
|
||||
position,
|
||||
lineBegin,
|
||||
lineEnd,
|
||||
scope: { type: scope.type, start: scope.startIndex, end: scope.endIndex },
|
||||
},
|
||||
"Remove content out of syntax scope",
|
||||
);
|
||||
return input.slice(0, scope.endIndex - position);
|
||||
}
|
||||
return input;
|
||||
};
|
||||
}
|
||||
|
|
@ -1,12 +1,13 @@
|
|||
import dedent from "dedent";
|
||||
import { v4 as uuid } from "uuid";
|
||||
import { CompletionContext } from "../CompletionContext";
|
||||
|
||||
// `║` is the cursor position
|
||||
export function documentContext(strings): CompletionContext {
|
||||
const doc = dedent(strings);
|
||||
return new CompletionContext({
|
||||
filepath: null,
|
||||
language: null,
|
||||
filepath: uuid(),
|
||||
language: "",
|
||||
text: doc.replace(/║/, ""),
|
||||
position: doc.indexOf("║"),
|
||||
});
|
||||
|
|
|
|||
|
|
@ -0,0 +1,45 @@
|
|||
import TreeSitterParser from "web-tree-sitter";
|
||||
import { isTest } from "../env";
|
||||
|
||||
// https://code.visualstudio.com/docs/languages/identifiers
|
||||
export const languagesConfigs: Record<string, string> = {
|
||||
javascript: "tsx",
|
||||
typescript: "tsx",
|
||||
javascriptreact: "tsx",
|
||||
typescriptreact: "tsx",
|
||||
python: "python",
|
||||
go: "go",
|
||||
rust: "rust",
|
||||
ruby: "ruby",
|
||||
};
|
||||
|
||||
var treeSitterInitialized = false;
|
||||
|
||||
async function createParser(languageConfig: string): Promise<TreeSitterParser> {
|
||||
if (!treeSitterInitialized) {
|
||||
await TreeSitterParser.init({
|
||||
locateFile(scriptName: string, scriptDirectory: string) {
|
||||
const paths = isTest ? [scriptDirectory, scriptName] : [scriptDirectory, "wasm", scriptName];
|
||||
return require("path").join(...paths);
|
||||
},
|
||||
});
|
||||
treeSitterInitialized = true;
|
||||
}
|
||||
const parser = new TreeSitterParser();
|
||||
const langWasmPaths = isTest
|
||||
? [process.cwd(), "wasm", `tree-sitter-${languageConfig}.wasm`]
|
||||
: [__dirname, "wasm", `tree-sitter-${languageConfig}.wasm`];
|
||||
parser.setLanguage(await TreeSitterParser.Language.load(require("path").join(...langWasmPaths)));
|
||||
return parser;
|
||||
}
|
||||
|
||||
const parsers = new Map<string, TreeSitterParser>();
|
||||
|
||||
export async function getParser(languageConfig: string): Promise<TreeSitterParser> {
|
||||
let parser = parsers.get(languageConfig);
|
||||
if (!parser) {
|
||||
parser = await createParser(languageConfig);
|
||||
parsers.set(languageConfig, parser);
|
||||
}
|
||||
return parser;
|
||||
}
|
||||
|
|
@ -0,0 +1,116 @@
|
|||
// https://github.com/tree-sitter/tree-sitter-typescript/blob/master/src/node-types.json
|
||||
export const typeList = {
|
||||
tsx: [
|
||||
[
|
||||
"jsx_element",
|
||||
"jsx_self_closing_element",
|
||||
|
||||
// exclude sentence level nodes for now
|
||||
// "expression_statement",
|
||||
// "lexical_declaration",
|
||||
|
||||
"for_statement",
|
||||
"for_in_statement",
|
||||
"if_statement",
|
||||
"while_statement",
|
||||
"do_statement",
|
||||
"switch_statement",
|
||||
"try_statement",
|
||||
"with_statement",
|
||||
"labeled_statement",
|
||||
|
||||
"class_declaration",
|
||||
"abstract_class_declaration",
|
||||
"interface_declaration",
|
||||
"enum_declaration",
|
||||
"type_alias_declaration",
|
||||
"function_declaration",
|
||||
"generator_function_declaration",
|
||||
"ambient_declaration",
|
||||
|
||||
"method_definition",
|
||||
|
||||
"import_statement",
|
||||
"export_statement",
|
||||
"module",
|
||||
],
|
||||
],
|
||||
|
||||
// https://github.com/tree-sitter/tree-sitter-python/blob/master/src/node-types.json
|
||||
python: [
|
||||
[
|
||||
"for_statement",
|
||||
"if_statement",
|
||||
"while_statement",
|
||||
"match_statement",
|
||||
"try_statement",
|
||||
"with_statement",
|
||||
|
||||
"function_definition",
|
||||
"decorated_definition",
|
||||
"class_definition",
|
||||
|
||||
"import_statement",
|
||||
"import_from_statement",
|
||||
],
|
||||
],
|
||||
|
||||
// https://github.com/tree-sitter/tree-sitter-go/blob/master/src/node-types.json
|
||||
go: [
|
||||
[
|
||||
"for_statement",
|
||||
"if_statement",
|
||||
"expression_switch_statement",
|
||||
"type_switch_statement",
|
||||
"select_statement",
|
||||
"labeled_statement",
|
||||
|
||||
"function_declaration",
|
||||
"method_declaration",
|
||||
"type_declaration",
|
||||
|
||||
"import_declaration",
|
||||
"package_clause",
|
||||
],
|
||||
],
|
||||
|
||||
// https://github.com/tree-sitter/tree-sitter-rust/blob/master/src/node-types.json
|
||||
rust: [
|
||||
[
|
||||
"for_expression",
|
||||
"if_expression",
|
||||
"while_expression",
|
||||
"loop_expression",
|
||||
"match_expression",
|
||||
"try_expression",
|
||||
|
||||
"function_item",
|
||||
"type_item",
|
||||
"enum_item",
|
||||
"struct_item",
|
||||
"union_item",
|
||||
"trait_item",
|
||||
"impl_item",
|
||||
|
||||
"use_declaration",
|
||||
],
|
||||
],
|
||||
|
||||
// https://github.com/tree-sitter/tree-sitter-ruby/blob/master/src/node-types.json
|
||||
ruby: [
|
||||
[
|
||||
"for",
|
||||
"if",
|
||||
"unless",
|
||||
"while",
|
||||
"until",
|
||||
"case",
|
||||
|
||||
"class",
|
||||
"singleton_class",
|
||||
"method",
|
||||
"singleton_method",
|
||||
"module",
|
||||
],
|
||||
],
|
||||
};
|
||||
|
|
@ -81,7 +81,10 @@ describe("agent golden test", () => {
|
|||
timeout: 4000,
|
||||
},
|
||||
postprocess: {
|
||||
limitScopeByIndentation: { experimentalKeepBlockScopeWhenCompletingLine: false },
|
||||
limitScope: {
|
||||
experimentalSyntax: false,
|
||||
indentation: { experimentalKeepBlockScopeWhenCompletingLine: false },
|
||||
},
|
||||
},
|
||||
logs: { level: "debug" },
|
||||
anonymousUsageTracking: { disable: true },
|
||||
|
|
@ -111,7 +114,7 @@ describe("agent golden test", () => {
|
|||
absolutePath: path.join(__dirname, "golden", file),
|
||||
};
|
||||
});
|
||||
goldenFiles.forEach((goldenFile, index) => {
|
||||
goldenFiles.forEach((goldenFile) => {
|
||||
it(goldenFile.path, async () => {
|
||||
const test = await createGoldenTest(goldenFile.absolutePath);
|
||||
requestId++;
|
||||
|
|
@ -130,7 +133,7 @@ describe("agent golden test", () => {
|
|||
absolutePath: path.join(__dirname, "bad_cases", file),
|
||||
};
|
||||
});
|
||||
badCasesFiles.forEach((goldenFile, index) => {
|
||||
badCasesFiles.forEach((goldenFile) => {
|
||||
it(goldenFile.path, async () => {
|
||||
const test = await createGoldenTest(goldenFile.absolutePath);
|
||||
requestId++;
|
||||
|
|
@ -143,6 +146,35 @@ describe("agent golden test", () => {
|
|||
});
|
||||
});
|
||||
|
||||
it("updateConfig experimental", async () => {
|
||||
requestId++;
|
||||
const updateConfigRequest = [
|
||||
requestId,
|
||||
{
|
||||
func: "updateConfig",
|
||||
args: ["postprocess.limitScope.experimentalSyntax", true],
|
||||
},
|
||||
];
|
||||
agent.stdin.write(JSON.stringify(updateConfigRequest) + "\n");
|
||||
await waitForResponse(requestId);
|
||||
const expectedConfig = { ...config };
|
||||
expectedConfig.postprocess.limitScope.experimentalSyntax = true;
|
||||
expect(output.shift()).to.deep.equal([0, { event: "configUpdated", config: expectedConfig }]);
|
||||
expect(output.shift()).to.deep.equal([requestId, true]);
|
||||
});
|
||||
badCasesFiles.forEach((goldenFile) => {
|
||||
it("experimental: " + goldenFile.path, async () => {
|
||||
const test = await createGoldenTest(goldenFile.absolutePath);
|
||||
requestId++;
|
||||
const request = [requestId, { func: "provideCompletions", args: [test.request] }];
|
||||
agent.stdin.write(JSON.stringify(request) + "\n");
|
||||
await waitForResponse(requestId);
|
||||
const response = output.shift();
|
||||
expect(response[0]).to.equal(requestId);
|
||||
expect(response[1].choices).to.deep.equal(test.expected.choices);
|
||||
});
|
||||
});
|
||||
|
||||
after(() => {
|
||||
agent.kill();
|
||||
});
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
import { defineConfig } from "tsup";
|
||||
import { copy } from "esbuild-plugin-copy";
|
||||
import { polyfillNode } from "esbuild-plugin-polyfill-node";
|
||||
import { dependencies } from "./package.json";
|
||||
|
||||
|
|
@ -16,6 +17,7 @@ export default async () => [
|
|||
name: "node-cjs",
|
||||
entry: ["src/index.ts"],
|
||||
platform: "node",
|
||||
target: "node18",
|
||||
format: ["cjs"],
|
||||
sourcemap: true,
|
||||
esbuildOptions(options) {
|
||||
|
|
@ -71,10 +73,21 @@ export default async () => [
|
|||
name: "cli",
|
||||
entry: ["src/cli.ts"],
|
||||
platform: "node",
|
||||
target: "node18",
|
||||
noExternal: Object.keys(dependencies),
|
||||
treeshake: "smallest",
|
||||
minify: true,
|
||||
sourcemap: true,
|
||||
esbuildPlugins: [
|
||||
copy({
|
||||
assets: [
|
||||
{
|
||||
from: "./wasm/*",
|
||||
to: "./wasm",
|
||||
},
|
||||
],
|
||||
}),
|
||||
],
|
||||
esbuildOptions(options) {
|
||||
defineEnvs(options, { browser: false });
|
||||
},
|
||||
|
|
|
|||
|
|
@ -0,0 +1,3 @@
|
|||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:1b69c5af834fd23053238e484c7fe9ed2f121d5b1fe32242af78576d67e49f1e
|
||||
size 240169
|
||||
|
|
@ -0,0 +1,3 @@
|
|||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:72d0f97ba6c3134d7873ec5c9d0fd3c1f5137f4eac4dda0709993d92809e62b6
|
||||
size 474189
|
||||
|
|
@ -0,0 +1,3 @@
|
|||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:1190cddd839b78c2aec737573399a71c23fe9a546d3543f86304c4c68ca73852
|
||||
size 990787
|
||||
|
|
@ -0,0 +1,3 @@
|
|||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:273f9ce6f2c595ad4e63b3195513b61974ae1ec513efcce39da1afa90574ef38
|
||||
size 844087
|
||||
|
|
@ -0,0 +1,3 @@
|
|||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:060422a330f9c819a10e7310788d336dbcb53cc6a4be0e91d40f644564080f97
|
||||
size 1182114
|
||||
|
|
@ -0,0 +1,3 @@
|
|||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:17382e1a69bd628107e8dfe37d31d57f7ba948e5f2da77e56a8aa010488dc5ae
|
||||
size 186526
|
||||
|
|
@ -208,6 +208,7 @@
|
|||
"@vscode/test-web": "^0.0.44",
|
||||
"@vscode/vsce": "^2.15.0",
|
||||
"assert": "^2.0.0",
|
||||
"esbuild-plugin-copy": "^2.1.1",
|
||||
"esbuild-plugin-polyfill-node": "^0.3.0",
|
||||
"eslint": "^8.20.0",
|
||||
"glob": "^8.0.3",
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
import { defineConfig } from "tsup";
|
||||
import { copy } from "esbuild-plugin-copy";
|
||||
import { polyfillNode } from "esbuild-plugin-polyfill-node";
|
||||
import { dependencies } from "./package.json";
|
||||
|
||||
|
|
@ -8,8 +9,19 @@ export default () => [
|
|||
entry: ["src/extension.ts"],
|
||||
outDir: "dist/node",
|
||||
platform: "node",
|
||||
target: "node18",
|
||||
external: ["vscode"],
|
||||
noExternal: Object.keys(dependencies),
|
||||
esbuildPlugins: [
|
||||
copy({
|
||||
assets: [
|
||||
{
|
||||
from: "../tabby-agent/dist/wasm/*.wasm",
|
||||
to: "./wasm",
|
||||
},
|
||||
],
|
||||
}),
|
||||
],
|
||||
clean: true,
|
||||
}),
|
||||
defineConfig({
|
||||
|
|
|
|||
26
yarn.lock
26
yarn.lock
|
|
@ -900,7 +900,7 @@ chalk@^2.4.2:
|
|||
escape-string-regexp "^1.0.5"
|
||||
supports-color "^5.3.0"
|
||||
|
||||
chalk@^4.0.0, chalk@^4.1.0:
|
||||
chalk@^4.0.0, chalk@^4.1.0, chalk@^4.1.2:
|
||||
version "4.1.2"
|
||||
resolved "https://registry.yarnpkg.com/chalk/-/chalk-4.1.2.tgz#aac4e2b7734a740867aeb16bf02aad556a1e7a01"
|
||||
integrity sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==
|
||||
|
|
@ -1390,6 +1390,16 @@ es-get-iterator@^1.1.3:
|
|||
isarray "^2.0.5"
|
||||
stop-iteration-iterator "^1.0.0"
|
||||
|
||||
esbuild-plugin-copy@^2.1.1:
|
||||
version "2.1.1"
|
||||
resolved "https://registry.yarnpkg.com/esbuild-plugin-copy/-/esbuild-plugin-copy-2.1.1.tgz#638308ecfd679e4c7c76b71c62f7dd9a4cc7f901"
|
||||
integrity sha512-Bk66jpevTcV8KMFzZI1P7MZKZ+uDcrZm2G2egZ2jNIvVnivDpodZI+/KnpL3Jnap0PBdIHU7HwFGB8r+vV5CVw==
|
||||
dependencies:
|
||||
chalk "^4.1.2"
|
||||
chokidar "^3.5.3"
|
||||
fs-extra "^10.0.1"
|
||||
globby "^11.0.3"
|
||||
|
||||
esbuild-plugin-polyfill-node@^0.3.0:
|
||||
version "0.3.0"
|
||||
resolved "https://registry.yarnpkg.com/esbuild-plugin-polyfill-node/-/esbuild-plugin-polyfill-node-0.3.0.tgz#e7e3804b8272df51ae4f8ebfb7445a03712504cb"
|
||||
|
|
@ -1706,6 +1716,15 @@ fs-constants@^1.0.0:
|
|||
resolved "https://registry.yarnpkg.com/fs-constants/-/fs-constants-1.0.0.tgz#6be0de9be998ce16af8afc24497b9ee9b7ccd9ad"
|
||||
integrity sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==
|
||||
|
||||
fs-extra@^10.0.1:
|
||||
version "10.1.0"
|
||||
resolved "https://registry.yarnpkg.com/fs-extra/-/fs-extra-10.1.0.tgz#02873cfbc4084dde127eaa5f9905eef2325d1abf"
|
||||
integrity sha512-oRXApq54ETRj4eMiFzGnHWGy+zo5raudjuxN0b8H7s/RU2oW0Wvsx9O0ACRN/kRq9E8Vu/ReskGB5o3ji+FzHQ==
|
||||
dependencies:
|
||||
graceful-fs "^4.2.0"
|
||||
jsonfile "^6.0.1"
|
||||
universalify "^2.0.0"
|
||||
|
||||
fs-extra@^11.1.1:
|
||||
version "11.1.1"
|
||||
resolved "https://registry.yarnpkg.com/fs-extra/-/fs-extra-11.1.1.tgz#da69f7c39f3b002378b0954bb6ae7efdc0876e2d"
|
||||
|
|
@ -4117,6 +4136,11 @@ vscode-uri@^3.0.7:
|
|||
resolved "https://registry.yarnpkg.com/vscode-uri/-/vscode-uri-3.0.7.tgz#6d19fef387ee6b46c479e5fb00870e15e58c1eb8"
|
||||
integrity sha512-eOpPHogvorZRobNqJGhapa0JdwaxpjVvyBp0QIUMRMSf8ZAlqOdEquKuRmw9Qwu0qXtJIWqFtMkmvJjUZmMjVA==
|
||||
|
||||
web-tree-sitter@^0.20.8:
|
||||
version "0.20.8"
|
||||
resolved "https://registry.yarnpkg.com/web-tree-sitter/-/web-tree-sitter-0.20.8.tgz#1e371cb577584789cadd75cb49c7ddfbc99d04c8"
|
||||
integrity sha512-weOVgZ3aAARgdnb220GqYuh7+rZU0Ka9k9yfKtGAzEYMa6GgiCzW9JjQRJyCJakvibQW+dfjJdihjInKuuCAUQ==
|
||||
|
||||
webidl-conversions@^4.0.2:
|
||||
version "4.0.2"
|
||||
resolved "https://registry.yarnpkg.com/webidl-conversions/-/webidl-conversions-4.0.2.tgz#a855980b1f0b6b359ba1d5d9fb39ae941faa63ad"
|
||||
|
|
|
|||
Loading…
Reference in New Issue