test(agent): add test bad case for limit scopy by indentation. (#757)

* test(agent): add test bad case for limitScopeByIndentation.

* test(agent): fix golden test config.

* fix: lint.

* test(agent): update test cases.

clients/tabby-agent/.prettierignore

@@ -1 +1,2 @@
-tests/golden/**
+tests/golden/**
+tests/bad_cases/**

clients/tabby-agent/src/postprocess/limitScopeByIndentation.test.ts

@@ -284,6 +284,69 @@ describe("postprocess", () => {
     });
   });
 
+  describe("limitScopeByIndentation: bad cases", () => {
+    let limitScopeByIndentationDefault = (context) => {
+      return limitScopeByIndentation(context, { experimentalKeepBlockScopeWhenCompletingLine: false });
+    };
+    it("cannot handle the case of indent that does'nt have a close line, e.g. chaining call", () => {
+      const context = {
+        ...documentContext`
+        function sortWords(input) {
+          const output = input.trim()
+            .split("\n")
+            .map((line) => line.split(" "))
+            ║
+        }
+        `,
+        language: "javascript",
+      };
+      const completion = inline`
+            ├.flat()
+            .sort()
+            .join(" ");
+          console.log(output);
+          return output;
+        }
+        sortWord("world hello");┤
+      `;
+      const expected = inline`
+            ├.flat()
+            .sort()
+            .join(" ");
+          console.log(output);
+          return output;
+        };┤
+      `;
+      expect(limitScopeByIndentationDefault(context)(completion)).not.to.eq(expected);
+    });
+
+    it("cannot handle the case of indent that does'nt have a close line, e.g. python def function", () => {
+      const context = {
+        ...documentContext`
+        def findMax(arr):
+          ║
+        `,
+        language: "python",
+      };
+      const completion = inline`
+          ├max = arr[0]
+          for i in range(1, len(arr)):
+            if arr[i] > max:
+              max = arr[i]
+          return max
+        findMax([1, 2, 3, 4, 5])┤
+      `;
+      const expected = inline`
+          ├max = arr[0]
+          for i in range(1, len(arr)):
+            if arr[i] > max:
+              max = arr[i]
+          return max┤
+      `;
+      expect(limitScopeByIndentationDefault(context)(completion)).not.to.eq(expected);
+    });
+  });
+
   describe("limitScopeByIndentation: with experimentalKeepBlockScopeWhenCompletingLine on", () => {
     let limitScopeByIndentationKeepBlock = (context) => {
       return limitScopeByIndentation(context, { experimentalKeepBlockScopeWhenCompletingLine: true });

clients/tabby-agent/tests/bad_cases/0-javascript.js

@@ -0,0 +1,15 @@
+const arr = ["a", "b", "c"];
+
+function myEncode(arr) {
+  return arr
+    .map(item => item.toLocaleUpperCase())
+    .map(item => `(${item})`)
+    .map(item => Buffer.from(item).toString("base64"));
+}
+console.log(myEncode(arr));
+
+function myDecode(arr) {
+  return arr
+    .map⏩⏭(item => Buffer.from(item, "base64").toString("ascii"))
+    .map(item => item.toLocaleUpperCase());
+}⏮}⏪

clients/tabby-agent/tests/bad_cases/1-python.py

@@ -0,0 +1,7 @@
+def foo(msg):
+  print(msg)
+  return 0
+foo("Hello, World!")
+def bar(msg):
+  print⏩⏭(msg)
+  return 1⏮⏪

clients/tabby-agent/tests/golden.test.ts

@@ -68,29 +68,28 @@ describe("agent golden test", () => {
     return { request, expected };
   };
 
+  const config = {
+    server: {
+      endpoint: "http://127.0.0.1:8087",
+      token: "",
+      requestHeaders: {},
+      requestTimeout: 30000,
+    },
+    completion: {
+      prompt: { experimentalStripAutoClosingCharacters: false, maxPrefixLines: 20, maxSuffixLines: 20 },
+      debounce: { mode: "adaptive", interval: 250 },
+      timeout: 4000,
+    },
+    postprocess: {
+      limitScopeByIndentation: { experimentalKeepBlockScopeWhenCompletingLine: false },
+    },
+    logs: { level: "debug" },
+    anonymousUsageTracking: { disable: true },
+  };
+
+  let requestId = 0;
   it("initialize", async () => {
-    const requestId = 1;
+    requestId++;
-    const config = {
-      server: {
-        endpoint: "http://127.0.0.1:8087",
-        token: "",
-        requestHeaders: {},
-        requestTimeout: 30000,
-      },
-      completion: {
-        prompt: { experimentalStripAutoClosingCharacters: false, maxPrefixLines: 20, maxSuffixLines: 20 },
-        debounce: { mode: "adaptive", interval: 250 },
-        timeout: { auto: 4000, manually: 4000 },
-      },
-      postprocess: {
-        limitScope: {
-          experimentalSyntax: false,
-          indentation: { experimentalKeepBlockScopeWhenCompletingLine: false },
-        },
-      },
-      logs: { level: "debug" },
-      anonymousUsageTracking: { disable: true },
-    };
     const initRequest = [
       requestId,
       {
@@ -108,15 +107,14 @@ describe("agent golden test", () => {
 
   const goldenFiles = fs.readdirSync(path.join(__dirname, "golden")).map((file) => {
     return {
-      name: file,
+      path: path.join("golden", file),
-      path: path.join(__dirname, "golden", file),
+      absolutePath: path.join(__dirname, "golden", file),
     };
   });
-  const baseRequestId = 2;
   goldenFiles.forEach((goldenFile, index) => {
-    it(goldenFile.name, async () => {
+    it(goldenFile.path, async () => {
-      const test = await createGoldenTest(goldenFile.path);
+      const test = await createGoldenTest(goldenFile.absolutePath);
-      const requestId = baseRequestId + index;
+      requestId++;
       const request = [requestId, { func: "provideCompletions", args: [test.request] }];
       agent.stdin.write(JSON.stringify(request) + "\n");
       await waitForResponse(requestId);
@@ -126,6 +124,25 @@ describe("agent golden test", () => {
     });
   });
 
+  const badCasesFiles = fs.readdirSync(path.join(__dirname, "bad_cases")).map((file) => {
+    return {
+      path: path.join("bad_cases", file),
+      absolutePath: path.join(__dirname, "bad_cases", file),
+    };
+  });
+  badCasesFiles.forEach((goldenFile, index) => {
+    it(goldenFile.path, async () => {
+      const test = await createGoldenTest(goldenFile.absolutePath);
+      requestId++;
+      const request = [requestId, { func: "provideCompletions", args: [test.request] }];
+      agent.stdin.write(JSON.stringify(request) + "\n");
+      await waitForResponse(requestId);
+      const response = output.shift();
+      expect(response[0]).to.equal(requestId);
+      expect(response[1].choices).not.to.deep.equal(test.expected.choices);
+    });
+  });
+
   after(() => {
     agent.kill();
   });