feat(agent): add back agent timeout config. (#739)

* feat(agent): add back agent timeout config, add option to disable warning for slow response time.

* feat(agent): Update completion timeout to single number. Add config type validation.

clients/tabby-agent/src/AgentConfig.ts

@@ -1,4 +1,5 @@
 import { isBrowser } from "./env";
+import { getProperty, deleteProperty } from "dot-prop";
 
 export type AgentConfig = {
   server: {
@@ -17,10 +18,7 @@ export type AgentConfig = {
       mode: "adaptive" | "fixed";
       interval: number;
     };
-    timeout: {
+    timeout: number;
-      auto: number;
-      manually: number;
-    };
   };
   postprocess: {
     limitScopeByIndentation: {
@@ -65,11 +63,7 @@ export const defaultAgentConfig: AgentConfig = {
       mode: "adaptive",
       interval: 250, // ms
     },
-    // Deprecated: There is a timeout of 3s on the server side since v0.3.0.
+    timeout: 4000, // ms
-    timeout: {
-      auto: 4000, // 4s
-      manually: 4000, // 4s
-    },
   },
   postprocess: {
     limitScopeByIndentation: {
@@ -101,6 +95,12 @@ const configTomlTemplate = `## Tabby agent configuration file
 # Header1 = "Value1" # list your custom headers here
 # Header2 = "Value2" # values can be strings, numbers or booleans
 
+## Completion
+## (Since 1.1.0) You can set the completion request timeout here.
+## Note that there is also a timeout config at the server side.
+# [completion]
+# timeout = 4000 # 4s
+
 ## Logs
 ## You can set the log level here. The log file is located at ~/.tabby-client/agent/logs/.
 # [logs]
@@ -116,6 +116,44 @@ const configTomlTemplate = `## Tabby agent configuration file
 
 `;
 
+const typeCheckSchema = {
+  server: "object",
+  "server.endpoint": "string",
+  "server.token": "string",
+  "server.requestHeaders": "object",
+  "server.requestTimeout": "number",
+  completion: "object",
+  "completion.prompt": "object",
+  "completion.prompt.experimentalStripAutoClosingCharacters": "boolean",
+  "completion.prompt.maxPrefixLines": "number",
+  "completion.prompt.maxSuffixLines": "number",
+  "completion.debounce": "object",
+  "completion.debounce.mode": "string",
+  "completion.debounce.interval": "number",
+  "completion.timeout": "number",
+  postprocess: "object",
+  "postprocess.limitScopeByIndentation": "object",
+  "postprocess.limitScopeByIndentation.experimentalKeepBlockScopeWhenCompletingLine": "boolean",
+  logs: "object",
+  "logs.level": "string",
+  anonymousUsageTracking: "object",
+  "anonymousUsageTracking.disable": "boolean",
+};
+
+function checkValueType(object, key, type) {
+  if (typeof getProperty(object, key) !== type) {
+    deleteProperty(object, key);
+  }
+}
+
+function validateConfig(config: PartialAgentConfig): PartialAgentConfig {
+  const validatedConfig = { ...config };
+  for (const key in typeCheckSchema) {
+    checkValueType(validatedConfig, key, typeCheckSchema[key]);
+  }
+  return validatedConfig;
+}
+
 export const userAgentConfig = isBrowser
   ? null
   : (() => {
@@ -149,7 +187,7 @@ export const userAgentConfig = isBrowser
               await this.createTemplate();
               return;
             }
-            this.data = data;
+            this.data = validateConfig(data);
           } catch (error) {
             if (error.code === "ENOENT") {
               await this.createTemplate();

clients/tabby-agent/src/CompletionProviderStats.ts

@@ -59,6 +59,18 @@ type WindowedStats = {
 
 export class CompletionProviderStats {
   private readonly logger = rootLogger.child({ component: "CompletionProviderStats" });
+  private config = {
+    windowSize: 10,
+    checks: {
+      disable: false,
+      // Mark status as healthy if the latency is less than the threshold for each latest windowSize requests.
+      healthy: { windowSize: 3, latency: 2400 },
+      // If there is at least {count} requests, and the average response time is higher than the {latency}, show warning
+      slowResponseTime: { latency: 3200, count: 3 },
+      // If there is at least {count} timeouts, and the timeout rate is higher than the {rate}, show warning
+      highTimeoutRate: { rate: 0.5, count: 3 },
+    },
+  };
 
   private autoCompletionCount = 0;
   private manualCompletionCount = 0;
@@ -71,7 +83,13 @@ export class CompletionProviderStats {
   private completionRequestCanceledStats = new Average();
   private completionRequestTimeoutCount = 0;
 
-  private recentCompletionRequestLatencies = new Windowed(10);
+  private recentCompletionRequestLatencies: Windowed;
+
+  updateConfigByRequestTimeout(timeout: number) {
+    this.config.checks.healthy.latency = timeout * 0.6;
+    this.config.checks.slowResponseTime.latency = timeout * 0.8;
+    this.resetWindowed();
+  }
 
   add(value: CompletionProviderStatsEntry): void {
     const { triggerMode, cacheHit, aborted, requestSent, requestLatency, requestCanceled, requestTimeout } = value;
@@ -120,7 +138,7 @@ export class CompletionProviderStats {
   }
 
   resetWindowed() {
-    this.recentCompletionRequestLatencies = new Windowed(10);
+    this.recentCompletionRequestLatencies = new Windowed(this.config.windowSize);
   }
 
   // stats for anonymous usage report
@@ -170,21 +188,28 @@ export class CompletionProviderStats {
     };
   }
 
-  static check(windowed: WindowedStats): "healthy" | "highTimeoutRate" | "slowResponseTime" | null {
+  check(windowed: WindowedStats): "healthy" | "highTimeoutRate" | "slowResponseTime" | null {
+    if (!!this.config.checks.disable) {
+      return null;
+    }
+    const config = this.config.checks;
+
     const {
       values: latencies,
       stats: { total, timeouts, responses, averageResponseTime },
     } = windowed;
-    // if the recent 3 requests all have latency less than 3s
+
-    if (latencies.slice(-3).every((latency) => latency < 3000)) {
+    if (
+      latencies
+        .slice(-Math.max(this.config.windowSize, config.healthy.windowSize))
+        .every((latency) => latency < config.healthy.latency)
+    ) {
       return "healthy";
     }
-    // if the recent requests timeout percentage is more than 50%, at least 3 timeouts
+    if (timeouts / total > config.highTimeoutRate.rate && timeouts >= config.highTimeoutRate.count) {
-    if (timeouts / total > 0.5 && timeouts >= 3) {
       return "highTimeoutRate";
     }
-    // if average response time is more than 4s, at least 3 requests
+    if (averageResponseTime > config.slowResponseTime.latency && responses >= config.slowResponseTime.count) {
-    if (responses >= 3 && averageResponseTime > 4000) {
       return "slowResponseTime";
     }
     return null;

clients/tabby-agent/src/TabbyAgent.ts

@@ -123,6 +123,12 @@ export class TabbyAgent extends EventEmitter implements Agent {
       }
     }
 
+    if (oldConfig.completion.timeout !== this.config.completion.timeout) {
+      this.completionProviderStats.updateConfigByRequestTimeout(this.config.completion.timeout);
+      this.popIssue("slowCompletionResponseTime");
+      this.popIssue("highCompletionTimeoutRate");
+    }
+
     const event: AgentEvent = { event: "configUpdated", config: this.config };
     this.logger.debug({ event }, "Config updated");
     super.emit("configUpdated", event);
@@ -524,9 +530,7 @@ export class TabbyAgent extends EventEmitter implements Agent {
               },
               {
                 signal,
-                timeout: request.manually
+                timeout: this.config.completion.timeout,
-                  ? this.config.completion.timeout.manually
-                  : this.config.completion.timeout.auto,
               },
             );
             stats.requestLatency = performance.now() - requestStartedAt;
@@ -588,7 +592,7 @@ export class TabbyAgent extends EventEmitter implements Agent {
 
         if (stats.requestSent && !stats.requestCanceled) {
           const windowedStats = this.completionProviderStats.windowed();
-          const checkResult = CompletionProviderStats.check(windowedStats);
+          const checkResult = this.completionProviderStats.check(windowedStats);
           switch (checkResult) {
             case "healthy":
               this.popIssue("slowCompletionResponseTime");

website/docs/extensions/configurations.md

@@ -32,6 +32,18 @@ Header1 = "Value1" # list your custom headers here
 Header2 = "Value2" # values can be strings, numbers or booleans
 ```
 
+## Completion
+
+If you have changed the default response timeout at Tabby server side, you may also need to change the timeout configurations here.
+
+```toml
+# Completion
+# (Since 1.1.0) You can set the completion request timeout here.
+# Note that there is also a timeout config at the server side.
+[completion]
+timeout = 4000 # 4s
+```
+
 ## Logs
 
 If you encounter any issues with the Tabby IDE extensions and need to report a bug, you can enable debug logs to help us investigate the issue.