feat: agent read config toml (#274)

* feat: agent read config toml.

* fix: move loading config toml from agent creating to initialization.

* docs: update agent config doc.

clients/tabby-agent/dist/index.d.ts

@@ -66,7 +66,18 @@ type AgentStatus = "notInitialized" | "ready" | "disconnected" | "unauthorized";
 interface AgentFunction {
     initialize(options: Partial<AgentInitOptions>): Promise<boolean>;
     updateConfig(config: Partial<AgentConfig>): Promise<boolean>;
+    /**
+     * @returns the current config
+     *
+     * Configuration precedence:
+     * 1. Default config
+     * 2. User config file `~/.tabby/agent/config.toml` (not available in browser)
+     * 3. Agent `initialize` and `updateConfig` methods
+     */
     getConfig(): AgentConfig;
+    /**
+     * @returns the current status
+     */
     getStatus(): AgentStatus;
     /**
      * @returns the auth url for redirecting, and the code for next step `waitingForAuth`, only return value when
@@ -141,6 +152,8 @@ declare class TabbyAgent extends EventEmitter implements Agent {
     private readonly logger;
     private anonymousUsageLogger;
     private config;
+    private userConfig;
+    private clientConfig;
     private status;
     private api;
     private auth;

clients/tabby-agent/dist/index.js

@@ -4,6 +4,9 @@ var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
 var __getOwnPropNames = Object.getOwnPropertyNames;
 var __getProtoOf = Object.getPrototypeOf;
 var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __esm = (fn, res) => function __init() {
+  return fn && (res = (0, fn[__getOwnPropNames(fn)[0]])(fn = 0)), res;
+};
 var __export = (target, all) => {
   for (var name2 in all)
     __defProp(target, name2, { get: all[name2], enumerable: true });
@@ -44,6 +47,43 @@ var __privateSet = (obj, member, value, setter) => {
   return value;
 };
 
+// src/env.ts
+var isBrowser;
+var init_env = __esm({
+  "src/env.ts"() {
+    isBrowser = false;
+  }
+});
+
+// src/logger.ts
+var logger_exports = {};
+__export(logger_exports, {
+  allLoggers: () => allLoggers,
+  rootLogger: () => rootLogger
+});
+var import_pino, stream, rootLogger, allLoggers;
+var init_logger = __esm({
+  "src/logger.ts"() {
+    import_pino = __toESM(require("pino"));
+    init_env();
+    stream = isBrowser ? null : (
+      /**
+       * Default rotating file locate at `~/.tabby/agent-logs/`.
+       */
+      require("rotating-file-stream").createStream("tabby-agent.log", {
+        path: require("path").join(require("os").homedir(), ".tabby", "agent-logs"),
+        size: "10M",
+        interval: "1d"
+      })
+    );
+    rootLogger = !!stream ? (0, import_pino.default)(stream) : (0, import_pino.default)();
+    allLoggers = [rootLogger];
+    rootLogger.onChild = (child) => {
+      allLoggers.push(child);
+    };
+  }
+});
+
 // src/index.ts
 var src_exports = {};
 __export(src_exports, {
@@ -474,7 +514,7 @@ var V1Service = class {
 var TabbyApi = class {
   constructor(config, HttpRequest = AxiosHttpRequest) {
     this.request = new HttpRequest({
-      BASE: config?.BASE ?? "https://playground.app.tabbyml.com/tabby",
+      BASE: config?.BASE ?? "https://playground.app.tabbyml.com",
       VERSION: config?.VERSION ?? "0.1.0",
       WITH_CREDENTIALS: config?.WITH_CREDENTIALS ?? false,
       CREDENTIALS: config?.CREDENTIALS ?? "include",
@@ -585,10 +625,8 @@ var CloudApi = class {
   }
 };
 
-// src/env.ts
-var isBrowser = false;
-
 // src/dataStore.ts
+init_env();
 var dataStore = isBrowser ? null : (() => {
   const dataFile = require("path").join(require("os").homedir(), ".tabby", "agent", "data.json");
   const fs = require("fs-extra");
@@ -603,25 +641,8 @@ var dataStore = isBrowser ? null : (() => {
   };
 })();
 
-// src/logger.ts
-var import_pino = __toESM(require("pino"));
-var stream = isBrowser ? null : (
-  /**
-   * Default rotating file locate at `~/.tabby/agent-logs/`.
-   */
-  require("rotating-file-stream").createStream("tabby-agent.log", {
-    path: require("path").join(require("os").homedir(), ".tabby", "agent-logs"),
-    size: "10M",
-    interval: "1d"
-  })
-);
-var rootLogger = !!stream ? (0, import_pino.default)(stream) : (0, import_pino.default)();
-var allLoggers = [rootLogger];
-rootLogger.onChild = (child) => {
-  allLoggers.push(child);
-};
-
 // src/Auth.ts
+init_logger();
 var _Auth = class extends import_events.EventEmitter {
   constructor(options) {
     super();
@@ -824,6 +845,7 @@ Auth.tokenStrategy = {
 };
 
 // src/AgentConfig.ts
+init_env();
 var defaultAgentConfig = {
   server: {
     endpoint: "http://localhost:8080"
@@ -835,11 +857,48 @@ var defaultAgentConfig = {
     disable: false
   }
 };
+var userAgentConfig = isBrowser ? null : (() => {
+  const EventEmitter3 = require("events");
+  const fs = require("fs-extra");
+  const toml = require("toml");
+  const chokidar = require("chokidar");
+  class ConfigFile extends EventEmitter3 {
+    constructor(filepath) {
+      super();
+      this.data = {};
+      this.watcher = null;
+      this.logger = (init_logger(), __toCommonJS(logger_exports)).rootLogger.child({ component: "ConfigFile" });
+      this.filepath = filepath;
+    }
+    get config() {
+      return this.data;
+    }
+    async load() {
+      try {
+        const fileContent = await fs.readFile(this.filepath, "utf8");
+        this.data = toml.parse(fileContent);
+        super.emit("updated", this.data);
+      } catch (error) {
+        this.logger.error({ error }, "Failed to load config file");
+      }
+    }
+    watch() {
+      this.watcher = chokidar.watch(this.filepath, {
+        interval: 1e3
+      });
+      this.watcher.on("add", this.load.bind(this));
+      this.watcher.on("change", this.load.bind(this));
+    }
+  }
+  const configFile = require("path").join(require("os").homedir(), ".tabby", "agent", "config.toml");
+  return new ConfigFile(configFile);
+})();
 
 // src/CompletionCache.ts
 var import_lru_cache = require("lru-cache");
 var import_object_hash = __toESM(require("object-hash"));
 var import_object_sizeof = __toESM(require("object-sizeof"));
+init_logger();
 var CompletionCache = class {
   constructor() {
     this.logger = rootLogger.child({ component: "CompletionCache" });
@@ -952,6 +1011,7 @@ var CompletionCache = class {
 };
 
 // src/postprocess/filter.ts
+init_logger();
 var logger = rootLogger.child({ component: "Postprocess" });
 var applyFilter = (filter) => {
   return async (response) => {
@@ -1042,12 +1102,17 @@ async function postprocess(request2, response) {
   return new Promise((resolve2) => resolve2(response)).then(applyFilter(limitScopeByIndentation(request2))).then(applyFilter(removeOverlapping(request2))).then(applyFilter(dropBlank()));
 }
 
+// src/TabbyAgent.ts
+init_logger();
+
 // package.json
 var name = "tabby-agent";
 var version = "0.0.1";
 
 // src/AnonymousUsageLogger.ts
 var import_uuid = require("uuid");
+init_env();
+init_logger();
 var AnonymousUsageLogger = class {
   constructor() {
     this.anonymousUsageTrackingApi = new CloudApi();
@@ -1110,6 +1175,10 @@ var _TabbyAgent = class extends import_events2.EventEmitter {
     super();
     this.logger = rootLogger.child({ component: "TabbyAgent" });
     this.config = defaultAgentConfig;
+    this.userConfig = {};
+    // config from `~/.tabby/agent/config.toml`
+    this.clientConfig = {};
+    // config from `initialize` and `updateConfig` method
     this.status = "notInitialized";
     this.dataStore = null;
     this.completionCache = new CompletionCache();
@@ -1129,6 +1198,7 @@ var _TabbyAgent = class extends import_events2.EventEmitter {
     return agent;
   }
   async applyConfig() {
+    this.config = import_deepmerge.default.all([defaultAgentConfig, this.userConfig, this.clientConfig]);
     allLoggers.forEach((logger2) => logger2.level = this.config.logs.level);
     this.anonymousUsageLogger.disabled = this.config.anonymousUsageTracking.disable;
     if (this.config.server.endpoint !== this.auth?.endpoint) {
@@ -1201,8 +1271,17 @@ var _TabbyAgent = class extends import_events2.EventEmitter {
     if (options.client) {
       allLoggers.forEach((logger2) => logger2.setBindings?.({ client: options.client }));
     }
+    if (userAgentConfig) {
+      await userAgentConfig.load();
+      this.userConfig = userAgentConfig.config;
+      userAgentConfig.on("updated", async (config) => {
+        this.userConfig = config;
+        await this.applyConfig();
+      });
+      userAgentConfig.watch();
+    }
     if (options.config) {
-      this.config = (0, import_deepmerge.default)(this.config, options.config);
+      this.clientConfig = (0, import_deepmerge.default)(this.clientConfig, options.config);
     }
     await this.applyConfig();
     if (this.status === "unauthorized") {
@@ -1216,10 +1295,10 @@ var _TabbyAgent = class extends import_events2.EventEmitter {
     return this.status !== "notInitialized";
   }
   async updateConfig(config) {
-    const mergedConfig = (0, import_deepmerge.default)(this.config, config);
+    const mergedConfig = (0, import_deepmerge.default)(this.clientConfig, config);
-    if (!(0, import_deep_equal.default)(this.config, mergedConfig)) {
+    if (!(0, import_deep_equal.default)(this.clientConfig, mergedConfig)) {
       const serverUpdated = !(0, import_deep_equal.default)(this.config.server, mergedConfig.server);
-      this.config = mergedConfig;
+      this.clientConfig = mergedConfig;
       await this.applyConfig();
       const event = { event: "configUpdated", config: this.config };
       this.logger.debug({ event }, "Config updated");

clients/tabby-agent/package.json

@@ -33,6 +33,7 @@
   },
   "dependencies": {
     "axios": "^1.4.0",
+    "chokidar": "^3.5.3",
     "deep-equal": "^2.2.1",
     "deepmerge": "^4.3.1",
     "form-data": "^4.0.0",
@@ -43,6 +44,7 @@
     "object-sizeof": "^2.6.1",
     "pino": "^8.14.1",
     "rotating-file-stream": "^3.1.0",
+    "toml": "^3.0.0",
     "uuid": "^9.0.0"
   }
 }

clients/tabby-agent/src/Agent.ts

@@ -29,7 +29,20 @@ export type AgentStatus = "notInitialized" | "ready" | "disconnected" | "unautho
 export interface AgentFunction {
   initialize(options: Partial<AgentInitOptions>): Promise<boolean>;
   updateConfig(config: Partial<AgentConfig>): Promise<boolean>;
+
+  /**
+   * @returns the current config
+   *
+   * Configuration precedence:
+   * 1. Default config
+   * 2. User config file `~/.tabby/agent/config.toml` (not available in browser)
+   * 3. Agent `initialize` and `updateConfig` methods
+   */
   getConfig(): AgentConfig;
+
+  /**
+   * @returns the current status
+   */
   getStatus(): AgentStatus;
 
   /**
@@ -72,7 +85,7 @@ export type ConfigUpdatedEvent = {
 };
 export type AuthRequiredEvent = {
   event: "authRequired";
-  server: AgentConfig["server"]
+  server: AgentConfig["server"];
 };
 
 export type AgentEvent = StatusChangedEvent | ConfigUpdatedEvent | AuthRequiredEvent;

clients/tabby-agent/src/AgentConfig.ts

@@ -1,3 +1,5 @@
+import { isBrowser } from "./env";
+
 export type AgentConfig = {
   server: {
     endpoint: string;
@@ -21,3 +23,49 @@ export const defaultAgentConfig: AgentConfig = {
     disable: false,
   },
 };
+
+export const userAgentConfig = isBrowser
+  ? null
+  : (() => {
+      const EventEmitter = require("events");
+      const fs = require("fs-extra");
+      const toml = require("toml");
+      const chokidar = require("chokidar");
+
+      class ConfigFile extends EventEmitter {
+        filepath: string;
+        data: Partial<AgentConfig> = {};
+        watcher: ReturnType<typeof chokidar.watch> | null = null;
+        logger = require("./logger").rootLogger.child({ component: "ConfigFile" });
+
+        constructor(filepath: string) {
+          super();
+          this.filepath = filepath;
+        }
+
+        get config(): Partial<AgentConfig> {
+          return this.data;
+        }
+
+        async load() {
+          try {
+            const fileContent = await fs.readFile(this.filepath, "utf8");
+            this.data = toml.parse(fileContent);
+            super.emit("updated", this.data);
+          } catch (error) {
+            this.logger.error({ error }, "Failed to load config file");
+          }
+        }
+
+        watch() {
+          this.watcher = chokidar.watch(this.filepath, {
+            interval: 1000,
+          });
+          this.watcher.on("add", this.load.bind(this));
+          this.watcher.on("change", this.load.bind(this));
+        }
+      }
+
+      const configFile = require("path").join(require("os").homedir(), ".tabby", "agent", "config.toml");
+      return new ConfigFile(configFile);
+    })();

clients/tabby-agent/src/TabbyAgent.ts

@@ -14,7 +14,7 @@ import {
   LogEventRequest,
 } from "./Agent";
 import { Auth } from "./Auth";
-import { AgentConfig, defaultAgentConfig } from "./AgentConfig";
+import { AgentConfig, defaultAgentConfig, userAgentConfig } from "./AgentConfig";
 import { CompletionCache } from "./CompletionCache";
 import { DataStore } from "./dataStore";
 import { postprocess } from "./postprocess";
@@ -33,6 +33,8 @@ export class TabbyAgent extends EventEmitter implements Agent {
   private readonly logger = rootLogger.child({ component: "TabbyAgent" });
   private anonymousUsageLogger: AnonymousUsageLogger;
   private config: AgentConfig = defaultAgentConfig;
+  private userConfig: Partial<AgentConfig> = {}; // config from `~/.tabby/agent/config.toml`
+  private clientConfig: Partial<AgentConfig> = {}; // config from `initialize` and `updateConfig` method
   private status: AgentStatus = "notInitialized";
   private api: TabbyApi;
   private auth: Auth;
@@ -60,6 +62,7 @@ export class TabbyAgent extends EventEmitter implements Agent {
   }
 
   private async applyConfig() {
+    this.config = deepMerge.all<AgentConfig>([defaultAgentConfig, this.userConfig, this.clientConfig]);
     allLoggers.forEach((logger) => (logger.level = this.config.logs.level));
     this.anonymousUsageLogger.disabled = this.config.anonymousUsageTracking.disable;
     if (this.config.server.endpoint !== this.auth?.endpoint) {
@@ -144,8 +147,17 @@ export class TabbyAgent extends EventEmitter implements Agent {
       // `pino.Logger.setBindings` is not present in the browser
       allLoggers.forEach((logger) => logger.setBindings?.({ client: options.client }));
     }
+    if (userAgentConfig) {
+      await userAgentConfig.load();
+      this.userConfig = userAgentConfig.config;
+      userAgentConfig.on("updated", async (config) => {
+        this.userConfig = config;
+        await this.applyConfig();
+      });
+      userAgentConfig.watch();
+    }
     if (options.config) {
-      this.config = deepMerge(this.config, options.config);
+      this.clientConfig = deepMerge(this.clientConfig, options.config);
     }
     await this.applyConfig();
     if (this.status === "unauthorized") {
@@ -160,10 +172,10 @@ export class TabbyAgent extends EventEmitter implements Agent {
   }
 
   public async updateConfig(config: Partial<AgentConfig>): Promise<boolean> {
-    const mergedConfig = deepMerge(this.config, config);
+    const mergedConfig = deepMerge(this.clientConfig, config);
-    if (!deepEqual(this.config, mergedConfig)) {
+    if (!deepEqual(this.clientConfig, mergedConfig)) {
       const serverUpdated = !deepEqual(this.config.server, mergedConfig.server);
-      this.config = mergedConfig;
+      this.clientConfig = mergedConfig;
       await this.applyConfig();
       const event: AgentEvent = { event: "configUpdated", config: this.config };
       this.logger.debug({ event }, "Config updated");

clients/tabby-agent/yarn.lock

@@ -502,7 +502,7 @@ check-error@^1.0.2:
   resolved "https://registry.yarnpkg.com/check-error/-/check-error-1.0.2.tgz#574d312edd88bb5dd8912e9286dd6c0aed4aac82"
   integrity sha512-BrgHpW9NURQgzoNyjfq0Wu6VFO6D7IZEmJNdtgNqpzGG8RuNFHt2jQxWlAs4HMe119chBnv+34syEZtc6IhLtA==
 
-chokidar@3.5.3, chokidar@^3.5.1:
+chokidar@3.5.3, chokidar@^3.5.1, chokidar@^3.5.3:
   version "3.5.3"
   resolved "https://registry.yarnpkg.com/chokidar/-/chokidar-3.5.3.tgz#1cf37c8707b932bd1af1ae22c0432e2acd1903bd"
   integrity sha512-Dr3sfKRP6oTcjf2JmUmFJfeVMvXBdegxB0iVQ5eb2V10uFJUCAS8OByZdVAyVb8xXNz3GjjTgj9kLWsZTqE6kw==
@@ -1906,6 +1906,11 @@ to-regex-range@^5.0.1:
   dependencies:
     is-number "^7.0.0"
 
+toml@^3.0.0:
+  version "3.0.0"
+  resolved "https://registry.yarnpkg.com/toml/-/toml-3.0.0.tgz#342160f1af1904ec9d204d03a5d61222d762c5ee"
+  integrity sha512-y/mWCZinnvxjTKYhJ+pYxwD0mRLVvOtdS2Awbgxln6iEnt4rk0yBxeSBHkGJcPucRiG0e55mwWp+g/05rsrd6w==
+
 tr46@^1.0.1:
   version "1.0.1"
   resolved "https://registry.yarnpkg.com/tr46/-/tr46-1.0.1.tgz#a8b13fd6bfd2489519674ccde55ba3693b706d09"

clients/vscode/yarn.lock

@@ -831,7 +831,7 @@ cheerio@^1.0.0-rc.9:
     parse5 "^7.0.0"
     parse5-htmlparser2-tree-adapter "^7.0.0"
 
-chokidar@3.5.3:
+chokidar@3.5.3, chokidar@^3.5.3:
   version "3.5.3"
   resolved "https://registry.yarnpkg.com/chokidar/-/chokidar-3.5.3.tgz#1cf37c8707b932bd1af1ae22c0432e2acd1903bd"
   integrity sha512-Dr3sfKRP6oTcjf2JmUmFJfeVMvXBdegxB0iVQ5eb2V10uFJUCAS8OByZdVAyVb8xXNz3GjjTgj9kLWsZTqE6kw==
@@ -2786,9 +2786,9 @@ pino-abstract-transport@v1.0.0:
     split2 "^4.0.0"
 
 pino-std-serializers@^6.0.0:
-  version "6.2.1"
+  version "6.2.2"
-  resolved "https://registry.yarnpkg.com/pino-std-serializers/-/pino-std-serializers-6.2.1.tgz#369f4ae2a19eb6d769ddf2c88a2164b76879a284"
+  resolved "https://registry.yarnpkg.com/pino-std-serializers/-/pino-std-serializers-6.2.2.tgz#d9a9b5f2b9a402486a5fc4db0a737570a860aab3"
-  integrity sha512-wHuWB+CvSVb2XqXM0W/WOYUkVSPbiJb9S5fNB7TBhd8s892Xq910bRxwHtC4l71hgztObTjXL6ZheZXFjhDrDQ==
+  integrity sha512-cHjPPsE+vhj/tnhCy/wiMh3M3z3h/j15zHQX+S9GkTBgqJuTuJzYJ4gUyACLhDaJ7kk9ba9iRDmbH2tJU03OiA==
 
 pino@^8.14.1:
   version "8.14.1"
@@ -2963,9 +2963,9 @@ readable-stream@^3.1.1, readable-stream@^3.4.0:
     util-deprecate "^1.0.1"
 
 readable-stream@^4.0.0:
-  version "4.4.0"
+  version "4.4.1"
-  resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-4.4.0.tgz#55ce132d60a988c460d75c631e9ccf6a7229b468"
+  resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-4.4.1.tgz#fa0f0878c3bc0c12b6a82e4e58c5dc160e1faaa2"
-  integrity sha512-kDMOq0qLtxV9f/SQv522h8cxZBqNZXuXNyjyezmfAAuribMyVXziljpQ/uQhfE1XLg2/TLTW2DsnoE4VAi/krg==
+  integrity sha512-llAHX9QC25bz5RPIoTeJxPaA/hgryaldValRhVZ2fK9bzbmFiscpz8fw6iBTvJfAk1w4FC1KXQme/nO7fbKyKg==
   dependencies:
     abort-controller "^3.0.0"
     buffer "^6.0.3"
@@ -3297,6 +3297,7 @@ supports-preserve-symlinks-flag@^1.0.0:
   version "0.0.1"
   dependencies:
     axios "^1.4.0"
+    chokidar "^3.5.3"
     deep-equal "^2.2.1"
     deepmerge "^4.3.1"
     form-data "^4.0.0"
@@ -3307,6 +3308,7 @@ supports-preserve-symlinks-flag@^1.0.0:
     object-sizeof "^2.6.1"
     pino "^8.14.1"
     rotating-file-stream "^3.1.0"
+    toml "^3.0.0"
     uuid "^9.0.0"
 
 tapable@^2.1.1, tapable@^2.2.0:
@@ -3395,6 +3397,11 @@ toidentifier@1.0.1:
   resolved "https://registry.yarnpkg.com/toidentifier/-/toidentifier-1.0.1.tgz#3be34321a88a820ed1bd80dfaa33e479fbb8dd35"
   integrity sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==
 
+toml@^3.0.0:
+  version "3.0.0"
+  resolved "https://registry.yarnpkg.com/toml/-/toml-3.0.0.tgz#342160f1af1904ec9d204d03a5d61222d762c5ee"
+  integrity sha512-y/mWCZinnvxjTKYhJ+pYxwD0mRLVvOtdS2Awbgxln6iEnt4rk0yBxeSBHkGJcPucRiG0e55mwWp+g/05rsrd6w==
+
 ts-loader@^9.3.1:
   version "9.4.3"
   resolved "https://registry.yarnpkg.com/ts-loader/-/ts-loader-9.4.3.tgz#55cfa7c28dd82a2de968ae45c3adb75fb888b27e"