diff --git a/deployment/config/vector.toml b/deployment/config/vector.toml index a91faa2..f40cb1a 100644 --- a/deployment/config/vector.toml +++ b/deployment/config/vector.toml @@ -6,14 +6,33 @@ address = "0.0.0.0:8686" type = "file" include = ["/data/logs/tabby-server/events.*.log"] -[transforms.process_tabby_server_logs] +[transforms.process_tabby_server_events] type = "remap" inputs = [ "tabby_server_logs" ] -source = ". = parse_json!(parse_json!(.message).record.message)" +source = """ +record = parse_json!(.message).record -[sinks.write_tabby_server_logs] +id = { + "process_id": record.process.id, + "thread_id": record.thread.id, + "timestamp": record.time.timestamp +} + +. = { + "id": id, + "data": parse_json!(record.message) +} +""" + +[sinks.write_tabby_server_events] type = "file" -inputs = [ "process_tabby_server_logs" ] +inputs = [ "process_tabby_server_events" ] +encoding = { codec = "json" } +framing = { method = "newline_delimited" } +path = "/data/logs/events/tabby-server/%Y-%m-%d.json" + +[sinks.all_events] +type = "console" +inputs = [ "process_tabby_server_events" ] encoding = { codec = "json" } framing = { method = "newline_delimited" } -path = "/data/logs/tabby-server/events.%Y-%m-%d.json" diff --git a/docs/internal/Events.md b/docs/internal/Events.md new file mode 100644 index 0000000..6716ce7 --- /dev/null +++ b/docs/internal/Events.md @@ -0,0 +1,31 @@ +# Events System + +In Tabby, we use the [`vector`](../../deployment/config/vector.toml) to collect logs from various sources, transform them into a standard `Event`, and persist them in `/data/logs/events`. + +## Schema + +### Event + +```jsx +{ + "id": EventId, + "data": Any +} +``` + +The `id` field can be used to uniquely identify an event. + +The `data` field is a standard JSON object, and its definition is left to downstream tasks. + +### EventId + +```jsx +{ + "process_id": Number, + "thread_id": Number, + // Unix timestamp + "timestamp": Number, +} +``` + +In the future, we might add `server_id` when Tabby evolves into a distributed environment. diff --git a/tabby/tools/analytic/main.sh b/tabby/tools/analytic/main.sh index 876a9cb..bb6bdd1 100755 --- a/tabby/tools/analytic/main.sh +++ b/tabby/tools/analytic/main.sh @@ -3,7 +3,7 @@ set -e DB_FILE=${DB_FILE:-"/data/logs/duckdb/duck.db"} LOGS_DIR=${LOGS_DIR:-"/data/logs"} -TABBY_SERVER_LOGS="${LOGS_DIR}/tabby-server/events.*.json" +TABBY_SERVER_LOGS="${LOGS_DIR}/events/tabby-server/*.json" # Init schema function init_scheme() { @@ -27,6 +27,9 @@ function collect_tabby_server_logs() { if compgen -G "${TABBY_SERVER_LOGS}" > /dev/null; then cat < 0) AS view, (SUM(IF(type == 'select', 1, 0)) > 0) AS select - FROM '${TABBY_SERVER_LOGS}' + FROM events WHERE completion_id IS NOT NULL GROUP BY 1 ) rhs ON (lhs.id = rhs.completion_id);