Add Completion Events & Acceptance Rate in metrics panel. (#26)

* Add duckdb * Add basic Metrics w/duckdb
2023-03-28 20:12:03 +08:00 · 2023-03-28 20:12:03 +08:00 · d966f05abd
parent 2f8714e6fe
commit d966f05abd
8 changed files with 189 additions and 3 deletions
--- a/deployment/docker-compose.yml
+++ b/deployment/docker-compose.yml
@ -33,6 +33,8 @@ services:
    command: streamlit run tabby/admin/Home.py
    ports:
      - "8501:8501"
+    volumes:
+      - ${LOGS_VOLUME}

  triton:
    image: tabbyml/fastertransformer_backend
@ -66,3 +68,15 @@ services:
    volumes:
      - ./config/vector.toml:/etc/vector/vector.toml:ro
      - ${LOGS_VOLUME}
+
+  dagu:
+    image: tabbyml/tabby
+    container_name: tabby-dagu
+    command: dagu scheduler
+    volumes:
+      - ${LOGS_VOLUME}
+    environment:
+      DAGU_DAGS: tabby/tasks
+    depends_on:
+      init:
+        condition: service_completed_successfully
--- a/poetry.lock
+++ b/poetry.lock
@ -670,6 +670,63 @@ files = [
    {file = "distlib-0.3.6.tar.gz", hash = "sha256:14bad2d9b04d3a36127ac97f30b12a19268f211063d8f8ee4f47108896e11b46"},
 ]

+[[package]]
+name = "duckdb"
+version = "0.7.1"
+description = "DuckDB embedded database"
+category = "main"
+optional = false
+python-versions = "*"
+files = [
+    {file = "duckdb-0.7.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:3e0170be6cc315c179169dfa3e06485ef7009ef8ce399cd2908f29105ef2c67b"},
+    {file = "duckdb-0.7.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:6360d41023e726646507d5479ba60960989a09f04527b36abeef3643c61d8c48"},
+    {file = "duckdb-0.7.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:578c269d7aa27184e8d45421694f89deda3f41fe6bd2a8ce48b262b9fc975326"},
+    {file = "duckdb-0.7.1-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:36aae9a923c9f78da1cf3fcf75873f62d32ea017d4cef7c706d16d3eca527ca2"},
+    {file = "duckdb-0.7.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:630e0122a02f19bb1fafae00786350b2c31ae8422fce97c827bd3686e7c386af"},
+    {file = "duckdb-0.7.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:9b9ca2d294725e523ce207bc37f28787478ae6f7a223e2cf3a213a2d498596c3"},
+    {file = "duckdb-0.7.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:0bd89f388205b6c99b62650169efe9a02933555ee1d46ddf79fbd0fb9e62652b"},
+    {file = "duckdb-0.7.1-cp310-cp310-win32.whl", hash = "sha256:a9e987565a268fd8da9f65e54621d28f39c13105b8aee34c96643074babe6d9c"},
+    {file = "duckdb-0.7.1-cp310-cp310-win_amd64.whl", hash = "sha256:5d986b5ad1307b069309f9707c0c5051323e29865aefa059eb6c3b22dc9751b6"},
+    {file = "duckdb-0.7.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:54606dfd24d7181d3098030ca6858f6be52f3ccbf42fff05f7587f2d9cdf4343"},
+    {file = "duckdb-0.7.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:bd9367ae650b6605ffe00412183cf0edb688a5fc9fbb03ed757e8310e7ec3b6c"},
+    {file = "duckdb-0.7.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:aaf33aeb543c7816bd915cd10141866d54f92f698e1b5712de9d8b7076da19df"},
+    {file = "duckdb-0.7.1-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2e56b0329c38c0356b40449917bab6fce6ac27d356257b9a9da613d2a0f064e0"},
+    {file = "duckdb-0.7.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:604b8b476d6cc6bf91625d8c2722ef9c50c402b3d64bc518c838d6c279e6d93b"},
+    {file = "duckdb-0.7.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:32a268508c6d7fdc99d5442736051de74c28a5166c4cc3dcbbf35d383299b941"},
+    {file = "duckdb-0.7.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:90794406fa2111414877ee9db154fef940911f3920c312c1cf69947621737c8d"},
+    {file = "duckdb-0.7.1-cp311-cp311-win32.whl", hash = "sha256:bf20c5ee62cbbf10b39ebdfd70d454ce914e70545c7cb6cb78cb5befef96328a"},
+    {file = "duckdb-0.7.1-cp311-cp311-win_amd64.whl", hash = "sha256:bb2700785cab37cd1e7a76c4547a5ab0f8a7c28ad3f3e4d02a8fae52be223090"},
+    {file = "duckdb-0.7.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:b09741cfa31388b8f9cdf5c5200e0995d55a5b54d2d1a75b54784e2f5c042f7f"},
+    {file = "duckdb-0.7.1-cp36-cp36m-win32.whl", hash = "sha256:766e6390f7ace7f1e322085c2ca5d0ad94767bde78a38d168253d2b0b4d5cd5c"},
+    {file = "duckdb-0.7.1-cp36-cp36m-win_amd64.whl", hash = "sha256:6a3f3315e2b553db3463f07324f62dfebaf3b97656a87558e59e2f1f816eaf15"},
+    {file = "duckdb-0.7.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:278edb8c912d836b3b77fd1695887e1dbd736137c3912478af3608c9d7307bb0"},
+    {file = "duckdb-0.7.1-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e189b558d10b58fe6ed85ce79f728e143eb4115db1e63147a44db613cd4dd0d9"},
+    {file = "duckdb-0.7.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6b91ec3544ee4dc9e6abbdf2669475d5adedaaea51987c67acf161673e6b7443"},
+    {file = "duckdb-0.7.1-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:3fe3f3dbd62b76a773144eef31aa29794578c359da932e77fef04516535318ca"},
+    {file = "duckdb-0.7.1-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:1e78c7f59325e99f0b3d9fe7c2bad4aaadf42d2c7711925cc26331d7647a91b2"},
+    {file = "duckdb-0.7.1-cp37-cp37m-win32.whl", hash = "sha256:bc2a12d9f4fc8ef2fd1022d610287c9fc9972ea06b7510fc87387f1fa256a390"},
+    {file = "duckdb-0.7.1-cp37-cp37m-win_amd64.whl", hash = "sha256:53e3db1bc0f445ee48b23cde47bfba08c7fa5a69976c740ec8cdf89543d2405d"},
+    {file = "duckdb-0.7.1-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:1247cc11bac17f2585d11681329806c86295e32242f84a10a604665e697d5c81"},
+    {file = "duckdb-0.7.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:5feaff16a012075b49dfa09d4cb24455938d6b0e06b08e1404ec00089119dba2"},
+    {file = "duckdb-0.7.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:b411a0c361eab9b26dcd0d0c7a0d1bc0ad6b214068555de7e946fbdd2619961a"},
+    {file = "duckdb-0.7.1-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c7c76d8694ecdb579241ecfeaf03c51d640b984dbbe8e1d9f919089ebf3cdea6"},
+    {file = "duckdb-0.7.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:193b896eed44d8751a755ccf002a137630020af0bc3505affa21bf19fdc90df3"},
+    {file = "duckdb-0.7.1-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:7da132ee452c80a3784b8daffd86429fa698e1b0e3ecb84660db96d36c27ad55"},
+    {file = "duckdb-0.7.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:5fd08c97c3e8cb5bec3822cf78b966b489213dcaab24b25c05a99f7caf8db467"},
+    {file = "duckdb-0.7.1-cp38-cp38-win32.whl", hash = "sha256:9cb956f94fa55c4782352dac7cc7572a58312bd7ce97332bb14591d6059f0ea4"},
+    {file = "duckdb-0.7.1-cp38-cp38-win_amd64.whl", hash = "sha256:289a5f65213e66d320ebcd51a94787e7097b9d1c3492d01a121a2c809812bf19"},
+    {file = "duckdb-0.7.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:8085ad58c9b5854ee3820804fa1797e6b3134429c1506c3faab3cb96e71b07e9"},
+    {file = "duckdb-0.7.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:b47c19d1f2f662a5951fc6c5f6939d0d3b96689604b529cdcffd9afdcc95bff2"},
+    {file = "duckdb-0.7.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:6a611f598226fd634b7190f509cc6dd668132ffe436b0a6b43847b4b32b99e4a"},
+    {file = "duckdb-0.7.1-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6730f03b5b78f3943b752c90bdf37b62ae3ac52302282a942cc675825b4a8dc9"},
+    {file = "duckdb-0.7.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fe23e938d29cd8ea6953d77dc828b7f5b95a4dbc7cd7fe5bcc3531da8cec3dba"},
+    {file = "duckdb-0.7.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:feffe503c2e2a99480e1e5e15176f37796b3675e4dadad446fe7c2cc672aed3c"},
+    {file = "duckdb-0.7.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:72fceb06f5bf24ad6bb5974c60d397a7a7e61b3d847507a22276de076f3392e2"},
+    {file = "duckdb-0.7.1-cp39-cp39-win32.whl", hash = "sha256:c4d5217437d20d05fe23317bbc161befa1f9363f3622887cd1d2f4719b407936"},
+    {file = "duckdb-0.7.1-cp39-cp39-win_amd64.whl", hash = "sha256:066885e1883464ce3b7d1fd844f9431227dcffe1ee39bfd2a05cd6d53f304557"},
+    {file = "duckdb-0.7.1.tar.gz", hash = "sha256:a7db6da0366b239ea1e4541fcc19556b286872f5015c9a54c2e347146e25a2ad"},
+]
+
 [[package]]
 name = "entrypoints"
 version = "0.4"
@ -3493,4 +3550,4 @@ testing = ["coverage (>=5.0.3)", "zope.event", "zope.testing"]
 [metadata]
 lock-version = "2.0"
 python-versions = "^3.10"
-content-hash = "895f90bc2f1bbe6847af480e6ae049b31c463ab134af7f5895d618d4662da551"
+content-hash = "8fa4ae65c0f2bfedd73b2784cb2d24cd99d151d130b8a434a6e8142bd2c16e24"
--- a/pyproject.toml
+++ b/pyproject.toml
@ -19,6 +19,7 @@ gitup = "^0.5.1"
 toml = "^0.10.2"
 gitpython = "^3.1.31"
 peft = {git = "https://github.com/huggingface/peft.git", rev = "v0.2.0"}
+duckdb = "^0.7.1"


 [tool.poetry.group.dev.dependencies]
--- a/tabby/admin/Home.py
+++ b/tabby/admin/Home.py
@ -12,7 +12,7 @@ def make_badge_markdown(x: ServiceInfo):
    return f"![{x.label}]({x.badge_url})"


-st.set_page_config(page_title="Tabby Admin")
+st.set_page_config(page_title="Tabby Admin - Home")

 st.markdown("## Tabby")
 st.markdown(" ".join(map(make_badge_markdown, SERVICES)))
--- a/tabby/admin/pages/Metrics.py
+++ b/tabby/admin/pages/Metrics.py
@ -0,0 +1,41 @@
+import os
+
+import duckdb
+import streamlit as st
+
+st.set_page_config(page_title="Tabby Admin - Metrics")
+
+
+def query_data():
+    filepath = os.environ.get("DB_FILE", "/logs/duckdb/duck.db")
+    if not os.path.isfile(filepath):
+        return []
+
+    conn = duckdb.connect(filepath)
+
+    df = conn.sql(
+        """
+    SELECT
+        date_trunc('hour', to_timestamp(CAST(created AS int64))) AS Date,
+        SUM(IF(view, 1, 0)) as "Views",
+        SUM(IF("select", 1, 0)) as "Acceptances"
+    FROM completion_events
+    GROUP BY 1;
+    """
+    ).df()
+
+    conn.close()
+    return df
+
+
+df = query_data()
+
+if len(df) > 0:
+    st.markdown("### Completion Events")
+    st.line_chart(df, x="Date")
+
+    st.markdown("### Acceptance Rate")
+    df["Acceptance Rate"] = df["Acceptances"] / df["Views"]
+    st.line_chart(df, x="Date", y="Acceptance Rate")
+else:
+    st.markdown("No data available")
--- a/tabby/tasks/duckdb.yaml
+++ b/tabby/tasks/duckdb.yaml
@ -0,0 +1,9 @@
+schedule: "*/3 * * * *" # Run every 3rd minute
+
+env:
+  - PATH: "/opt/conda/bin:$PATH"
+  - APP_DIR: /app
+steps:
+  - name: Collect Tabby
+    dir: $APP_DIR
+    command: ./tabby/tools/duckdb/main.sh collect_tabby_server_logs
--- a/tabby/tasks/trainer.yaml
+++ b/tabby/tasks/trainer.yaml
@ -1,4 +1,4 @@
-schedule: "5 4 * * *" # Run at 04:05.
+# schedule: "5 4 * * *" # Run at 04:05.

 env:
  - PATH: "/opt/conda/bin:$PATH"
--- a/tabby/tools/duckdb/main.sh
+++ b/tabby/tools/duckdb/main.sh
@ -0,0 +1,64 @@
+#!/bin/bash
+set -e
+
+DB_FILE=${DB_FILE:-"/logs/duckdb/duck.db"}
+LOGS_DIR=${LOGS_DIR:-"/logs"}
+TABBY_SERVER_LOGS="${LOGS_DIR}/tabby-server/events.*.json"
+
+# Init schema
+function init_scheme() {
+mkdir -p $(dirname $DB_FILE)
+cat <<EOF | duckdb
+CREATE TABLE IF NOT EXISTS completion_events (
+  id STRING,
+  created uint64,
+  prompt STRING,
+  choices STRUCT(index UINT64, text STRING)[],
+  view BOOLEAN,
+  "select" BOOLEAN
+);
+CREATE UNIQUE INDEX IF NOT EXISTS completion_events_id ON completion_events (id);
+EOF
+}
+
+# Update table
+function collect_tabby_server_logs() {
+cat <<EOF | duckdb
+CREATE TEMP TABLE t AS
+SELECT id, created, prompt, choices, IFNULL(rhs.view, false) AS view, IFNULL(rhs.select, false) AS select
+FROM
+  (
+    SELECT
+      id,
+      FIRST(created) AS created,
+      FIRST(prompt) AS prompt,
+      FIRST(choices) AS choices
+    FROM '${TABBY_SERVER_LOGS}' WHERE id IS NOT NULL GROUP BY 1) lhs
+LEFT JOIN (
+    SELECT
+      completion_id,
+      (SUM(IF(type == 'view', 1, 0)) > 0) AS view,
+      (SUM(IF(type == 'select', 1, 0)) > 0) AS select
+    FROM '${TABBY_SERVER_LOGS}'
+    WHERE completion_id IS NOT NULL
+    GROUP BY 1
+) rhs ON (lhs.id = rhs.completion_id);
+
+INSERT INTO completion_events SELECT t.* FROM t LEFT JOIN completion_events rhs ON (t.id = rhs.id) WHERE rhs.id IS NULL;
+EOF
+}
+
+function duckdb() {
+  local SQL=$(tee)
+  cat << EOF | python3 -
+import sys
+import duckdb
+conn = duckdb.connect('$DB_FILE')
+print(conn.sql("""
+$SQL
+"""))
+EOF
+}
+
+init_scheme
+"$@"