feat(experimental): visualize with streamlit (#290)

experimental/eval/eval.sh

@@ -17,9 +17,7 @@ while ! curl -X POST http://localhost:8080/v1/health; do
   sleep 5
 done
 
-papermill main.ipynb ./reports.ipynb -r filepattern "./tabby/dataset/*.jsonl" -r max_records "${MAX_RECORDS:-3}"
-
-jupyter nbconvert reports.ipynb --TagRemovePreprocessor.enabled=True --TagRemovePreprocessor.remove_cell_tags remove --to html
+python main.py "./tabby/dataset/*.jsonl" ${MAX_RECORDS:-3} > reports.jsonl
 
 docker-compose down
 

experimental/eval/eval_sky.sh

@@ -7,5 +7,4 @@ if ! sky exec $ARGS; then
   sky launch -c $ARGS
 fi
 
-scp tabby-eval:~/sky_workdir/reports.ipynb ./
-scp tabby-eval:~/sky_workdir/reports.html ./
+scp tabby-eval:~/sky_workdir/reports.jsonl ./

experimental/eval/processing.py

@@ -3,7 +3,6 @@ from typing import Iterator
 import glob
 import json
 from dataclasses import dataclass
-from transformers import HfArgumentParser
 
 
 @dataclass

experimental/eval/requirements.txt

@@ -1,6 +1,2 @@
-papermill
 git+https://github.com/TabbyML/tabby.git#egg=tabby-python-client&subdirectory=clients/tabby-python-client
-transformers
 editdistance
-matplotlib
-notebook

experimental/eval/visualize.py

@@ -0,0 +1,23 @@
+import streamlit as st
+import pandas as pd
+import altair as alt
+
+st.set_page_config(layout="wide")
+
+df = pd.read_json("reports.jsonl", lines=True)
+
+for _, v in df.iterrows():
+    col1, col2, col3 = st.columns(3)
+    with col1:
+        st.write("prompt")
+        st.code(v.prompt)
+    with col2:
+        st.write("prediction")
+        st.code(v.prediction)
+        st.write("label")
+        st.code(v.label)
+    with col3:
+        col1, col2 = st.columns(2)
+        st.metric("Line score", v.line_score)
+        st.metric("Block score", v.block_score)
+    st.divider()