From 0f5a9592695b898eba10c788d5526b4698fb31a8 Mon Sep 17 00:00:00 2001
From: Meng Zhang <meng@tabbyml.com>
Date: Fri, 24 Mar 2023 09:44:12 +0800
Subject: [PATCH] Remove tokenizer in testdata/

---
 deployment/.gitignore                             | 3 +--
 deployment/docker-compose.triton.yml              | 4 +---
 deployment/docker-compose.yml                     | 6 +++---
 testdata/tiny-70M/tokenizer/.gitattributes        | 1 -
 testdata/tiny-70M/tokenizer/tokenizer.json        | 3 ---
 testdata/tiny-70M/tokenizer/tokenizer_config.json | 7 -------
 6 files changed, 5 insertions(+), 19 deletions(-)
 delete mode 100644 testdata/tiny-70M/tokenizer/.gitattributes
 delete mode 100644 testdata/tiny-70M/tokenizer/tokenizer.json
 delete mode 100644 testdata/tiny-70M/tokenizer/tokenizer_config.json

diff --git a/deployment/.gitignore b/deployment/.gitignore
index f9a1874..1269488 100644
--- a/deployment/.gitignore
+++ b/deployment/.gitignore
@@ -1,2 +1 @@
-logs
-hf_cache
+data
diff --git a/deployment/docker-compose.triton.yml b/deployment/docker-compose.triton.yml
index f2c91d6..a4eb427 100644
--- a/deployment/docker-compose.triton.yml
+++ b/deployment/docker-compose.triton.yml
@@ -5,9 +5,7 @@ services:
     image: tabbyml/tabby
     environment:
       - MODEL_BACKEND=triton
-      - TRITON_TOKENIZER_NAME=/tokenizer
-    volumes:
-      - ../testdata/tiny-70M/tokenizer:/tokenizer
+      - TRITON_TOKENIZER_NAME=EleutherAI/pythia-70m-deduped
     links:
       - triton
 
diff --git a/deployment/docker-compose.yml b/deployment/docker-compose.yml
index 4b0e441..4fdb86e 100644
--- a/deployment/docker-compose.yml
+++ b/deployment/docker-compose.yml
@@ -12,8 +12,8 @@ services:
     ports:
       - "5000:5000"
     volumes:
-      - ./logs:/logs
-      - ./hf_cache:/root/.cache/huggingface
+      - ./data/logs:/logs
+      - ./data/hf_cache:/root/.cache/huggingface
     links:
       - vector
 
@@ -33,4 +33,4 @@ services:
     container_name: tabby-vector
     volumes:
       - ./config/vector.toml:/etc/vector/vector.toml:ro
-      - ./logs:/logs
+      - ./data/logs:/logs
diff --git a/testdata/tiny-70M/tokenizer/.gitattributes b/testdata/tiny-70M/tokenizer/.gitattributes
deleted file mode 100644
index fb67717..0000000
--- a/testdata/tiny-70M/tokenizer/.gitattributes
+++ /dev/null
@@ -1 +0,0 @@
-tokenizer.json filter=lfs diff=lfs merge=lfs -text
diff --git a/testdata/tiny-70M/tokenizer/tokenizer.json b/testdata/tiny-70M/tokenizer/tokenizer.json
deleted file mode 100644
index ca8e44a..0000000
--- a/testdata/tiny-70M/tokenizer/tokenizer.json
+++ /dev/null
@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:e74ca9064c4f0d2232253bfe5f48caa78b6827212e87d831899d8fc64542e62b
-size 2113711
diff --git a/testdata/tiny-70M/tokenizer/tokenizer_config.json b/testdata/tiny-70M/tokenizer/tokenizer_config.json
deleted file mode 100644
index 83d54d0..0000000
--- a/testdata/tiny-70M/tokenizer/tokenizer_config.json
+++ /dev/null
@@ -1,7 +0,0 @@
-{
-  "add_prefix_space": false,
-  "bos_token": "<|endoftext|>",
-  "eos_token": "<|endoftext|>",
-  "tokenizer_class": "GPTNeoXTokenizer",
-  "unk_token": "<|endoftext|>"
-}