Remove tokenizer in testdata/

deployment/.gitignore

@@ -1,2 +1 @@
-logs
+data
-hf_cache

deployment/docker-compose.triton.yml

@@ -5,9 +5,7 @@ services:
     image: tabbyml/tabby
     environment:
       - MODEL_BACKEND=triton
-      - TRITON_TOKENIZER_NAME=/tokenizer
+      - TRITON_TOKENIZER_NAME=EleutherAI/pythia-70m-deduped
-    volumes:
-      - ../testdata/tiny-70M/tokenizer:/tokenizer
     links:
       - triton
 

deployment/docker-compose.yml

@@ -12,8 +12,8 @@ services:
     ports:
       - "5000:5000"
     volumes:
-      - ./logs:/logs
+      - ./data/logs:/logs
-      - ./hf_cache:/root/.cache/huggingface
+      - ./data/hf_cache:/root/.cache/huggingface
     links:
       - vector
 
@@ -33,4 +33,4 @@ services:
     container_name: tabby-vector
     volumes:
       - ./config/vector.toml:/etc/vector/vector.toml:ro
-      - ./logs:/logs
+      - ./data/logs:/logs

testdata/tiny-70M/tokenizer/.gitattributes

@@ -1 +0,0 @@
-tokenizer.json filter=lfs diff=lfs merge=lfs -text

testdata/tiny-70M/tokenizer/tokenizer.json

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:e74ca9064c4f0d2232253bfe5f48caa78b6827212e87d831899d8fc64542e62b
-size 2113711

testdata/tiny-70M/tokenizer/tokenizer_config.json

@@ -1,7 +0,0 @@
-{
-  "add_prefix_space": false,
-  "bos_token": "<|endoftext|>",
-  "eos_token": "<|endoftext|>",
-  "tokenizer_class": "GPTNeoXTokenizer",
-  "unk_token": "<|endoftext|>"
-}