diff --git a/.dockerignore b/.dockerignore index 5bb8e94..3cab3d6 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,4 +1,6 @@ +data deployment +deployment-next **/.git **/node_modules diff --git a/.gitignore b/.gitignore index bee8a64..a00feb9 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ __pycache__ +data diff --git a/Makefile b/Makefile index ed870c6..829bf2d 100644 --- a/Makefile +++ b/Makefile @@ -14,19 +14,3 @@ $(PRE_COMMIT_HOOK): poetry run pre-commit install --install-hooks setup-development-environment: install-poetry $(PRE_COMMIT_HOOK) - - -UP_FLAGS := up --remove-orphans --remove-orphans -DEV_FLAGS := $(UP_FLAGS) --build - -up: - docker-compose -f deployment/docker-compose.yml $(UP_FLAGS) - -up-triton: - docker-compose -f deployment/docker-compose.yml -f deployment/docker-compose.triton.yml $(UP_FLAGS) - -dev: - docker-compose -f deployment/docker-compose.yml -f deployment/docker-compose.dev.yml $(DEV_FLAGS) - -dev-triton: - docker-compose -f deployment/docker-compose.yml -f deployment/docker-compose.triton.yml -f deployment/docker-compose.dev.yml $(DEV_FLAGS) diff --git a/README.md b/README.md index 2dcd22f..6dd56cd 100644 --- a/README.md +++ b/README.md @@ -18,8 +18,5 @@ Assuming Linux workstation with: Use `make setup-development-environment` to setup basic dev environment, and `make dev` to start local development server. ## Deployment -1. `make up-triton` -2. Open Admin Panel [http://localhost:8501](http://localhost:8501) -3. Types some code in editor! -![Screenshot from 2023-03-25 21-31-24](https://user-images.githubusercontent.com/388154/227720212-7e7480a1-abb1-4baf-b971-d391ea8136c8.png) +See [deployment](./deployment/README.md) diff --git a/deployment/.gitignore b/deployment-next/.gitignore similarity index 100% rename from deployment/.gitignore rename to deployment-next/.gitignore diff --git a/deployment-next/Makefile b/deployment-next/Makefile new file mode 100644 index 0000000..5837a0b --- /dev/null +++ b/deployment-next/Makefile @@ -0,0 +1,14 @@ +UP_FLAGS := up --remove-orphans --remove-orphans +DEV_FLAGS := $(UP_FLAGS) --build + +up: + docker-compose -f docker-compose.yml $(UP_FLAGS) + +up-triton: + docker-compose -f docker-compose.yml -f docker-compose.triton.yml $(UP_FLAGS) + +dev: + docker-compose -f docker-compose.yml -f docker-compose.dev.yml $(DEV_FLAGS) + +dev-triton: + docker-compose -f docker-compose.yml -f docker-compose.triton.yml -f docker-compose.dev.yml $(DEV_FLAGS) diff --git a/deployment-next/README.md b/deployment-next/README.md new file mode 100644 index 0000000..998a839 --- /dev/null +++ b/deployment-next/README.md @@ -0,0 +1,12 @@ +## Deployment + +1. Start service + ```bash + docker-compose up + ``` +2. Test API endpoint with curl + ```bash + curl -X POST http://localhost:5000/v1/completions -H 'Content-Type: application/json' --data '{ + "prompt": "def binarySearch(arr, left, right, x):\n mid = (left +" + }' + ``` diff --git a/deployment/config/repository.toml b/deployment-next/config/repository.toml similarity index 100% rename from deployment/config/repository.toml rename to deployment-next/config/repository.toml diff --git a/deployment/config/vector.toml b/deployment-next/config/vector.toml similarity index 100% rename from deployment/config/vector.toml rename to deployment-next/config/vector.toml diff --git a/deployment/docker-compose.dev.yml b/deployment-next/docker-compose.dev.yml similarity index 100% rename from deployment/docker-compose.dev.yml rename to deployment-next/docker-compose.dev.yml diff --git a/deployment/docker-compose.triton.yml b/deployment-next/docker-compose.triton.yml similarity index 94% rename from deployment/docker-compose.triton.yml rename to deployment-next/docker-compose.triton.yml index 11794d9..48b92bc 100644 --- a/deployment/docker-compose.triton.yml +++ b/deployment-next/docker-compose.triton.yml @@ -16,7 +16,7 @@ services: shm_size: 1gb volumes: - ./scripts:/scripts - - ./data/hf_cache:/root/.cache/huggingface + - ../data/hf_cache:/root/.cache/huggingface deploy: resources: reservations: diff --git a/deployment-next/docker-compose.yml b/deployment-next/docker-compose.yml new file mode 100644 index 0000000..c30f9c8 --- /dev/null +++ b/deployment-next/docker-compose.yml @@ -0,0 +1,50 @@ +version: '3.3' + +services: + init: + image: tabbyml/tabby + container_name: tabby-init + command: python -m tabby.tools.model_preload --repo_id TabbyML/NeoX-70M + volumes: + - ../data/hf_cache:/root/.cache/huggingface + + server: + image: tabbyml/tabby + container_name: tabby-server + command: uvicorn tabby.server:app --host 0.0.0.0 --port 5000 + environment: + - MODEL_NAME=TabbyML/NeoX-70M + - EVENTS_LOG_DIR=/logs + ports: + - "5000:5000" + volumes: + - ../data/logs/tabby-server:/logs + - ../data/hf_cache:/root/.cache/huggingface + depends_on: + init: + condition: service_completed_successfully + + admin: + image: tabbyml/tabby + container_name: tabby-admin + command: streamlit run tabby/admin/Home.py + ports: + - "8501:8501" + + vector: + image: timberio/vector:0.28.1-alpine + container_name: tabby-vector + volumes: + - ./config/vector.toml:/etc/vector/vector.toml:ro + - ../data/logs:/logs + + dagu: + image: tabbyml/tabby + container_name: tabby-dagu + command: dagu server --host 0.0.0.0 --port 8080 + volumes: + - ./config:/config:ro + - ../data/repositories:/repositories + - ../data/dataset:/dataset + environment: + DAGU_DAGS: tabby/tasks diff --git a/deployment-next/scripts/triton.sh b/deployment-next/scripts/triton.sh new file mode 100755 index 0000000..c86f3eb --- /dev/null +++ b/deployment-next/scripts/triton.sh @@ -0,0 +1,18 @@ +#!/bin/bash +set -e + +# Get model dir. +MODEL_DIR=$(python3 <