root
/
tabby
mirror of https://gitee.com/zhang_1334717033/tabby.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

feat: allow set num_replicas_per_device for CUDA to increase throughput

r0.2
Meng Zhang 2023-10-03 15:52:25 -07:00
parent 375675c6b4
commit 1afba47059
1 changed files with 0 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
crates/tabby/src/serve/mod.rs
View File

@ -244,10 +244,6 @@ fn fallback() -> routing::MethodRouter {
}
fn valid_args(args: &ServeArgs) {
if args.device == Device::Cuda && args.num_replicas_per_device != 1 {
fatal!("CUDA device only supports 1 replicas per device");
}
if args.device == Device::Cpu && (args.device_indices.len() != 1 || args.device_indices[0] != 0)
{
fatal!("CPU device only supports device indices = [0]");