root
/
tabby
mirror of https://gitee.com/zhang_1334717033/tabby.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Revert "feat: deprecate num_replicas_per_thread, generate default value for it" 

This reverts commit b3b498624c.
r0.3
Meng Zhang 2023-10-19 00:12:28 -07:00
parent aa6f39985c
commit 6ae42edeb9
2 changed files with 4 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
crates/tabby/src/serve/engine.rs
View File

@ -5,7 +5,6 @@ use serde::Deserialize;
use tabby_common::path::ModelDir;
use tabby_inference::TextGeneration;
use super::Device;
use crate::fatal;
pub fn create_engine(
@ -69,26 +68,13 @@ fn create_ctranslate2_engine(
) -> Box<dyn TextGeneration> {
let device = format!("{}", args.device);
let compute_type = format!("{}", args.compute_type);
let num_replicas_per_device = {
let num_cpus = std::thread::available_parallelism()
.expect("Failed to read # of cpu")
.get();
if args.device == Device::Cuda {
// When device is cuda, set parallelism to be number of thread.
num_cpus
} else {
// Otherwise, adjust the number based on threads per replica.
// https://github.com/OpenNMT/CTranslate2/blob/master/src/utils.cc#L77
std::cmp::max(num_cpus / 4, 1)
}
};
let options = CTranslate2EngineOptionsBuilder::default()
.model_path(model_dir.ctranslate2_dir())
.tokenizer_path(model_dir.tokenizer_file())
.device(device)
.model_type(metadata.auto_model.clone())
.device_indices(args.device_indices.clone())
.num_replicas_per_device(num_replicas_per_device)
.num_replicas_per_device(args.num_replicas_per_device)
.compute_type(compute_type)
.build()
.unwrap();

10
crates/tabby/src/serve/mod.rs
View File

@ -139,9 +139,9 @@ pub struct ServeArgs {
#[clap(long, default_values_t=[0])]
device_indices: Vec<i32>,
/// DEPRECATED: Do not use.
#[clap(long)]
num_replicas_per_device: Option<usize>,
/// Number of replicas per device, only applicable for CPU.
#[clap(long, default_value_t = 1)]
num_replicas_per_device: usize,
/// Compute type
#[clap(long, default_value_t=ComputeType::Auto)]
@ -281,10 +281,6 @@ fn fallback() -> routing::MethodRouter {
}
fn valid_args(args: &ServeArgs) {
if args.num_replicas_per_device.is_some() {
warn!("num_replicas_per_device is deprecated and will be removed in future release.");
}
if args.device == Device::Cpu && (args.device_indices.len() != 1 || args.device_indices[0] != 0)
{
fatal!("CPU device only supports device indices = [0]");