root
/
tabby
mirror of https://gitee.com/zhang_1334717033/tabby.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

fix: use int8_float16 to fix SantaCoder-1B (#237) 

#236
improve-workflow
Meng Zhang 2023-06-13 01:13:06 -07:00 committed by GitHub
parent 2691b302f0
commit 5985d91782
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 1 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
crates/ctranslate2-bindings/src/ctranslate2.cc
View File

@ -118,7 +118,7 @@ std::shared_ptr<TextInferenceEngine> create_engine(
if (loader.device == ctranslate2::Device::CPU) { if (loader.device == ctranslate2::Device::CPU) {
loader.compute_type = ctranslate2::ComputeType::INT8; loader.compute_type = ctranslate2::ComputeType::INT8;
} else if (loader.device == ctranslate2::Device::CUDA) { } else if (loader.device == ctranslate2::Device::CUDA) {
loader.compute_type = ctranslate2::ComputeType::FLOAT16; loader.compute_type = ctranslate2::ComputeType::INT8_FLOAT16;
} }
if (model_type_str == "AutoModelForCausalLM") { if (model_type_str == "AutoModelForCausalLM") {