root
/
tabby
mirror of https://gitee.com/zhang_1334717033/tabby.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

fix: llama.cpp warmp logic

r0.5
Meng Zhang 2023-11-04 14:12:31 -07:00
parent 0b6108dfc2
commit 01ce18fe1a
1 changed files with 3 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
crates/llama-cpp-bindings/src/engine.cc
View File

@ -90,7 +90,8 @@ class TextInferenceEngineImpl : public TextInferenceEngine {
batch_ = llama_batch_init(N_CTX * N_CONCURRENT_REQUESTS, 0, 1); batch_ = llama_batch_init(N_CTX * N_CONCURRENT_REQUESTS, 0, 1);
// warm up // warm up
{ {
for (int i = 0; i < 16; ++i) { batch_.n_tokens = 16;
for (int i = 0; i < batch_.n_tokens; ++i) {
batch_.token[i] = 0; batch_.token[i] = 0;
batch_.pos[i] = i; batch_.pos[i] = i;
batch_.n_seq_id[0] = 1; batch_.n_seq_id[0] = 1;
@ -98,7 +99,7 @@ class TextInferenceEngineImpl : public TextInferenceEngine {
batch_.logits[i] = false; batch_.logits[i] = false;
} }
if (!llama_decode(ctx_.get(), batch_)) { if (llama_decode(ctx_.get(), batch_)) {
fprintf(stderr, "%s: warmup failed\n", __func__); fprintf(stderr, "%s: warmup failed\n", __func__);
} }