root
/
tabby
mirror of https://gitee.com/zhang_1334717033/tabby.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Update

add-more-languages
Meng Zhang 2023-03-20 23:07:20 +08:00
parent 0d8d7097be
commit fbcab616d7
1 changed files with 2 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
server/triton.py
View File

@ -25,10 +25,12 @@ class TritonService:
) )
def generate(self, data: CompletionsRequest) -> List[Choice]: def generate(self, data: CompletionsRequest) -> List[Choice]:
# FIXME(meng): Make following vars configurable
n = 1 n = 1
np_type = np.uint32 np_type = np.uint32
max_tokens = 128 max_tokens = 128
model_name = "fastertransformer" model_name = "fastertransformer"
stop_words = ["\n\n"]
prompt = data.prompt prompt = data.prompt
input_start_ids = np.expand_dims(self.tokenizer.encode(prompt), 0) input_start_ids = np.expand_dims(self.tokenizer.encode(prompt), 0)
@ -39,7 +41,6 @@ class TritonService:
prompt_tokens: int = input_len[0][0] prompt_tokens: int = input_len[0][0]
output_len = np.ones_like(input_len).astype(np_type) * max_tokens output_len = np.ones_like(input_len).astype(np_type) * max_tokens
stop_words = ["\n\n"]
stop_word_list = np.repeat( stop_word_list = np.repeat(
to_word_list_format([stop_words], self.tokenizer), to_word_list_format([stop_words], self.tokenizer),
input_start_ids.shape[0], input_start_ids.shape[0],