BUG: handle Qwen update (#307)

xorbitsai · Aug 4, 2023 · 98765f2 · 98765f2
1 parent b0723ba
commit 98765f2
Show file tree

Hide file tree

Showing 2 changed files with 7 additions and 1 deletion.
diff --git a/setup.cfg b/setup.cfg
@@ -70,6 +70,7 @@ all =
     bitsandbytes
     protobuf
     einops
+    tiktoken
 ggml =
     chatglm-cpp
     llama-cpp-python>=0.1.77
@@ -82,6 +83,7 @@ pytorch =
     bitsandbytes
     protobuf
     einops
+    tiktoken
 doc =
     ipython>=6.5.0
     sphinx>=3.0.0,<5.0.0

diff --git a/xinference/model/llm/pytorch/utils.py b/xinference/model/llm/pytorch/utils.py
@@ -104,7 +104,11 @@ def generate_stream(
         temperature, repetition_penalty, top_p, top_k
     )
 
-    input_ids = tokenizer(prompt).input_ids
+    if "qwen" in str(type(model)).lower():
+        # TODO: hacky
+        input_ids = tokenizer(prompt, allowed_special="all").input_ids
+    else:
+        input_ids = tokenizer(prompt).input_ids
     output_ids = list(input_ids)
 
     if model.config.is_encoder_decoder: