Merge pull request #137 from kldzj/fix-tool-calling

fix: openai tool calling
runpod-workers · Dec 16, 2024 · d2e355e · d2e355e
2 parents d9f54ce + b7787d8
commit d2e355e
Show file tree

Hide file tree

Showing 2 changed files with 3 additions and 3 deletions.
diff --git a/src/engine.py b/src/engine.py
@@ -146,6 +146,8 @@ async def _initialize_engines(self):
             base_model_paths=self.base_model_paths,
             response_role=self.response_role,
             chat_template=self.tokenizer.tokenizer.chat_template,
+            enable_auto_tools=os.getenv('ENABLE_AUTO_TOOL_CHOICE', 'false').lower() == 'true',
+            tool_parser=os.getenv('TOOL_CALL_PARSER', "") or None,
             lora_modules=lora_modules,
             prompt_adapters=None,
             request_logger=None

diff --git a/src/engine_args.py b/src/engine_args.py
@@ -89,9 +89,7 @@
     "qlora_adapter_name_or_path": os.getenv('QLORA_ADAPTER_NAME_OR_PATH', None),
     "disable_logprobs_during_spec_decoding": os.getenv('DISABLE_LOGPROBS_DURING_SPEC_DECODING', None),
     "otlp_traces_endpoint": os.getenv('OTLP_TRACES_ENDPOINT', None),
-    "use_v2_block_manager": os.getenv('USE_V2_BLOCK_MANAGER', 'true'),
-    "enable_auto_tool_choice": os.getenv('ENABLE_AUTO_TOOL_CHOICE', 'false').lower() == 'true',
-    "tool_call_parser": os.getenv('TOOL_CALL_PARSER', "") or None
+    "use_v2_block_manager": os.getenv('USE_V2_BLOCK_MANAGER', 'true')
 }
 
 def match_vllm_args(args):