added model load functionality for /v1/cat/completion

extensions/openai/models.py

@@ -24,9 +24,12 @@ def list_dummy_models():
         "data": []
     }
 
+    models = get_available_models()[1:]
     # these are expected by so much, so include some here as a dummy
     for model in ['gpt-3.5-turbo', 'text-embedding-ada-002']:
         result["data"].append(model_info_dict(model))
+    for model in models:
+        result["data"].append(model_info_dict(model))
 
     return result
 
@@ -42,8 +45,8 @@ def model_info_dict(model_name: str) -> dict:
 
 def _load_model(data):
     model_name = data["model_name"]
-    args = data["args"]
-    settings = data["settings"]
+    args = data.get("args", None)
+    settings = data.get("settings", None)
 
     unload_model()
     model_settings = get_model_metadata(model_name)

extensions/openai/script.py

@@ -119,6 +119,17 @@ async def generator():
 
 @app.post('/v1/chat/completions', response_model=ChatCompletionResponse, dependencies=check_key)
 async def openai_chat_completions(request: Request, request_data: ChatCompletionRequest):
+    requested_model = request_data.model
+    payload = OAImodels.get_current_model_info()
+    current_model = payload["model_name"]
+    if not current_model == requested_model:
+        requested_model_dict = {"model_name": requested_model}
+        try:
+            OAImodels._load_model(requested_model_dict)
+        except:
+            traceback.print_exc()
+            return HTTPException(status_code=400, detail="Failed to load the model.")
+
     path = request.url.path
     is_legacy = "/generate" in path