From 98298eea5a578928882075c2c6e8313dbe122cf7 Mon Sep 17 00:00:00 2001 From: saikolasani Date: Wed, 12 Jun 2024 08:22:14 -0700 Subject: [PATCH 1/3] Add Support for Other Providers OpenAI, AzureOpenAI, Bedrock, LiteLLM, Langchain, Huggingface --- ragulate/cli_commands/query.py | 19 ++++++++++++++++++ ragulate/pipelines/query_pipeline.py | 30 +++++++++++++++++++++++++--- 2 files changed, 46 insertions(+), 3 deletions(-) diff --git a/ragulate/cli_commands/query.py b/ragulate/cli_commands/query.py index 6853199..2cfc56c 100644 --- a/ragulate/cli_commands/query.py +++ b/ragulate/cli_commands/query.py @@ -3,9 +3,12 @@ from ragulate.datasets import find_dataset from ragulate.pipelines import QueryPipeline +<<<<<<< HEAD from ..utils import convert_vars_to_ingredients +======= +>>>>>>> 273a5ff (Add Support for Other Providers OpenAI, AzureOpenAI, Bedrock, LiteLLM, Langchain, Huggingface) def setup_query(subparsers): query_parser = subparsers.add_parser("query", help="Run an query pipeline") query_parser.add_argument( @@ -90,6 +93,18 @@ def setup_query(subparsers): ), action="store_true", ) + query_parser.add_argument( + "--llm_provider", + type=str, + help=("The name of the LLM Provider to use for Evaluation."), + choices=["OpenAI", "AzureOpenAI", "Bedrock", "LiteLLM", "Langchain", "Huggingface"], + + ) + query_parser.add_argument( + "--model_name", + type=str, + help=("The name or id of the LLM model or deployment to use for Evaluation."), + ) query_parser.set_defaults(func=lambda args: call_query(**vars(args))) def call_query( @@ -103,6 +118,8 @@ def call_query( sample: float, seed: int, restart: bool, + llm_provider: str, + model_name: str, **kwargs, ): if sample <= 0.0 or sample > 1.0: @@ -131,5 +148,7 @@ def call_query( sample_percent=sample, random_seed=seed, restart_pipeline=restart, + llm_provider=llm_provider, + model_name=model_name, ) query_pipeline.query() diff --git a/ragulate/pipelines/query_pipeline.py b/ragulate/pipelines/query_pipeline.py index 20aa441..d9e71e2 100644 --- a/ragulate/pipelines/query_pipeline.py +++ b/ragulate/pipelines/query_pipeline.py @@ -5,7 +5,8 @@ from tqdm import tqdm from trulens_eval import Tru, TruChain -from trulens_eval.feedback.provider import OpenAI +from trulens_eval.feedback.provider import OpenAI, AzureOpenAI, Bedrock, LiteLLM, Langchain, Huggingface +from trulens_eval.feedback.provider.base import LLMProvider from trulens_eval.schema.feedback import FeedbackMode, FeedbackResultStatus from ragulate.datasets import BaseDataset @@ -47,7 +48,9 @@ def __init__( datasets: List[BaseDataset], sample_percent: float = 1.0, random_seed: Optional[int] = None, - restart_pipeline: bool = False, + restart_pipeline: Optional[bool] = False, + llm_provider: Optional[str] = "OpenAI", + model_name: Optional[str] = None, **kwargs, ): super().__init__( @@ -61,6 +64,8 @@ def __init__( self.sample_percent = sample_percent self.random_seed = random_seed self.restart_pipeline = restart_pipeline + self.llm_provider = llm_provider + self.model_name=model_name # Set up the signal handler for SIGINT (Ctrl-C) signal.signal(signal.SIGINT, self.signal_handler) @@ -139,11 +144,30 @@ def update_progress(self, query_change: int = 0): self._finished_feedbacks = done + def get_provider(self) -> LLMProvider: + provider_name = self.provider_name.lower() + model_name = self.model_name + + if provider_name == 'openai': + return OpenAI(model_engine=model_name) + elif provider_name == 'azureopenai': + return AzureOpenAI(deployment_name=model_name) + elif provider_name == 'bedrock': + return Bedrock(model_id=model_name) + elif provider_name == 'litellm': + return LiteLLM(model_engine=model_name) + elif provider_name == 'Langchain': + return Langchain(model_engine=model_name) + elif provider_name == 'huggingface': + return Huggingface(name=model_name) + else: + raise ValueError(f"Unsupported provider: {provider_name}") + def query(self): query_method = self.get_method() pipeline = query_method(**self.ingredients) - llm_provider = OpenAI(model_engine="gpt-3.5-turbo") + llm_provider = self.get_provider() feedbacks = Feedbacks(llm_provider=llm_provider, pipeline=pipeline) From d6efb3b6bc63f2d87617855cd4d50baf7349fdbc Mon Sep 17 00:00:00 2001 From: Eric Pinzur Date: Thu, 27 Jun 2024 12:35:11 +0200 Subject: [PATCH 2/3] fmt --- ragulate/cli_commands/query.py | 13 ++++++++----- ragulate/pipelines/query_pipeline.py | 23 +++++++++++++++-------- 2 files changed, 23 insertions(+), 13 deletions(-) diff --git a/ragulate/cli_commands/query.py b/ragulate/cli_commands/query.py index 2cfc56c..1c262bd 100644 --- a/ragulate/cli_commands/query.py +++ b/ragulate/cli_commands/query.py @@ -3,12 +3,9 @@ from ragulate.datasets import find_dataset from ragulate.pipelines import QueryPipeline -<<<<<<< HEAD from ..utils import convert_vars_to_ingredients -======= ->>>>>>> 273a5ff (Add Support for Other Providers OpenAI, AzureOpenAI, Bedrock, LiteLLM, Langchain, Huggingface) def setup_query(subparsers): query_parser = subparsers.add_parser("query", help="Run an query pipeline") query_parser.add_argument( @@ -97,8 +94,14 @@ def setup_query(subparsers): "--llm_provider", type=str, help=("The name of the LLM Provider to use for Evaluation."), - choices=["OpenAI", "AzureOpenAI", "Bedrock", "LiteLLM", "Langchain", "Huggingface"], - + choices=[ + "OpenAI", + "AzureOpenAI", + "Bedrock", + "LiteLLM", + "Langchain", + "Huggingface", + ], ) query_parser.add_argument( "--model_name", diff --git a/ragulate/pipelines/query_pipeline.py b/ragulate/pipelines/query_pipeline.py index d9e71e2..6513cad 100644 --- a/ragulate/pipelines/query_pipeline.py +++ b/ragulate/pipelines/query_pipeline.py @@ -5,7 +5,14 @@ from tqdm import tqdm from trulens_eval import Tru, TruChain -from trulens_eval.feedback.provider import OpenAI, AzureOpenAI, Bedrock, LiteLLM, Langchain, Huggingface +from trulens_eval.feedback.provider import ( + AzureOpenAI, + Bedrock, + Huggingface, + Langchain, + LiteLLM, + OpenAI, +) from trulens_eval.feedback.provider.base import LLMProvider from trulens_eval.schema.feedback import FeedbackMode, FeedbackResultStatus @@ -65,7 +72,7 @@ def __init__( self.random_seed = random_seed self.restart_pipeline = restart_pipeline self.llm_provider = llm_provider - self.model_name=model_name + self.model_name = model_name # Set up the signal handler for SIGINT (Ctrl-C) signal.signal(signal.SIGINT, self.signal_handler) @@ -148,17 +155,17 @@ def get_provider(self) -> LLMProvider: provider_name = self.provider_name.lower() model_name = self.model_name - if provider_name == 'openai': + if provider_name == "openai": return OpenAI(model_engine=model_name) - elif provider_name == 'azureopenai': + elif provider_name == "azureopenai": return AzureOpenAI(deployment_name=model_name) - elif provider_name == 'bedrock': + elif provider_name == "bedrock": return Bedrock(model_id=model_name) - elif provider_name == 'litellm': + elif provider_name == "litellm": return LiteLLM(model_engine=model_name) - elif provider_name == 'Langchain': + elif provider_name == "Langchain": return Langchain(model_engine=model_name) - elif provider_name == 'huggingface': + elif provider_name == "huggingface": return Huggingface(name=model_name) else: raise ValueError(f"Unsupported provider: {provider_name}") From 21047c829babf9953d29fde2320bf2e9c587acd8 Mon Sep 17 00:00:00 2001 From: Eric Pinzur Date: Thu, 27 Jun 2024 14:04:35 +0200 Subject: [PATCH 3/3] shorter param names --- ragulate/cli_commands/query.py | 30 +++++++++++++++++------------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/ragulate/cli_commands/query.py b/ragulate/cli_commands/query.py index 1c262bd..88f7759 100644 --- a/ragulate/cli_commands/query.py +++ b/ragulate/cli_commands/query.py @@ -17,14 +17,14 @@ def setup_query(subparsers): ) query_parser.add_argument( "-s", - "--script_path", + "--script", type=str, help="The path to the python script that contains the query method", required=True, ) query_parser.add_argument( "-m", - "--method-name", + "--method", type=str, help="The name of the method in the script to run query", required=True, @@ -91,7 +91,7 @@ def setup_query(subparsers): action="store_true", ) query_parser.add_argument( - "--llm_provider", + "--provider", type=str, help=("The name of the LLM Provider to use for Evaluation."), choices=[ @@ -102,18 +102,22 @@ def setup_query(subparsers): "Langchain", "Huggingface", ], + default="OpenAI", ) query_parser.add_argument( - "--model_name", + "--model", type=str, - help=("The name or id of the LLM model or deployment to use for Evaluation."), + help=( + "The name or id of the LLM model or deployment to use for Evaluation.", + "Generally used in combination with the --provider param.", + ), ) query_parser.set_defaults(func=lambda args: call_query(**vars(args))) def call_query( name: str, - script_path: str, - method_name: str, + script: str, + method: str, var_name: List[str], var_value: List[str], dataset: List[str], @@ -121,8 +125,8 @@ def call_query( sample: float, seed: int, restart: bool, - llm_provider: str, - model_name: str, + provider: str, + model: str, **kwargs, ): if sample <= 0.0 or sample > 1.0: @@ -144,14 +148,14 @@ def call_query( query_pipeline = QueryPipeline( recipe_name=name, - script_path=script_path, - method_name=method_name, + script_path=script, + method_name=method, ingredients=ingredients, datasets=datasets, sample_percent=sample, random_seed=seed, restart_pipeline=restart, - llm_provider=llm_provider, - model_name=model_name, + llm_provider=provider, + model_name=model, ) query_pipeline.query()