From 4c0c205b4dae335c2c81fb0b83030da3ae830ae6 Mon Sep 17 00:00:00 2001 From: Peter Jung Date: Thu, 31 Oct 2024 15:21:55 +0100 Subject: [PATCH] Remove duplicated functions available in PMAT --- prediction_prophet/benchmark/agents.py | 10 +-- .../functions/evaluate_question.py | 61 ------------------ .../functions/is_predictable_and_binary.py | 62 ------------------- 3 files changed, 5 insertions(+), 128 deletions(-) delete mode 100644 prediction_prophet/functions/evaluate_question.py delete mode 100644 prediction_prophet/functions/is_predictable_and_binary.py diff --git a/prediction_prophet/benchmark/agents.py b/prediction_prophet/benchmark/agents.py index 6203071b..6f54adeb 100644 --- a/prediction_prophet/benchmark/agents.py +++ b/prediction_prophet/benchmark/agents.py @@ -11,7 +11,6 @@ from prediction_prophet.autonolas.research import EmbeddingModel from prediction_prophet.autonolas.research import make_prediction, get_urls_from_queries from prediction_prophet.autonolas.research import research as research_autonolas -from prediction_prophet.functions.evaluate_question import is_predictable from prediction_prophet.functions.rephrase_question import rephrase_question from prediction_prophet.functions.research import Research, research as prophet_research from prediction_prophet.functions.search import search @@ -26,6 +25,7 @@ from pydantic.types import SecretStr from prediction_prophet.autonolas.research import Prediction as LLMCompletionPredictionDict from prediction_market_agent_tooling.tools.langfuse_ import observe +from prediction_market_agent_tooling.tools.is_predictable import is_predictable_binary from prediction_market_agent_tooling.tools.tavily.tavily_storage import TavilyStorage if t.TYPE_CHECKING: @@ -112,11 +112,11 @@ def __init__( self.embedding_model = embedding_model def is_predictable(self, market_question: str) -> bool: - (result, _) = is_predictable(question=market_question) + result = is_predictable_binary(question=market_question) return result def is_predictable_restricted(self, market_question: str, time_restriction_up_to: datetime) -> bool: - (result, _) = is_predictable(question=market_question) + result = is_predictable_binary(question=market_question) return result def research(self, market_question: str) -> str: @@ -187,11 +187,11 @@ def __init__( self.logger = logger def is_predictable(self, market_question: str) -> bool: - (result, _) = is_predictable(question=market_question) + result = is_predictable_binary(question=market_question) return result def is_predictable_restricted(self, market_question: str, time_restriction_up_to: datetime) -> bool: - (result, _) = is_predictable(question=market_question) + result = is_predictable_binary(question=market_question) return result def research(self, market_question: str) -> Research: diff --git a/prediction_prophet/functions/evaluate_question.py b/prediction_prophet/functions/evaluate_question.py deleted file mode 100644 index d971c4b9..00000000 --- a/prediction_prophet/functions/evaluate_question.py +++ /dev/null @@ -1,61 +0,0 @@ -import json -from prediction_prophet.autonolas.research import clean_completion_json -from langchain_openai import ChatOpenAI -from langchain.prompts import ChatPromptTemplate -from prediction_prophet.functions.cache import persistent_inmemory_cache -from pydantic.types import SecretStr -from prediction_market_agent_tooling.tools.utils import secret_str_from_env -from prediction_market_agent_tooling.gtypes import secretstr_to_v1_secretstr - - -# I tried to make it return a JSON, but it didn't work well in combo with asking it to do chain of thought. -QUESTION_EVALUATE_PROMPT = """Main signs about an answerable question (sometimes reffered to as a "market"): -- The question needs to be specific, without use of pronouns. -- The question needs to have a clear future event. -- The question needs to have a clear time frame. -- The answer is probably Google-able, after the event happened. -- The question can not be about itself. - -Follow a chain of thought to evaluate if the question is answerable: - -First, write the parts of the following question: - -"{question}" - -Then, write down what is the future event of the question, what it reffers to and when that event will happen if the question contains it. - -Then, give your final decision about whether the question is answerable. - -Return a JSON object with the following structure: - -{{ - "is_predictable": bool, - "reasoning": string -}} - -Output only the JSON object in your response. Do not include any other contents in your response. -""" - - - -@persistent_inmemory_cache -def is_predictable( - question: str, - engine: str = "gpt-4-0125-preview", - prompt_template: str = QUESTION_EVALUATE_PROMPT, - api_key: SecretStr | None = None -) -> tuple[bool, str]: - """ - Evaluate if the question is actually answerable. - """ - - if api_key == None: - api_key = secret_str_from_env("OPENAI_API_KEY") - llm = ChatOpenAI(model=engine, temperature=0.0, api_key=secretstr_to_v1_secretstr(api_key)) - - prompt = ChatPromptTemplate.from_template(template=prompt_template) - messages = prompt.format_messages(question=question) - completion = str(llm(messages, max_tokens=256).content) - response = json.loads(clean_completion_json(completion)) - - return (response["is_predictable"], response["reasoning"]) diff --git a/prediction_prophet/functions/is_predictable_and_binary.py b/prediction_prophet/functions/is_predictable_and_binary.py deleted file mode 100644 index e579a091..00000000 --- a/prediction_prophet/functions/is_predictable_and_binary.py +++ /dev/null @@ -1,62 +0,0 @@ -import json -from prediction_prophet.autonolas.research import clean_completion_json -from langchain_openai import ChatOpenAI -from langchain.prompts import ChatPromptTemplate -from prediction_prophet.functions.cache import persistent_inmemory_cache -from pydantic.types import SecretStr -from prediction_market_agent_tooling.tools.utils import secret_str_from_env -from prediction_market_agent_tooling.gtypes import secretstr_to_v1_secretstr - - -# I tried to make it return a JSON, but it didn't work well in combo with asking it to do chain of thought. -QUESTION_EVALUATE_PROMPT = """Main signs about an answerable question (sometimes reffered to as a "market"): -- The question needs to be specific, without use of pronouns. -- The question needs to have a clear future event. -- The question needs to have a clear time frame. -- The answer is probably Google-able, after the event happened. -- The question can not be about itself. -- The question needs to be a yes or no question. - -Follow a chain of thought to evaluate if the question is answerable: - -First, write the parts of the following question: - -"{question}" - -Then, write down what is the future event of the question, what it reffers to and when that event will happen if the question contains it. - -Then, give your final decision about whether the question is answerable. - -Return a JSON object with the following structure: - -{{ - "is_predictable": bool, - "reasoning": string -}} - -Output only the JSON object in your response. Do not include any other contents in your response. -""" - - - -@persistent_inmemory_cache -def is_predictable_and_binary( - question: str, - engine: str = "gpt-4-0125-preview", - prompt_template: str = QUESTION_EVALUATE_PROMPT, - api_key: SecretStr | None = None -) -> tuple[bool, str]: - """ - Evaluate if the question is actually answerable. - """ - - if api_key == None: - api_key = secret_str_from_env("OPENAI_API_KEY") - llm = ChatOpenAI(model=engine, temperature=0.0, api_key=secretstr_to_v1_secretstr(api_key)) - - prompt = ChatPromptTemplate.from_template(template=prompt_template) - messages = prompt.format_messages(question=question) - completion = str(llm(messages, max_tokens=256).content) - response = json.loads(clean_completion_json(completion)) - - return (response["is_predictable"], response["reasoning"])