From ec0ff5bfbf909763d35110e93cebdabce7b92e94 Mon Sep 17 00:00:00 2001 From: Peter Jung Date: Tue, 13 Feb 2024 11:01:42 +0100 Subject: [PATCH] Small tweaks to the EV and Market model --- evo_researcher/benchmark/benchmark.py | 32 ++++++++++++----- evo_researcher/benchmark/utils.py | 51 +++++++++++++++++++++------ 2 files changed, 63 insertions(+), 20 deletions(-) diff --git a/evo_researcher/benchmark/benchmark.py b/evo_researcher/benchmark/benchmark.py index 9a6b1a0f..6c07fb96 100644 --- a/evo_researcher/benchmark/benchmark.py +++ b/evo_researcher/benchmark/benchmark.py @@ -318,21 +318,35 @@ def calculate_expected_returns(self, prediction: Prediction, market: Market) -> # TODO: Add support for different bet sizes and calculate shares based on the market's odds. bet_units = 10 # Assuming the agent always bet 10 units per market. - receive_shares = 20 # Because we assume markets trades at 50/50. buy_yes_threshold = 0.5 # If the agent's prediction is > 50% it should buy "yes", otherwise "no". assert prediction.outcome_prediction is not None - yes_shares = receive_shares if prediction.outcome_prediction.p_yes > buy_yes_threshold else 0 - no_shares = receive_shares if prediction.outcome_prediction.p_yes <= buy_yes_threshold else 0 - - expected_returns_pct = ( + # Assume that market starts at 50/50 and so the price is 0.5 at the time we are buying it, + # we can't use {yes,no}_outcome_price atm, because it would just cancel out to EV = 0.0, + # as it's the same as the probability. + yes_shares = ( + bet_units / 0.5 # market.yes_outcome_price + if prediction.outcome_prediction.p_yes > buy_yes_threshold and market.yes_outcome_price > 0 + else 0 + ) + no_shares = ( + bet_units / 0.5 # market.no_outcome_price + if prediction.outcome_prediction.p_yes <= buy_yes_threshold and market.no_outcome_price > 0 + else 0 + ) + + # If we don't bet, we don't have any expected returns. + if yes_shares == 0 and no_shares == 0: + return None + + expected_value = ( yes_shares * market.p_yes + no_shares * (1 - market.p_yes) - bet_units ) - expected_returns = 100 * expected_returns_pct / bet_units + expected_returns_perc = 100 * expected_value / bet_units - return expected_returns + return expected_returns_perc def compute_expected_returns_summary(self) -> t.Tuple[dict[str, list[str | float]], dict[str, list[str | float | None]]]: overall_summary: dict[str, list[str | float]] = defaultdict(list) @@ -341,8 +355,8 @@ def compute_expected_returns_summary(self) -> t.Tuple[dict[str, list[str | float expected_returns = [] for market in self.markets: - if (prediction := self.get_prediction(agent.agent_name, market.question)).is_answered: - expected_returns.append(check_not_none(self.calculate_expected_returns(prediction, market))) + if (prediction := self.get_prediction(agent.agent_name, market.question)).is_answered and (expected_return := self.calculate_expected_returns(prediction, market)) is not None: + expected_returns.append(expected_return) overall_summary["Agent"].append(agent.agent_name) overall_summary["Mean expected returns"].append(float(np.mean(expected_returns))) diff --git a/evo_researcher/benchmark/utils.py b/evo_researcher/benchmark/utils.py index 11050c70..703006fb 100644 --- a/evo_researcher/benchmark/utils.py +++ b/evo_researcher/benchmark/utils.py @@ -2,7 +2,7 @@ import json import requests import typing as t -from pydantic import BaseModel +from pydantic import BaseModel, validator from evo_researcher.functions.evaluate_question import EvalautedQuestion @@ -18,6 +18,30 @@ class Market(BaseModel): p_yes: float volume: float is_resolved: bool + resolution: str | None + outcomePrices: list[float] | None + + @validator("outcomePrices", pre=True) + def _validate_outcome_prices(cls, value: t.Any) -> list[float] | None: + if value is None: + return None + if len(value) != 2: + raise ValueError("outcomePrices must have exactly 2 elements.") + return value + + @property + def p_no(self) -> float: + return 1 - self.p_yes + + @property + def yes_outcome_price(self) -> float: + # Use the outcome price if available, otherwise assume it's p_yes. + return self.outcomePrices[0] if self.outcomePrices else self.p_yes + + @property + def no_outcome_price(self) -> float: + # Use the outcome price if available, otherwise assume it's p_yes. + return self.outcomePrices[1] if self.outcomePrices else 1 - self.p_yes class OutcomePrediction(BaseModel): @@ -70,13 +94,13 @@ def load(path: str) -> "PredictionsCache": def get_manifold_markets( - number: int = 100, excluded_questions: t.List[str] = [] + number: int = 100, excluded_questions: t.List[str] = [], filter_: t.Literal["open", "closed", "resolved", "closing-this-month", "closing-next-month"] = "open" ) -> t.List[Market]: url = "https://api.manifold.markets/v0/search-markets" params = { "term": "", "sort": "liquidity", - "filter": "open", + "filter": filter_, "limit": f"{number + len(excluded_questions)}", "contractType": "BINARY", # TODO support CATEGORICAL markets } @@ -97,7 +121,6 @@ def _map_fields(old: dict[str, str], mapping: dict[str, str]) -> dict[str, str]: return {mapping.get(k, k): v for k, v in old.items()} markets = [Market.parse_obj(_map_fields(m, fields_map)) for m in markets_json] - markets = [m for m in markets if not m.is_resolved] # Filter out markets with excluded questions markets = [m for m in markets if m.question not in excluded_questions] @@ -106,16 +129,21 @@ def _map_fields(old: dict[str, str], mapping: dict[str, str]) -> dict[str, str]: def get_polymarket_markets( - number: int = 100, excluded_questions: t.List[str] = [] + number: int = 100, excluded_questions: t.List[str] = [], active: bool | None = True, closed: bool | None = False ) -> t.List[Market]: - if number > 100: - raise ValueError("Polymarket API only returns 100 markets at a time") - - api_uri = f"https://strapi-matic.poly.market/markets?_limit={number + len(excluded_questions)}&active=true&closed=false" - ms_json = requests.get(api_uri).json() + params = { + "_limit": number + len(excluded_questions), + } + if active is not None: + params["active"] = "true" if active else "false" + if closed is not None: + params["closed"] = "true" if closed else "false" + api_uri = f"https://strapi-matic.poly.market/markets" + ms_json = requests.get(api_uri, params=params).json() markets: t.List[Market] = [] for m_json in ms_json: # Skip non-binary markets. Unfortunately no way to filter in the API call + # TODO support CATEGORICAL markets if m_json["outcomes"] != ["Yes", "No"]: continue @@ -127,7 +155,8 @@ def get_polymarket_markets( Market( question=m_json["question"], url=f"https://polymarket.com/event/{m_json['slug']}", - p_yes=m_json["outcomePrices"][0], + p_yes=m_json["outcomePrices"][0], # For binary markets on Polymarket, the first outcome is "Yes" and outcomePrices are equal to probabilities. + outcomePrices=m_json["outcomePrices"], volume=m_json["volume"], is_resolved=False, source=MarketSource.POLYMARKET,