Skip to content

Commit

Permalink
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Small tweaks to the EV and Market model
Browse files Browse the repository at this point in the history
kongzii committed Feb 13, 2024
1 parent 876dc6b commit ec0ff5b
Showing 2 changed files with 63 additions and 20 deletions.
32 changes: 23 additions & 9 deletions evo_researcher/benchmark/benchmark.py
Original file line number Diff line number Diff line change
@@ -318,21 +318,35 @@ def calculate_expected_returns(self, prediction: Prediction, market: Market) ->

# TODO: Add support for different bet sizes and calculate shares based on the market's odds.
bet_units = 10 # Assuming the agent always bet 10 units per market.
receive_shares = 20 # Because we assume markets trades at 50/50.
buy_yes_threshold = 0.5 # If the agent's prediction is > 50% it should buy "yes", otherwise "no".

assert prediction.outcome_prediction is not None
yes_shares = receive_shares if prediction.outcome_prediction.p_yes > buy_yes_threshold else 0
no_shares = receive_shares if prediction.outcome_prediction.p_yes <= buy_yes_threshold else 0

expected_returns_pct = (
# Assume that market starts at 50/50 and so the price is 0.5 at the time we are buying it,
# we can't use {yes,no}_outcome_price atm, because it would just cancel out to EV = 0.0,
# as it's the same as the probability.
yes_shares = (
bet_units / 0.5 # market.yes_outcome_price
if prediction.outcome_prediction.p_yes > buy_yes_threshold and market.yes_outcome_price > 0
else 0
)
no_shares = (
bet_units / 0.5 # market.no_outcome_price
if prediction.outcome_prediction.p_yes <= buy_yes_threshold and market.no_outcome_price > 0
else 0
)

# If we don't bet, we don't have any expected returns.
if yes_shares == 0 and no_shares == 0:
return None

expected_value = (
yes_shares * market.p_yes
+ no_shares * (1 - market.p_yes)
- bet_units
)
expected_returns = 100 * expected_returns_pct / bet_units
expected_returns_perc = 100 * expected_value / bet_units

return expected_returns
return expected_returns_perc

def compute_expected_returns_summary(self) -> t.Tuple[dict[str, list[str | float]], dict[str, list[str | float | None]]]:
overall_summary: dict[str, list[str | float]] = defaultdict(list)
@@ -341,8 +355,8 @@ def compute_expected_returns_summary(self) -> t.Tuple[dict[str, list[str | float
expected_returns = []

for market in self.markets:
if (prediction := self.get_prediction(agent.agent_name, market.question)).is_answered:
expected_returns.append(check_not_none(self.calculate_expected_returns(prediction, market)))
if (prediction := self.get_prediction(agent.agent_name, market.question)).is_answered and (expected_return := self.calculate_expected_returns(prediction, market)) is not None:
expected_returns.append(expected_return)

overall_summary["Agent"].append(agent.agent_name)
overall_summary["Mean expected returns"].append(float(np.mean(expected_returns)))
51 changes: 40 additions & 11 deletions evo_researcher/benchmark/utils.py
Original file line number Diff line number Diff line change
@@ -2,7 +2,7 @@
import json
import requests
import typing as t
from pydantic import BaseModel
from pydantic import BaseModel, validator
from evo_researcher.functions.evaluate_question import EvalautedQuestion


@@ -18,6 +18,30 @@ class Market(BaseModel):
p_yes: float
volume: float
is_resolved: bool
resolution: str | None
outcomePrices: list[float] | None

@validator("outcomePrices", pre=True)
def _validate_outcome_prices(cls, value: t.Any) -> list[float] | None:
if value is None:
return None
if len(value) != 2:
raise ValueError("outcomePrices must have exactly 2 elements.")
return value

@property
def p_no(self) -> float:
return 1 - self.p_yes

@property
def yes_outcome_price(self) -> float:
# Use the outcome price if available, otherwise assume it's p_yes.
return self.outcomePrices[0] if self.outcomePrices else self.p_yes

@property
def no_outcome_price(self) -> float:
# Use the outcome price if available, otherwise assume it's p_yes.
return self.outcomePrices[1] if self.outcomePrices else 1 - self.p_yes


class OutcomePrediction(BaseModel):
@@ -70,13 +94,13 @@ def load(path: str) -> "PredictionsCache":


def get_manifold_markets(
number: int = 100, excluded_questions: t.List[str] = []
number: int = 100, excluded_questions: t.List[str] = [], filter_: t.Literal["open", "closed", "resolved", "closing-this-month", "closing-next-month"] = "open"
) -> t.List[Market]:
url = "https://api.manifold.markets/v0/search-markets"
params = {
"term": "",
"sort": "liquidity",
"filter": "open",
"filter": filter_,
"limit": f"{number + len(excluded_questions)}",
"contractType": "BINARY", # TODO support CATEGORICAL markets
}
@@ -97,7 +121,6 @@ def _map_fields(old: dict[str, str], mapping: dict[str, str]) -> dict[str, str]:
return {mapping.get(k, k): v for k, v in old.items()}

markets = [Market.parse_obj(_map_fields(m, fields_map)) for m in markets_json]
markets = [m for m in markets if not m.is_resolved]

# Filter out markets with excluded questions
markets = [m for m in markets if m.question not in excluded_questions]
@@ -106,16 +129,21 @@ def _map_fields(old: dict[str, str], mapping: dict[str, str]) -> dict[str, str]:


def get_polymarket_markets(
number: int = 100, excluded_questions: t.List[str] = []
number: int = 100, excluded_questions: t.List[str] = [], active: bool | None = True, closed: bool | None = False
) -> t.List[Market]:
if number > 100:
raise ValueError("Polymarket API only returns 100 markets at a time")

api_uri = f"https://strapi-matic.poly.market/markets?_limit={number + len(excluded_questions)}&active=true&closed=false"
ms_json = requests.get(api_uri).json()
params = {
"_limit": number + len(excluded_questions),
}
if active is not None:
params["active"] = "true" if active else "false"
if closed is not None:
params["closed"] = "true" if closed else "false"
api_uri = f"https://strapi-matic.poly.market/markets"
ms_json = requests.get(api_uri, params=params).json()
markets: t.List[Market] = []
for m_json in ms_json:
# Skip non-binary markets. Unfortunately no way to filter in the API call
# TODO support CATEGORICAL markets
if m_json["outcomes"] != ["Yes", "No"]:
continue

@@ -127,7 +155,8 @@ def get_polymarket_markets(
Market(
question=m_json["question"],
url=f"https://polymarket.com/event/{m_json['slug']}",
p_yes=m_json["outcomePrices"][0],
p_yes=m_json["outcomePrices"][0], # For binary markets on Polymarket, the first outcome is "Yes" and outcomePrices are equal to probabilities.
outcomePrices=m_json["outcomePrices"],
volume=m_json["volume"],
is_resolved=False,
source=MarketSource.POLYMARKET,

0 comments on commit ec0ff5b

Please sign in to comment.