Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Specify the maximum number of elements per page each endpoint can return. #359

Closed
wants to merge 3 commits into from
Closed

Specify the maximum number of elements per page each endpoint can return. #359

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
1083eff
Adapt http.request to support pagination
graille Nov 12, 2022
28f0865
Update http.py methods to support max_elements_per_page attribute
graille Nov 12, 2022
70b8b66
Update changelog with previous changes
graille Nov 12, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 1 addition & 2 deletions docs/changelog.rst
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,10 +9,9 @@ Master
- Updated docstring regarding new HypeTrain contribution method ``OTHER`` for :attr:`~twitchio.HypeTrainContribution.type`
- Add support for ``ciso8601`` if installed
- Added ``speed`` install flag (``pip install twitchio[speed]``) to install all available speedups

- API endpoint call can now return the maximum number of data allowed by their "first" parameter.
- Bug fixes
- Fix :func:`~twitchio.PartialUser.fetch_bits_leaderboard` not handling ``started_at`` and :class:`~twitchio.BitsLeaderboard` not correctly parsing

- ext.eventsub
- Additions
- Updated docs regarding new HypeTrain contribution method ``other`` for :attr:`~twitchio.ext.eventsub.HypeTrainContributor.type`
Expand Down
174 changes: 135 additions & 39 deletions twitchio/http.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -97,7 +97,15 @@ def __init__(
self.bucket = RateBucket(method="http")
self.scopes = kwargs.get("scopes", [])

async def request(self, route: Route, *, paginate=True, limit=100, full_body=False, force_app_token=False):
async def request(
self,
route: Route, *,
paginate: bool = True,
limit: Optional[int] = None,
full_body: bool = False,
force_app_token: bool = False,
max_elements_per_page: Optional[int] = None
):
"""
Fulfills an API request

Expand All @@ -106,22 +114,33 @@ async def request(self, route: Route, *, paginate=True, limit=100, full_body=Fal
route : :class:`twitchio.http.Route`
The route to follow
paginate : :class:`bool`
whether or not to paginate the requests where possible. Defaults to True
Whether or not to paginate the requests where possible. Defaults to True
limit : :class:`int`
The data limit per request when paginating. Defaults to 100
The (maximum) total number of data to return. Defaults to None
full_body : class:`bool`
Whether to return the full response body or to accumulate the `data` key. Defaults to False. `paginate` must be False if this is True.
Whether to return the full response body or to accumulate the `data` key. Defaults to False.
`paginate` is inefficient if this is True.
force_app_token : :class:`bool`
Forcibly use the client_id and client_secret generated token, if available. Otherwise fail the request immediately
Forcibly use the client_id and client_secret generated token, if available.
Otherwise, fail the request immediately.
max_elements_per_page : :class:`int`
The maximum number of elements per page.
For paginated endpoints, this is the maximum value of the `first` parameter of the HTTP Request
This value is usually equal to 100, but for some specific endpoint, this value can be different.
Moreover, some endpoints doesn't use pagination or doesn't allow to specify the number of output elements,
In that situation, this parameter must be set to None to avoid the usage of `first` HTTP parameter.
"""
if full_body:
assert not paginate
if (not self.client_id or not self.nick) and self.token:
await self.validate(token=self.token)
if not self.client_id:
raise errors.NoClientID("A Client ID is required to use the Twitch API")
headers = route.headers or {}

# When limit is not specified, it means we want the maximum number of elements allowed in only one request
# This number is equal to the value of the `max_elements_per_page`parameter
if limit is None:
limit = max_elements_per_page

if force_app_token and "Authorization" not in headers:
if not self.client_secret:
raise errors.NoToken(
Expand All @@ -132,7 +151,8 @@ async def request(self, route: Route, *, paginate=True, limit=100, full_body=Fal
headers["Authorization"] = f"Bearer {self.app_token}"
elif not self.token and not self.client_secret and "Authorization" not in headers:
raise errors.NoToken(
"Authorization is required to use the Twitch API. Pass token and/or client_secret to the Client constructor"
"Authorization is required to use the Twitch API. "
"Pass token and/or client_secret to the Client constructor"
)
if "Authorization" not in headers:
if not self.token:
Expand All @@ -147,14 +167,22 @@ async def request(self, route: Route, *, paginate=True, limit=100, full_body=Fal
cursor = None
data = []

def reached_limit():
return limit and len(data) >= limit
def reached_limit() -> bool:
# If we are not using pagination, we are done at the end of the request
if paginate is False:
return True

def get_limit():
# If we don't have any limit, we use the build-in limit of the endpoint,
# (which should be equal to `max_elements_per_page`).
# In that case, we are done after the first request.
if limit is None:
return "100"
return True

return len(data) >= limit

def get_nb_elements_in_next_page() -> str:
to_get = limit - len(data)
return str(to_get) if to_get < 100 else "100"
return str(min(to_get, max_elements_per_page))

is_finished = False
while not is_finished:
Expand All @@ -164,7 +192,12 @@ def get_limit():
q = route.query or []
if cursor is not None:
q = [("after", cursor), *q]
q = [("first", get_limit()), *q]

# HTTP `first` parameter specify the maximum number of elements the returned page will contain.
# If `max_elements_per_page` is None, it means the endpoint doesn't use this parameter
# In this case, we don't specify it
if max_elements_per_page is not None:
q = [("first", get_nb_elements_in_next_page()), *q]
path = path.with_query(q)
body, is_text = await self._request(route, path, headers)
if is_text:
Expand All @@ -180,7 +213,7 @@ def get_limit():
else:
if not cursor:
break
is_finished = reached_limit() if limit is not None else True if paginate else True
is_finished = reached_limit()
return data

async def _request(self, route, path, headers, utilize_bucket=True):
Expand Down Expand Up @@ -350,7 +383,10 @@ async def get_extension_transactions(self, extension_id: str, ids: List[Any] = N
q = [("extension_id", extension_id)]
if ids:
q.extend(("id", id) for id in ids)
return await self.request(Route("GET", "extensions/transactions", "", query=q))
return await self.request(
Route("GET", "extensions/transactions", "", query=q),
max_elements_per_page=100
)

async def create_reward(
self,
Expand Down Expand Up @@ -462,14 +498,18 @@ async def get_reward_redemptions(
sort: str = "OLDEST",
first: int = 20,
):
params = [("broadcaster_id", str(broadcaster_id)), ("reward_id", reward_id), ("first", first)]
params = [("broadcaster_id", str(broadcaster_id)), ("reward_id", reward_id)]
if redemption_id:
params.append(("id", redemption_id))
if status:
params.append(("status", status))
if sort:
params.append(("sort", sort))
return await self.request(Route("GET", "channel_points/custom_rewards/redemptions", query=params, token=token))
return await self.request(
Route("GET", "channel_points/custom_rewards/redemptions", query=params, token=token),
limit=first,
max_elements_per_page=50
)

async def update_reward_redemption_status(
self, token: str, broadcaster_id: int, reward_id: str, custom_reward_id: str, status: bool
Expand All @@ -496,7 +536,10 @@ async def get_predictions(

if prediction_id:
params.extend(("prediction_id", prediction_id))
return await self.request(Route("GET", "predictions", query=params, token=token), paginate=False)
return await self.request(
Route("GET", "predictions", query=params, token=token),
max_elements_per_page=25
)

async def patch_prediction(
self, token: str, broadcaster_id: str, prediction_id: str, status: str, winning_outcome_id: Optional[str] = None
Expand Down Expand Up @@ -572,7 +615,10 @@ async def get_clips(
q.extend(("id", id) for id in ids)
query = [x for x in q if x[1] is not None]

return await self.request(Route("GET", "clips", query=query, token=token))
return await self.request(
Route("GET", "clips", query=query, token=token),
max_elements_per_page=100
)

async def post_entitlements_upload(self, manifest_id: str, type="bulk_drops_grant"):
return await self.request(
Expand All @@ -581,7 +627,8 @@ async def post_entitlements_upload(self, manifest_id: str, type="bulk_drops_gran

async def get_entitlements(self, id: str = None, user_id: str = None, game_id: str = None):
return await self.request(
Route("GET", "entitlements/drops", query=[("id", id), ("user_id", user_id), ("game_id", game_id)])
Route("GET", "entitlements/drops", query=[("id", id), ("user_id", user_id), ("game_id", game_id)]),
max_elements_per_page=1000
)

async def get_code_status(self, codes: List[str], user_id: int):
Expand All @@ -597,7 +644,10 @@ async def post_redeem_code(self, user_id: int, codes: List[str]):
return await self.request(Route("POST", "entitlements/code", query=q))

async def get_top_games(self):
return await self.request(Route("GET", "games/top"))
return await self.request(
Route("GET", "games/top"),
max_elements_per_page=100
)

async def get_games(self, game_ids: List[Any], game_names: List[str]):
q = []
Expand All @@ -614,7 +664,8 @@ async def get_hype_train(self, broadcaster_id: str, id: Optional[str] = None, to
"hypetrain/events",
query=[x for x in [("broadcaster_id", broadcaster_id), ("id", id)] if x[1] is not None],
token=token,
)
),
max_elements_per_page=100
)

async def post_automod_check(self, token: str, broadcaster_id: str, *msgs: List[Dict[str, str]]):
Expand All @@ -639,13 +690,19 @@ async def get_channel_bans(self, token: str, broadcaster_id: str, user_ids: List
q = [("broadcaster_id", broadcaster_id)]
if user_ids:
q.extend(("user_id", id) for id in user_ids)
return await self.request(Route("GET", "moderation/banned", query=q, token=token))
return await self.request(
Route("GET", "moderation/banned", query=q, token=token),
max_elements_per_page=100
)

async def get_channel_moderators(self, token: str, broadcaster_id: str, user_ids: List[str] = None):
q = [("broadcaster_id", broadcaster_id)]
if user_ids:
q.extend(("user_id", id) for id in user_ids)
return await self.request(Route("GET", "moderation/moderators", query=q, token=token))
return await self.request(
Route("GET", "moderation/moderators", query=q, token=token),
max_elements_per_page=100
)

async def get_channel_mod_events(self, token: str, broadcaster_id: str, user_ids: List[str] = None):
q = [("broadcaster_id", broadcaster_id)]
Expand All @@ -654,11 +711,15 @@ async def get_channel_mod_events(self, token: str, broadcaster_id: str, user_ids
return await self.request(Route("GET", "moderation/moderators/events", query=q, token=token))

async def get_search_categories(self, query: str, token: str = None):
return await self.request(Route("GET", "search/categories", query=[("query", query)], token=token))
return await self.request(
Route("GET", "search/categories", query=[("query", query)], token=token),
max_elements_per_page=100
)

async def get_search_channels(self, query: str, token: str = None, live: bool = False):
return await self.request(
Route("GET", "search/channels", query=[("query", query), ("live_only", str(live))], token=token)
Route("GET", "search/channels", query=[("query", query), ("live_only", str(live))], token=token),
max_elements_per_page=100
)

async def get_stream_key(self, token: str, broadcaster_id: str):
Expand All @@ -683,7 +744,10 @@ async def get_streams(
q.extend(("user_login", l) for l in user_logins)
if languages:
q.extend(("language", l) for l in languages)
return await self.request(Route("GET", "streams", query=q, token=token))
return await self.request(
Route("GET", "streams", query=q, token=token),
max_elements_per_page=100
)

async def post_stream_marker(self, token: str, user_id: str, description: str = None):
return await self.request(
Expand All @@ -697,7 +761,8 @@ async def get_stream_markers(self, token: str, user_id: str = None, video_id: st
"streams/markers",
query=[x for x in [("user_id", user_id), ("video_id", video_id)] if x[1] is not None],
token=token,
)
),
max_elements_per_page=100
)

async def get_channels(self, broadcaster_id: str, token: Optional[str] = None):
Expand Down Expand Up @@ -743,7 +808,6 @@ async def get_channel_schedule(
x
for x in [
("broadcaster_id", broadcaster_id),
("first", first),
("start_time", start_time),
("utc_offset", utc_offset),
]
Expand All @@ -752,19 +816,30 @@ async def get_channel_schedule(

if segment_ids:
q.extend(("id", id) for id in segment_ids)
return await self.request(Route("GET", "schedule", query=q), paginate=False, full_body=True)
return await self.request(
Route("GET", "schedule", query=q),
full_body=True,
limit=first,
max_elements_per_page=25
)

async def get_channel_subscriptions(self, token: str, broadcaster_id: str, user_ids: Optional[List[str]] = None):
q = [("broadcaster_id", broadcaster_id)]
if user_ids:
q.extend(("user_id", u) for u in user_ids)
return await self.request(Route("GET", "subscriptions", query=q, token=token))
return await self.request(
Route("GET", "subscriptions", query=q, token=token),
max_elements_per_page=100
)

async def get_stream_tags(self, tag_ids: Optional[List[str]] = None):
q = []
if tag_ids:
q.extend(("tag_id", u) for u in tag_ids)
return await self.request(Route("GET", "tags/streams", query=q or None))
return await self.request(
Route("GET", "tags/streams", query=q or None),
max_elements_per_page=100
)

async def get_channel_tags(self, broadcaster_id: str):
return await self.request(Route("GET", "streams/tags", query=[("broadcaster_id", broadcaster_id)]))
Expand Down Expand Up @@ -812,7 +887,8 @@ async def get_user_follows(
"users/follows",
query=[x for x in [("from_id", from_id), ("to_id", to_id)] if x[1] is not None],
token=token,
)
),
max_elements_per_page=100
)

async def put_update_user(self, token: str, description: str):
Expand Down Expand Up @@ -863,7 +939,18 @@ async def get_videos(

if ids:
q.extend(("id", id) for id in ids)
return await self.request(Route("GET", "videos", query=q, token=token))

# According to documentation, `first` parameter should only be specified when user_id or game_id are specified
if (user_id is None) and (game_id is None):
return await self.request(
Route("GET", "videos", query=q, token=token),
max_elements_per_page=None
)

return await self.request(
Route("GET", "videos", query=q, token=token),
max_elements_per_page=100
)

async def delete_videos(self, token: str, ids: List[int]):
q = [("id", str(x)) for x in ids]
Expand Down Expand Up @@ -899,11 +986,16 @@ async def get_polls(
raise ValueError("poll_ids can only have up to 100 entries")
if first and (first > 25 or first < 1):
raise ValueError("first can only be between 1 and 20")
q = [("broadcaster_id", broadcaster_id), ("first", first)]
q = [("broadcaster_id", broadcaster_id)]

if poll_ids:
q.extend(("id", poll_id) for poll_id in poll_ids)
return await self.request(Route("GET", "polls", query=q, token=token), paginate=False, full_body=True)
return await self.request(
Route("GET", "polls", query=q, token=token),
full_body=True,
limit=first,
max_elements_per_page=20
)

async def post_poll(
self,
Expand Down Expand Up @@ -1032,14 +1124,18 @@ async def delete_channel_moderator(self, token: str, broadcaster_id: str, user_i
async def get_channel_vips(
self, token: str, broadcaster_id: str, first: int = 20, user_ids: Optional[List[int]] = None
):
q = [("broadcaster_id", broadcaster_id), ("first", first)]
q = [("broadcaster_id", broadcaster_id)]
if first > 100:
raise ValueError("You can only get up to 100 VIPs at once")
if user_ids:
if len(user_ids) > 100:
raise ValueError("You can can only specify up to 100 VIPs")
q.extend(("user_id", str(user_id)) for user_id in user_ids)
return await self.request(Route("GET", "channels/vips", query=q, token=token))
return await self.request(
Route("GET", "channels/vips", query=q, token=token),
limit=first,
max_elements_per_page=100
)

async def post_channel_vip(self, token: str, broadcaster_id: str, user_id: str):
q = [("broadcaster_id", broadcaster_id), ("user_id", user_id)]
Expand Down