From 0d8ee26b7ecdd321d12e1dde80d2437dfeb3f6ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20N=C3=B8rgaard?= Date: Tue, 14 May 2024 16:54:55 +0100 Subject: [PATCH 01/15] first run at handling discord tokens --- config.template.toml | 3 +++ core/server.py | 5 ++++- launcher.py | 5 +++-- requirements.txt | 3 ++- types_/config.py | 7 ++++++- views/api.py | 38 ++++++++++++++++++++++++++++++++++++++ 6 files changed, 56 insertions(+), 5 deletions(-) diff --git a/config.template.toml b/config.template.toml index 5bfe96a..f1260b2 100644 --- a/config.template.toml +++ b/config.template.toml @@ -23,3 +23,6 @@ global_limit = { rate = 21600, per = 86400, priority = 1, bucket = "ip" } char_limit = 300_000 file_limit = 5 name_limit = 25 + +[GITHUB] # optional key +token = "..." # a github token capable of creating gists, non-optional if the above key is provided diff --git a/core/server.py b/core/server.py index 6f2d00f..3f0b394 100644 --- a/core/server.py +++ b/core/server.py @@ -18,6 +18,7 @@ import logging +import aiohttp import starlette_plus from starlette.middleware import Middleware from starlette.routing import Mount, Route @@ -34,9 +35,11 @@ class Application(starlette_plus.Application): - def __init__(self, *, database: Database) -> None: + def __init__(self, *, database: Database, session: aiohttp.ClientSession | None = None) -> None: self.database: Database = database + self.session: aiohttp.ClientSession | None = session self.schemas: SchemaGenerator | None = None + self._gist_token: str | None = CONFIG.get("GITHUB", {}).get("token") views: list[starlette_plus.View] = [HTMXView(self), APIView(self), DocsView(self)] routes: list[Mount | Route] = [Mount("/static", app=StaticFiles(directory="web/static"), name="static")] diff --git a/launcher.py b/launcher.py index cdc7899..46af322 100644 --- a/launcher.py +++ b/launcher.py @@ -19,6 +19,7 @@ import asyncio import logging +import aiohttp import starlette_plus import uvicorn @@ -31,8 +32,8 @@ async def main() -> None: - async with core.Database(dsn=core.CONFIG["DATABASE"]["dsn"]) as database: - app: core.Application = core.Application(database=database) + async with core.Database(dsn=core.CONFIG["DATABASE"]["dsn"]) as database, aiohttp.ClientSession() as session: + app: core.Application = core.Application(database=database, session=session) host: str = core.CONFIG["SERVER"]["host"] port: int = core.CONFIG["SERVER"]["port"] diff --git a/requirements.txt b/requirements.txt index b1a4174..0b255d8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,4 +5,5 @@ asyncpg-stubs bleach humanize python-multipart -pyyaml \ No newline at end of file +pyyaml +aiohttp diff --git a/types_/config.py b/types_/config.py index a94b8ad..e92ea50 100644 --- a/types_/config.py +++ b/types_/config.py @@ -16,7 +16,7 @@ along with this program. If not, see . """ -from typing import TypedDict +from typing import NotRequired, TypedDict import starlette_plus @@ -52,9 +52,14 @@ class Pastes(TypedDict): name_limit: int +class Github(TypedDict): + token: str + + class Config(TypedDict): SERVER: Server DATABASE: Database REDIS: Redis LIMITS: Limits PASTES: Pastes + GITHUB: NotRequired[Github] diff --git a/views/api.py b/views/api.py index 9756d50..3881701 100644 --- a/views/api.py +++ b/views/api.py @@ -20,6 +20,7 @@ import datetime import json +import re from typing import TYPE_CHECKING, Any import starlette_plus @@ -31,11 +32,45 @@ if TYPE_CHECKING: from core import Application +DISCORD_TOKEN_REGEX: re.Pattern[str] = re.compile(r"[a-zA-Z0-9_-]{23,28}\.[a-zA-Z0-9_-]{6,7}\.[a-zA-Z0-9_-]{27,}") + class APIView(starlette_plus.View, prefix="api"): def __init__(self, app: Application) -> None: self.app: Application = app + async def _handle_discord_tokens(self, *bodies: dict[str, str]) -> None: + # bodies is tuple[{content: ..., filename: ...}] + if not self.app._gist_token or not self.app.session: + return + + formatted_bodies = "\n".join(b["content"] for b in bodies) + + tokens = list(DISCORD_TOKEN_REGEX.finditer(formatted_bodies)) + + if not tokens: + return + tokens = "\n".join([m[0] for m in tokens]) + + await self._post_gist_of_tokens(tokens) + + async def _post_gist_of_tokens(self, tokens: str, /) -> None: + assert self.app.session # guarded in caller + + filename = str(datetime.datetime.now(datetime.UTC)) + "-tokens.txt" + json_payload = { + "description": "MystBin found these Discord tokens in a public paste, and posted them here to invalidate them. If you intended to share these, please apply a password to the paste.", + "files": {filename: {"content": tokens}}, + "public": True, + } + github_headers = { + "Accept": "application/vnd.github+json", + "Authorization": f"Bearer {self.app._gist_token}", + "X-GitHub-Api-Version": "2022-11-28", + } + + await self.app.session.post("https://api.github.com/gists", headers=github_headers, json=json_payload) + @starlette_plus.route("/paste/{id}", methods=["GET"]) @starlette_plus.limit(**CONFIG["LIMITS"]["paste_get"]) @starlette_plus.limit(**CONFIG["LIMITS"]["paste_get_day"]) @@ -259,6 +294,9 @@ async def paste_post(self, request: starlette_plus.Request) -> starlette_plus.Re body = (await request.body()).decode(encoding="UTF-8") data = {"files": [{"content": body, "filename": None}]} if isinstance(body, str) else body + + await self._handle_discord_tokens(*data["files"]) + if resp := validate_paste(data): return resp From e1646639c34f8ad4063c0288e8abfd177e9846f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20N=C3=B8rgaard?= Date: Tue, 14 May 2024 16:56:51 +0100 Subject: [PATCH 02/15] only invlidate on public pastes --- views/api.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/views/api.py b/views/api.py index 3881701..b34b1dc 100644 --- a/views/api.py +++ b/views/api.py @@ -295,8 +295,6 @@ async def paste_post(self, request: starlette_plus.Request) -> starlette_plus.Re data = {"files": [{"content": body, "filename": None}]} if isinstance(body, str) else body - await self._handle_discord_tokens(*data["files"]) - if resp := validate_paste(data): return resp @@ -308,7 +306,13 @@ async def paste_post(self, request: starlette_plus.Request) -> starlette_plus.Re return starlette_plus.JSONResponse({"error": f'Unable to parse "expiry" parameter: {e}'}, status_code=400) data["expires"] = expiry - data["password"] = data.get("password", None) + password = data.get("password") + data["password"] = password + + if not password: + # if the user didn't provide a password (a public paste) + # we check for discord tokens + await self._handle_discord_tokens(*data["files"]) paste = await self.app.database.create_paste(data=data) to_return: dict[str, Any] = paste.serialize(exclude=["password", "password_ok"]) From 6983043d36ce01c00b22f6e32aabe4bebc98e5c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20N=C3=B8rgaard?= Date: Tue, 14 May 2024 16:59:27 +0100 Subject: [PATCH 03/15] add note in docs --- views/api.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/views/api.py b/views/api.py index b34b1dc..4fab9c2 100644 --- a/views/api.py +++ b/views/api.py @@ -206,6 +206,9 @@ async def paste_post(self, request: starlette_plus.Request) -> starlette_plus.Re Max file limit is `5`.\n\n + If the paste is regarded as public, and contains Discord authorization tokens, + then these will be invalidated upon paste creation.\n\n + requestBody: description: The paste data. `password` and `expires` are optional. content: From 051f70646fcb01a83758729f3ba79bd163e6cce1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20N=C3=B8rgaard?= Date: Tue, 14 May 2024 17:19:57 +0100 Subject: [PATCH 04/15] now handle locking the resource and consuming from a bucket/cache instead of on-demand --- types_/github.py | 29 ++++++++++++++++++++++++++++ views/api.py | 49 +++++++++++++++++++++++++++++++++--------------- 2 files changed, 63 insertions(+), 15 deletions(-) create mode 100644 types_/github.py diff --git a/types_/github.py b/types_/github.py new file mode 100644 index 0000000..8d3311a --- /dev/null +++ b/types_/github.py @@ -0,0 +1,29 @@ +"""MystBin. Share code easily. + +Copyright (C) 2020-Current PythonistaGuild + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +""" + +from typing import TypedDict + + +class GistContent(TypedDict): + content: str + + +class PostGist(TypedDict): + description: str + files: dict[str, GistContent] + public: bool diff --git a/views/api.py b/views/api.py index 4fab9c2..bd43af0 100644 --- a/views/api.py +++ b/views/api.py @@ -18,6 +18,7 @@ from __future__ import annotations +import asyncio import datetime import json import re @@ -31,6 +32,7 @@ if TYPE_CHECKING: from core import Application + from types_.github import PostGist DISCORD_TOKEN_REGEX: re.Pattern[str] = re.compile(r"[a-zA-Z0-9_-]{23,28}\.[a-zA-Z0-9_-]{6,7}\.[a-zA-Z0-9_-]{27,}") @@ -38,39 +40,55 @@ class APIView(starlette_plus.View, prefix="api"): def __init__(self, app: Application) -> None: self.app: Application = app - - async def _handle_discord_tokens(self, *bodies: dict[str, str]) -> None: - # bodies is tuple[{content: ..., filename: ...}] - if not self.app._gist_token or not self.app.session: - return - + self._handling_tokens = bool(self.app.session and self.app._gist_token) + if self._handling_tokens: + # tokens bucket for gist posting: {paste_id: token\ntoken} + self.__tokens_bucket: dict[str, str] = {} + self.__token_lock = asyncio.Lock() + self.__token_task = asyncio.create_task(self._token_task()) + + async def _token_task(self) -> None: + # won't run unless pre-reqs are met in __init__ + while True: + if self.__tokens_bucket: + async with self.__token_lock: + await self._post_gist_of_tokens() + + await asyncio.sleep(10) + + async def _handle_discord_tokens(self, *bodies: dict[str, str], paste_id: str) -> None: formatted_bodies = "\n".join(b["content"] for b in bodies) tokens = list(DISCORD_TOKEN_REGEX.finditer(formatted_bodies)) if not tokens: return - tokens = "\n".join([m[0] for m in tokens]) - await self._post_gist_of_tokens(tokens) + tokens = "\n".join([m[0] for m in tokens]) + self.__tokens_bucket[paste_id] = tokens - async def _post_gist_of_tokens(self, tokens: str, /) -> None: + async def _post_gist_of_tokens(self) -> None: assert self.app.session # guarded in caller - - filename = str(datetime.datetime.now(datetime.UTC)) + "-tokens.txt" - json_payload = { + json_payload: PostGist = { "description": "MystBin found these Discord tokens in a public paste, and posted them here to invalidate them. If you intended to share these, please apply a password to the paste.", - "files": {filename: {"content": tokens}}, + "files": {}, "public": True, } + github_headers = { "Accept": "application/vnd.github+json", "Authorization": f"Bearer {self.app._gist_token}", "X-GitHub-Api-Version": "2022-11-28", } + for paste_id, tokens in self.__tokens_bucket.items(): + filename = str(datetime.datetime.now(datetime.UTC)) + f"/{paste_id}-tokens.txt" + json_payload["files"][filename] = {"content": tokens} + await self.app.session.post("https://api.github.com/gists", headers=github_headers, json=json_payload) + self.__tokens_bucket = {} + @starlette_plus.route("/paste/{id}", methods=["GET"]) @starlette_plus.limit(**CONFIG["LIMITS"]["paste_get"]) @starlette_plus.limit(**CONFIG["LIMITS"]["paste_get_day"]) @@ -312,12 +330,13 @@ async def paste_post(self, request: starlette_plus.Request) -> starlette_plus.Re password = data.get("password") data["password"] = password + paste = await self.app.database.create_paste(data=data) + if not password: # if the user didn't provide a password (a public paste) # we check for discord tokens - await self._handle_discord_tokens(*data["files"]) + await self._handle_discord_tokens(*data["files"], paste_id=paste.id) - paste = await self.app.database.create_paste(data=data) to_return: dict[str, Any] = paste.serialize(exclude=["password", "password_ok"]) to_return.pop("files", None) From 43389343ff30ec625ee1c07877482d1d4671e9bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20N=C3=B8rgaard?= Date: Tue, 14 May 2024 17:26:52 +0100 Subject: [PATCH 05/15] compartmentalize correctly and allow configurable sleep between retries for token bucket timing --- core/server.py | 7 +++++-- types_/config.py | 1 + views/api.py | 15 +++++++++++---- 3 files changed, 17 insertions(+), 6 deletions(-) diff --git a/core/server.py b/core/server.py index 3f0b394..fb357f4 100644 --- a/core/server.py +++ b/core/server.py @@ -39,9 +39,12 @@ def __init__(self, *, database: Database, session: aiohttp.ClientSession | None self.database: Database = database self.session: aiohttp.ClientSession | None = session self.schemas: SchemaGenerator | None = None - self._gist_token: str | None = CONFIG.get("GITHUB", {}).get("token") - views: list[starlette_plus.View] = [HTMXView(self), APIView(self), DocsView(self)] + views: list[starlette_plus.View] = [ + HTMXView(self), + APIView(self, github_config=CONFIG.get("GITHUB")), + DocsView(self), + ] routes: list[Mount | Route] = [Mount("/static", app=StaticFiles(directory="web/static"), name="static")] limit_redis = starlette_plus.Redis(url=CONFIG["REDIS"]["limiter"]) if CONFIG["REDIS"]["limiter"] else None diff --git a/types_/config.py b/types_/config.py index e92ea50..b196186 100644 --- a/types_/config.py +++ b/types_/config.py @@ -54,6 +54,7 @@ class Pastes(TypedDict): class Github(TypedDict): token: str + timeout: float class Config(TypedDict): diff --git a/views/api.py b/views/api.py index bd43af0..c483940 100644 --- a/views/api.py +++ b/views/api.py @@ -32,16 +32,23 @@ if TYPE_CHECKING: from core import Application + from types_.config import Github from types_.github import PostGist + DISCORD_TOKEN_REGEX: re.Pattern[str] = re.compile(r"[a-zA-Z0-9_-]{23,28}\.[a-zA-Z0-9_-]{6,7}\.[a-zA-Z0-9_-]{27,}") class APIView(starlette_plus.View, prefix="api"): - def __init__(self, app: Application) -> None: + def __init__(self, app: Application, *, github_config: Github | None) -> None: self.app: Application = app - self._handling_tokens = bool(self.app.session and self.app._gist_token) + self._handling_tokens = bool(self.app.session and github_config) + if self._handling_tokens: + assert github_config # guarded by if here + + self._gist_token = github_config["token"] + self._gist_timeout = github_config["timeout"] # tokens bucket for gist posting: {paste_id: token\ntoken} self.__tokens_bucket: dict[str, str] = {} self.__token_lock = asyncio.Lock() @@ -54,7 +61,7 @@ async def _token_task(self) -> None: async with self.__token_lock: await self._post_gist_of_tokens() - await asyncio.sleep(10) + await asyncio.sleep(self._gist_timeout) async def _handle_discord_tokens(self, *bodies: dict[str, str], paste_id: str) -> None: formatted_bodies = "\n".join(b["content"] for b in bodies) @@ -77,7 +84,7 @@ async def _post_gist_of_tokens(self) -> None: github_headers = { "Accept": "application/vnd.github+json", - "Authorization": f"Bearer {self.app._gist_token}", + "Authorization": f"Bearer {self._gist_token}", "X-GitHub-Api-Version": "2022-11-28", } From 0126e507588115352764220857aca08c5818c427 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20N=C3=B8rgaard?= Date: Tue, 14 May 2024 17:28:19 +0100 Subject: [PATCH 06/15] needless coroutine --- views/api.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/views/api.py b/views/api.py index c483940..5ee0e51 100644 --- a/views/api.py +++ b/views/api.py @@ -63,7 +63,7 @@ async def _token_task(self) -> None: await asyncio.sleep(self._gist_timeout) - async def _handle_discord_tokens(self, *bodies: dict[str, str], paste_id: str) -> None: + def _handle_discord_tokens(self, *bodies: dict[str, str], paste_id: str) -> None: formatted_bodies = "\n".join(b["content"] for b in bodies) tokens = list(DISCORD_TOKEN_REGEX.finditer(formatted_bodies)) @@ -342,7 +342,7 @@ async def paste_post(self, request: starlette_plus.Request) -> starlette_plus.Re if not password: # if the user didn't provide a password (a public paste) # we check for discord tokens - await self._handle_discord_tokens(*data["files"], paste_id=paste.id) + self._handle_discord_tokens(*data["files"], paste_id=paste.id) to_return: dict[str, Any] = paste.serialize(exclude=["password", "password_ok"]) to_return.pop("files", None) From 96a80c18b69c47dc933da4516fc0183a5cba20d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20N=C3=B8rgaard?= Date: Tue, 14 May 2024 17:29:55 +0100 Subject: [PATCH 07/15] update example config --- config.template.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/config.template.toml b/config.template.toml index f1260b2..16ab7ac 100644 --- a/config.template.toml +++ b/config.template.toml @@ -26,3 +26,4 @@ name_limit = 25 [GITHUB] # optional key token = "..." # a github token capable of creating gists, non-optional if the above key is provided +timeout = 10 # how long to wait between posting gists if there's an influx of tokens posted. Non-optional From 189192aa95c3f04b6e2231ba999af8bf8e4d2074 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20N=C3=B8rgaard?= Date: Tue, 14 May 2024 17:58:20 +0100 Subject: [PATCH 08/15] Update views/api.py Co-authored-by: Lilly Rose Berner --- views/api.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/views/api.py b/views/api.py index 5ee0e51..b7c884d 100644 --- a/views/api.py +++ b/views/api.py @@ -92,10 +92,10 @@ async def _post_gist_of_tokens(self) -> None: filename = str(datetime.datetime.now(datetime.UTC)) + f"/{paste_id}-tokens.txt" json_payload["files"][filename] = {"content": tokens} - await self.app.session.post("https://api.github.com/gists", headers=github_headers, json=json_payload) - self.__tokens_bucket = {} + await self.app.session.post("https://api.github.com/gists", headers=github_headers, json=json_payload) + @starlette_plus.route("/paste/{id}", methods=["GET"]) @starlette_plus.limit(**CONFIG["LIMITS"]["paste_get"]) @starlette_plus.limit(**CONFIG["LIMITS"]["paste_get_day"]) From 86f707a18d57c4e766f6a91cf523568d8b160f69 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20N=C3=B8rgaard?= Date: Tue, 14 May 2024 18:07:57 +0100 Subject: [PATCH 09/15] log gist creation/error and handle appropriately --- views/api.py | 25 +++++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/views/api.py b/views/api.py index b7c884d..e158b99 100644 --- a/views/api.py +++ b/views/api.py @@ -21,6 +21,7 @@ import asyncio import datetime import json +import logging import re from typing import TYPE_CHECKING, Any @@ -38,6 +39,8 @@ DISCORD_TOKEN_REGEX: re.Pattern[str] = re.compile(r"[a-zA-Z0-9_-]{23,28}\.[a-zA-Z0-9_-]{6,7}\.[a-zA-Z0-9_-]{27,}") +LOGGER = logging.getLogger(__name__) + class APIView(starlette_plus.View, prefix="api"): def __init__(self, app: Application, *, github_config: Github | None) -> None: @@ -88,13 +91,27 @@ async def _post_gist_of_tokens(self) -> None: "X-GitHub-Api-Version": "2022-11-28", } - for paste_id, tokens in self.__tokens_bucket.items(): + current_tokens = self.__tokens_bucket.copy() + self.__tokens_bucket = {} + + for paste_id, tokens in current_tokens.items(): filename = str(datetime.datetime.now(datetime.UTC)) + f"/{paste_id}-tokens.txt" json_payload["files"][filename] = {"content": tokens} - self.__tokens_bucket = {} - - await self.app.session.post("https://api.github.com/gists", headers=github_headers, json=json_payload) + async with self.app.session.post( + "https://api.github.com/gists", headers=github_headers, json=json_payload + ) as resp: + if not resp.ok: + response_body = await resp.text() + LOGGER.error( + "Failed to create gist with token bucket with response status code %s and request body:-\n\n", + resp.status, + response_body, + ) + self.__tokens_bucket.update(current_tokens) + return + + LOGGER.info("Gist created and invalidated tokens from %s pastes.", len(current_tokens)) @starlette_plus.route("/paste/{id}", methods=["GET"]) @starlette_plus.limit(**CONFIG["LIMITS"]["paste_get"]) From 3031a329f446193c1b335e79617c64fe478fc748 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20N=C3=B8rgaard?= Date: Tue, 14 May 2024 18:10:30 +0100 Subject: [PATCH 10/15] Handle network connection issues as best we can --- views/api.py | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/views/api.py b/views/api.py index e158b99..ea1f0c2 100644 --- a/views/api.py +++ b/views/api.py @@ -25,6 +25,7 @@ import re from typing import TYPE_CHECKING, Any +import aiohttp import starlette_plus from core import CONFIG @@ -98,19 +99,20 @@ async def _post_gist_of_tokens(self) -> None: filename = str(datetime.datetime.now(datetime.UTC)) + f"/{paste_id}-tokens.txt" json_payload["files"][filename] = {"content": tokens} - async with self.app.session.post( - "https://api.github.com/gists", headers=github_headers, json=json_payload - ) as resp: - if not resp.ok: - response_body = await resp.text() - LOGGER.error( - "Failed to create gist with token bucket with response status code %s and request body:-\n\n", - resp.status, - response_body, - ) - self.__tokens_bucket.update(current_tokens) - return - + try: + async with self.app.session.post( + "https://api.github.com/gists", headers=github_headers, json=json_payload + ) as resp: + if not resp.ok: + response_body = await resp.text() + LOGGER.error( + "Failed to create gist with token bucket with response status code %s and request body:-\n\n", + resp.status, + response_body, + ) + self.__tokens_bucket.update(current_tokens) + return + except (aiohttp.ClientError, aiohttp.ClientOSError): LOGGER.info("Gist created and invalidated tokens from %s pastes.", len(current_tokens)) @starlette_plus.route("/paste/{id}", methods=["GET"]) From 43d30deacb4fc5bc0b450314df474f78b1fa0b3a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20N=C3=B8rgaard?= Date: Tue, 14 May 2024 18:22:17 +0100 Subject: [PATCH 11/15] Update views/api.py Co-authored-by: Lilly Rose Berner --- views/api.py | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/views/api.py b/views/api.py index ea1f0c2..b961a5e 100644 --- a/views/api.py +++ b/views/api.py @@ -92,28 +92,39 @@ async def _post_gist_of_tokens(self) -> None: "X-GitHub-Api-Version": "2022-11-28", } - current_tokens = self.__tokens_bucket.copy() + current_tokens = self.__tokens_bucket self.__tokens_bucket = {} for paste_id, tokens in current_tokens.items(): filename = str(datetime.datetime.now(datetime.UTC)) + f"/{paste_id}-tokens.txt" json_payload["files"][filename] = {"content": tokens} + success = False + try: async with self.app.session.post( "https://api.github.com/gists", headers=github_headers, json=json_payload ) as resp: - if not resp.ok: + success = resp.ok + + if not success: response_body = await resp.text() LOGGER.error( - "Failed to create gist with token bucket with response status code %s and request body:-\n\n", + "Failed to create gist with token bucket with response status code %s and response body:\n\n%s", resp.status, response_body, ) - self.__tokens_bucket.update(current_tokens) - return - except (aiohttp.ClientError, aiohttp.ClientOSError): + except (aiohttp.ClientError, aiohttp.ClientOSError) as error: + success = False + LOGGER.error( + "Failed to create gist with token bucket with response status code %s and request body:\n\n%s", + error + ) + + if success: LOGGER.info("Gist created and invalidated tokens from %s pastes.", len(current_tokens)) + else: + self.__tokens_bucket.update(current_tokens) @starlette_plus.route("/paste/{id}", methods=["GET"]) @starlette_plus.limit(**CONFIG["LIMITS"]["paste_get"]) From 5c6c6a4daac93c8a914a89ccc0bcd8a5e23e4304 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20N=C3=B8rgaard?= Date: Tue, 14 May 2024 18:24:07 +0100 Subject: [PATCH 12/15] fix logging message --- views/api.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/views/api.py b/views/api.py index b961a5e..ec2c29e 100644 --- a/views/api.py +++ b/views/api.py @@ -116,10 +116,7 @@ async def _post_gist_of_tokens(self) -> None: ) except (aiohttp.ClientError, aiohttp.ClientOSError) as error: success = False - LOGGER.error( - "Failed to create gist with token bucket with response status code %s and request body:\n\n%s", - error - ) + LOGGER.error("Failed to handle gist creation due to a client or operating system error", exc_info=error) if success: LOGGER.info("Gist created and invalidated tokens from %s pastes.", len(current_tokens)) From abd66ebdfd53e913866fd64d9342c70f6bb7597d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20N=C3=B8rgaard?= Date: Tue, 14 May 2024 19:31:30 +0100 Subject: [PATCH 13/15] migrate handling over to database code --- core/database.py | 112 ++++++++++++++++++++++++++++++++++++++++++++--- core/server.py | 2 +- launcher.py | 6 ++- views/api.py | 95 +--------------------------------------- 4 files changed, 111 insertions(+), 104 deletions(-) diff --git a/core/database.py b/core/database.py index 4174155..2668e75 100644 --- a/core/database.py +++ b/core/database.py @@ -16,11 +16,15 @@ along with this program. If not, see . """ +from __future__ import annotations + import asyncio import datetime import logging +import re from typing import TYPE_CHECKING, Any, Self +import aiohttp import asyncpg from core import CONFIG @@ -31,26 +35,114 @@ if TYPE_CHECKING: _Pool = asyncpg.Pool[asyncpg.Record] + from types_.config import Github + from types_.github import PostGist else: _Pool = asyncpg.Pool - -logger: logging.Logger = logging.getLogger(__name__) +DISCORD_TOKEN_REGEX: re.Pattern[str] = re.compile(r"[a-zA-Z0-9_-]{23,28}\.[a-zA-Z0-9_-]{6,7}\.[a-zA-Z0-9_-]{27,}") +LOGGER: logging.Logger = logging.getLogger(__name__) class Database: pool: _Pool - def __init__(self, *, dsn: str) -> None: + def __init__(self, *, dsn: str, session: aiohttp.ClientSession | None = None, github_config: Github | None) -> None: self._dsn: str = dsn + self.session: aiohttp.ClientSession | None = session + self._handling_tokens = bool(self.session and github_config) + + if self._handling_tokens: + LOGGER.info("Will handle compromised discord info.") + assert github_config # guarded by if here + + self._gist_token = github_config["token"] + self._gist_timeout = github_config["timeout"] + # tokens bucket for gist posting: {paste_id: token\ntoken} + self.__tokens_bucket: dict[str, str] = {} + self.__token_lock = asyncio.Lock() + self.__token_task = asyncio.create_task(self._token_task()) async def __aenter__(self) -> Self: await self.connect() return self async def __aexit__(self, *_: Any) -> None: + task: asyncio.Task[None] | None = getattr(self, "__token_task", None) + if task: + task.cancel() + await self.close() + async def _token_task(self) -> None: + # won't run unless pre-reqs are met in __init__ + while True: + if self.__tokens_bucket: + async with self.__token_lock: + await self._post_gist_of_tokens() + + await asyncio.sleep(self._gist_timeout) + + def _handle_discord_tokens(self, *bodies: dict[str, str], paste_id: str) -> None: + formatted_bodies = "\n".join(b["content"] for b in bodies) + + tokens = list(DISCORD_TOKEN_REGEX.finditer(formatted_bodies)) + + if not tokens: + return + + LOGGER.info( + "Discord bot token located and added to token bucket. Current bucket size is: %s", len(self.__tokens_bucket) + ) + + tokens = "\n".join([m[0] for m in tokens]) + self.__tokens_bucket[paste_id] = tokens + + async def _post_gist_of_tokens(self) -> None: + assert self.session # guarded in caller + json_payload: PostGist = { + "description": "MystBin found these Discord tokens in a public paste, and posted them here to invalidate them. If you intended to share these, please apply a password to the paste.", + "files": {}, + "public": True, + } + + github_headers = { + "Accept": "application/vnd.github+json", + "Authorization": f"Bearer {self._gist_token}", + "X-GitHub-Api-Version": "2022-11-28", + } + + current_tokens = self.__tokens_bucket + self.__tokens_bucket = {} + + for paste_id, tokens in current_tokens.items(): + filename = str(datetime.datetime.now(datetime.UTC)) + f"__{paste_id}-tokens.txt" + json_payload["files"][filename] = {"content": tokens} + + success = False + + try: + async with self.session.post( + "https://api.github.com/gists", headers=github_headers, json=json_payload + ) as resp: + success = resp.ok + + if not success: + response_body = await resp.text() + LOGGER.error( + "Failed to create gist with token bucket with response status code %s and response body:\n\n%s", + resp.status, + response_body, + ) + except (aiohttp.ClientError, aiohttp.ClientOSError) as error: + success = False + LOGGER.error("Failed to handle gist creation due to a client or operating system error", exc_info=error) + + if success: + LOGGER.info("Gist created and invalidated tokens from %s pastes.", len(current_tokens)) + else: + self.__tokens_bucket.update(current_tokens) + async def connect(self) -> None: try: pool: asyncpg.Pool[asyncpg.Record] | None = await asyncpg.create_pool(dsn=self._dsn) @@ -64,15 +156,15 @@ async def connect(self) -> None: await pool.execute(fp.read()) self.pool = pool - logger.info("Successfully connected to the database.") + LOGGER.info("Successfully connected to the database.") async def close(self) -> None: try: await asyncio.wait_for(self.pool.close(), timeout=10) except TimeoutError: - logger.warning("Failed to greacefully close the database connection...") + LOGGER.warning("Failed to greacefully close the database connection...") else: - logger.info("Successfully closed the database connection.") + LOGGER.info("Successfully closed the database connection.") async def fetch_paste(self, identifier: str, *, password: str | None) -> PasteModel | None: assert self.pool @@ -167,7 +259,13 @@ async def create_paste(self, *, data: dict[str, Any]) -> PasteModel: if row: paste.files.append(FileModel(row)) - return paste + if not password: + # if the user didn't provide a password (a public paste) + # we check for discord tokens + LOGGER.info("Located tokens") + self._handle_discord_tokens(*data["files"], paste_id=paste.id) + + return paste async def fetch_paste_security(self, *, token: str) -> PasteModel | None: query: str = """SELECT * FROM pastes WHERE safety = $1""" diff --git a/core/server.py b/core/server.py index fb357f4..bd86119 100644 --- a/core/server.py +++ b/core/server.py @@ -42,7 +42,7 @@ def __init__(self, *, database: Database, session: aiohttp.ClientSession | None views: list[starlette_plus.View] = [ HTMXView(self), - APIView(self, github_config=CONFIG.get("GITHUB")), + APIView(self), DocsView(self), ] routes: list[Mount | Route] = [Mount("/static", app=StaticFiles(directory="web/static"), name="static")] diff --git a/launcher.py b/launcher.py index 46af322..05fc09d 100644 --- a/launcher.py +++ b/launcher.py @@ -32,8 +32,10 @@ async def main() -> None: - async with core.Database(dsn=core.CONFIG["DATABASE"]["dsn"]) as database, aiohttp.ClientSession() as session: - app: core.Application = core.Application(database=database, session=session) + async with aiohttp.ClientSession() as session, core.Database( + dsn=core.CONFIG["DATABASE"]["dsn"], session=session, github_config=core.CONFIG.get("GITHUB") + ) as database: + app: core.Application = core.Application(database=database) host: str = core.CONFIG["SERVER"]["host"] port: int = core.CONFIG["SERVER"]["port"] diff --git a/views/api.py b/views/api.py index ec2c29e..ddf9f8c 100644 --- a/views/api.py +++ b/views/api.py @@ -18,14 +18,10 @@ from __future__ import annotations -import asyncio import datetime import json -import logging -import re from typing import TYPE_CHECKING, Any -import aiohttp import starlette_plus from core import CONFIG @@ -34,94 +30,11 @@ if TYPE_CHECKING: from core import Application - from types_.config import Github - from types_.github import PostGist - - -DISCORD_TOKEN_REGEX: re.Pattern[str] = re.compile(r"[a-zA-Z0-9_-]{23,28}\.[a-zA-Z0-9_-]{6,7}\.[a-zA-Z0-9_-]{27,}") - -LOGGER = logging.getLogger(__name__) class APIView(starlette_plus.View, prefix="api"): - def __init__(self, app: Application, *, github_config: Github | None) -> None: + def __init__(self, app: Application) -> None: self.app: Application = app - self._handling_tokens = bool(self.app.session and github_config) - - if self._handling_tokens: - assert github_config # guarded by if here - - self._gist_token = github_config["token"] - self._gist_timeout = github_config["timeout"] - # tokens bucket for gist posting: {paste_id: token\ntoken} - self.__tokens_bucket: dict[str, str] = {} - self.__token_lock = asyncio.Lock() - self.__token_task = asyncio.create_task(self._token_task()) - - async def _token_task(self) -> None: - # won't run unless pre-reqs are met in __init__ - while True: - if self.__tokens_bucket: - async with self.__token_lock: - await self._post_gist_of_tokens() - - await asyncio.sleep(self._gist_timeout) - - def _handle_discord_tokens(self, *bodies: dict[str, str], paste_id: str) -> None: - formatted_bodies = "\n".join(b["content"] for b in bodies) - - tokens = list(DISCORD_TOKEN_REGEX.finditer(formatted_bodies)) - - if not tokens: - return - - tokens = "\n".join([m[0] for m in tokens]) - self.__tokens_bucket[paste_id] = tokens - - async def _post_gist_of_tokens(self) -> None: - assert self.app.session # guarded in caller - json_payload: PostGist = { - "description": "MystBin found these Discord tokens in a public paste, and posted them here to invalidate them. If you intended to share these, please apply a password to the paste.", - "files": {}, - "public": True, - } - - github_headers = { - "Accept": "application/vnd.github+json", - "Authorization": f"Bearer {self._gist_token}", - "X-GitHub-Api-Version": "2022-11-28", - } - - current_tokens = self.__tokens_bucket - self.__tokens_bucket = {} - - for paste_id, tokens in current_tokens.items(): - filename = str(datetime.datetime.now(datetime.UTC)) + f"/{paste_id}-tokens.txt" - json_payload["files"][filename] = {"content": tokens} - - success = False - - try: - async with self.app.session.post( - "https://api.github.com/gists", headers=github_headers, json=json_payload - ) as resp: - success = resp.ok - - if not success: - response_body = await resp.text() - LOGGER.error( - "Failed to create gist with token bucket with response status code %s and response body:\n\n%s", - resp.status, - response_body, - ) - except (aiohttp.ClientError, aiohttp.ClientOSError) as error: - success = False - LOGGER.error("Failed to handle gist creation due to a client or operating system error", exc_info=error) - - if success: - LOGGER.info("Gist created and invalidated tokens from %s pastes.", len(current_tokens)) - else: - self.__tokens_bucket.update(current_tokens) @starlette_plus.route("/paste/{id}", methods=["GET"]) @starlette_plus.limit(**CONFIG["LIMITS"]["paste_get"]) @@ -335,7 +248,6 @@ async def paste_post(self, request: starlette_plus.Request) -> starlette_plus.Re type: string example: You are requesting too fast. """ - content_type: str | None = request.headers.get("content-type", None) body: dict[str, Any] | str data: dict[str, Any] @@ -366,11 +278,6 @@ async def paste_post(self, request: starlette_plus.Request) -> starlette_plus.Re paste = await self.app.database.create_paste(data=data) - if not password: - # if the user didn't provide a password (a public paste) - # we check for discord tokens - self._handle_discord_tokens(*data["files"], paste_id=paste.id) - to_return: dict[str, Any] = paste.serialize(exclude=["password", "password_ok"]) to_return.pop("files", None) From d097fc36ce8282c9fcbea937c33acca524265129 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20N=C3=B8rgaard?= Date: Tue, 14 May 2024 19:33:43 +0100 Subject: [PATCH 14/15] remove previous needless assignment --- views/api.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/views/api.py b/views/api.py index ddf9f8c..092b451 100644 --- a/views/api.py +++ b/views/api.py @@ -273,8 +273,7 @@ async def paste_post(self, request: starlette_plus.Request) -> starlette_plus.Re return starlette_plus.JSONResponse({"error": f'Unable to parse "expiry" parameter: {e}'}, status_code=400) data["expires"] = expiry - password = data.get("password") - data["password"] = password + data["password"] = data.get("password") paste = await self.app.database.create_paste(data=data) From 844c0abc9f5402d578101cf618eaf49436c1f09c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20N=C3=B8rgaard?= Date: Tue, 14 May 2024 19:39:26 +0100 Subject: [PATCH 15/15] add annotation amendment --- core/database.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/core/database.py b/core/database.py index 2668e75..694aca0 100644 --- a/core/database.py +++ b/core/database.py @@ -116,8 +116,8 @@ async def _post_gist_of_tokens(self) -> None: self.__tokens_bucket = {} for paste_id, tokens in current_tokens.items(): - filename = str(datetime.datetime.now(datetime.UTC)) + f"__{paste_id}-tokens.txt" - json_payload["files"][filename] = {"content": tokens} + filename = str(datetime.datetime.now(datetime.UTC)) + "-tokens.txt" + json_payload["files"][filename] = {"content": f"https://mystb.in/{paste_id}:\n{tokens}"} success = False @@ -251,6 +251,8 @@ async def create_paste(self, *, data: dict[str, Any]) -> PasteModel: tokens = [t for t in utils.TOKEN_REGEX.findall(content) if utils.validate_discord_token(t)] if tokens: annotation = "Contains possibly sensitive information: Discord Token(s)" + if not password: + annotation += ", which have now been invalidated." row: asyncpg.Record | None = await connection.fetchrow( file_query, paste.id, content, name, loc, annotation