From 0368e0cf3da9cb7d7cad457e06570823e156e08a Mon Sep 17 00:00:00 2001 From: "Paul J. Dorn" Date: Sun, 17 Dec 2023 16:35:43 +0100 Subject: [PATCH] WIP: typing --- .github/workflows/buildpackage.yml | 32 +++- .github/workflows/lint.yml | 4 +- .github/workflows/tox.yml | 33 +++- gunicorn/__init__.pyi | 2 +- gunicorn/app/base.py | 10 +- gunicorn/app/base.pyi | 31 ++-- gunicorn/app/pasterapp.pyi | 9 +- gunicorn/app/wsgiapp.pyi | 13 +- gunicorn/arbiter.py | 17 +- gunicorn/config.py | 41 +++++ gunicorn/debug.pyi | 13 +- gunicorn/errors.pyi | 4 +- gunicorn/glogging.pyi | 17 +- gunicorn/http/body.pyi | 63 ++++---- gunicorn/http/errors.pyi | 49 +++--- gunicorn/http/message.pyi | 79 +++++---- gunicorn/http/unreader.pyi | 10 +- gunicorn/http/wsgi.pyi | 71 +++++---- gunicorn/packaging_support.py | 145 +++++++++++++++++ gunicorn/pidfile.pyi | 13 +- gunicorn/reloader.pyi | 23 +-- gunicorn/sock.pyi | 11 +- gunicorn/util.py | 30 +++- gunicorn/util.pyi | 55 ++++--- gunicorn/workers/base.py | 18 ++- gunicorn/workers/base.pyi | 57 ++++--- gunicorn/workers/base_async.pyi | 31 ++-- gunicorn/workers/geventlet.py | 14 +- gunicorn/workers/geventlet.pyi | 24 ++- gunicorn/workers/ggevent.py | 15 +- gunicorn/workers/ggevent.pyi | 38 +++-- gunicorn/workers/gthread.pyi | 56 ++++--- gunicorn/workers/gtornado.py | 7 +- gunicorn/workers/gtornado.pyi | 10 +- gunicorn/workers/sync.pyi | 24 +-- pyproject.toml | 73 +++++++-- tests/requests/invalid/version_03.http | 2 + tests/requests/invalid/version_03.py | 2 + tests/test_e2e.py | 211 +++++++++++++++++++++++++ tests/test_http.py | 12 ++ tests/treq.py | 2 +- tox.ini | 61 ++++--- 42 files changed, 1035 insertions(+), 397 deletions(-) create mode 100644 gunicorn/packaging_support.py create mode 100644 tests/requests/invalid/version_03.http create mode 100644 tests/requests/invalid/version_03.py create mode 100644 tests/test_e2e.py diff --git a/.github/workflows/buildpackage.yml b/.github/workflows/buildpackage.yml index ae8305ce6..12e68ebec 100644 --- a/.github/workflows/buildpackage.yml +++ b/.github/workflows/buildpackage.yml @@ -51,7 +51,7 @@ jobs: ( cd workaround/ && tar --extract --file gunicorn_21.2.0.orig.tar.gz gunicorn-21.2.0 ) test -s workaround/gunicorn-21.2.0/pyproject.toml rsync -vrlt source/.github/packaging/debian/ workaround/gunicorn-21.2.0/debian - echo 'extend-diff-ignore = "^setup\.cfg$"' > workaround/gunicorn-21.2.0/debian/source/options + echo 'extend-diff-ignore = "^setup\.cfg$"' >> workaround/gunicorn-21.2.0/debian/source/options mv --verbose workaround/gunicorn-21.2.0/debian/setup.cfg workaround/gunicorn-21.2.0/ chmod --changes +x workaround/gunicorn-21.2.0/debian/control ls -l workaround/gunicorn-21.2.0/ @@ -88,7 +88,8 @@ jobs: test -s workaround/gunicorn-21.2.0/debian/control test -d workaround/gunicorn-21.2.0/tests ( cd workaround/gunicorn-21.2.0/ && dpkg-buildpackage --unsigned-source --unsigned-changes ) - cp --verbose --archive workaround/*.deb upload/workaround/ + # note that Ubuntu 22.04 does not allow zstd in dpkg tools + cp --verbose --archive workaround/*.{deb,tar.gz,tar.xz,tar.zstd,buildinfo,changes,dsc} upload/workaround/ - name: build deb (clean) continue-on-error: true run: | @@ -96,8 +97,27 @@ jobs: test -s debian/gunicorn-21.2.0/debian/control test -d debian/gunicorn-21.2.0/tests ( cd debian/gunicorn-21.2.0/ && dpkg-buildpackage --unsigned-source --unsigned-changes ) - cp --verbose --archive debian/*.deb upload/ - - uses: actions/upload-artifact@v3 + # note that Ubuntu 22.04 does not allow zstd in dpkg tools + cp --verbose --archive debian/*.{deb,tar.gz,tar.xz,tar.zstd,buildinfo,changes,dsc} upload/ + - uses: actions/upload-artifact@v4 with: - path: upload - name: dist + path: | + upload/workaround/*.deb + upload/workaround/*.tar.gz + upload/workaround/*.tar.xz + upload/workaround/*.tar.zstd + upload/workaround/*.buildinfo + upload/workaround/*.changes + upload/workaround/*.dsc + upload/*.deb + upload/*.tar.gz + upload/*.tar.xz + upload/*.tar.zstd + upload/*.buildinfo + upload/*.changes + upload/*.dsc + name: deb + retention-days: 5 + # deb and source tarball are already compressed + compression-level: 0 + if-no-files-found: error diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 05b61d937..aa67e033e 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -35,10 +35,12 @@ jobs: - uses: actions/checkout@v4 - name: Using Python ${{ matrix.python-version }} uses: actions/setup-python@v5 + id: setup-python with: python-version: ${{ matrix.python-version }} cache: pip - - name: Install Dependencies + cache-dependency-path: pyproject.toml + - name: Install Dependencies (cache hit: ${{ steps.setup-python.outputs.cache-hit }}) run: | python -m pip install --upgrade pip python -m pip install tox diff --git a/.github/workflows/tox.yml b/.github/workflows/tox.yml index 3de88fb53..63fafb747 100644 --- a/.github/workflows/tox.yml +++ b/.github/workflows/tox.yml @@ -25,7 +25,14 @@ jobs: os: [ubuntu-latest, macos-latest] python-version: [ "3.7", "3.8", "3.9", "3.10", "3.11", "3.12", "pypy-3.10" ] unsupported: [false] + mindep: [false] include: + - os: ubuntu-20.04 + python-version: "3.8" + mindep: true + - os: ubuntu-latest + python-version: "3.13" + unsupported: true - os: windows-latest python-version: "3.12" unsupported: true @@ -41,18 +48,26 @@ jobs: - uses: actions/checkout@v4 - name: Using Python ${{ matrix.python-version }} uses: actions/setup-python@v5 + id: "setup-python" with: python-version: ${{ matrix.python-version }} + allow-prereleases: ${{ matrix.unsupported }} cache: pip - cache-dependency-path: requirements_test.txt - check-latest: true - - name: Install test Dependencies + cache-dependency-path: pyproject.toml + - name: Install test Dependencies (cache hit: ${{ steps.setup-python.outputs.cache-hit }}) run: | python -m pip install --upgrade pip python -m pip install tox - run: tox -e run-module + timeout-minutes: 2 - run: tox -e run-entrypoint + timeout-minutes: 2 - run: tox -e py + timeout-minutes: 5 + continue-on-error: ${{ matrix.unsupported }} + - if: ${{ mindep }} + timeout-minutes: 8 + run: tox -e py-mindep continue-on-error: ${{ matrix.unsupported }} - name: Install dist Dependencies run: | @@ -69,7 +84,13 @@ jobs: # windows doe not do --archive --verbose on cp cp dist/*.tar.gz upload/ cp dist/*.whl upload/ - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@v4 with: - path: upload - name: dist + path: | + upload/*.tar.gz + upload/*.whl + name: dist-${{ matrix.os }}-${{ matrix.python-version }} + retention-days: 5 + # sdist.tar.gz and bdist.whl are compressed by default + compression-level: 0 + if-no-files-found: error diff --git a/gunicorn/__init__.pyi b/gunicorn/__init__.pyi index 1d66ec521..0e0961bdb 100644 --- a/gunicorn/__init__.pyi +++ b/gunicorn/__init__.pyi @@ -1,6 +1,6 @@ from _typeshed import Incomplete -version_info: Incomplete +version_info: tuple[int, int, int] __version__: str SERVER: str SERVER_SOFTWARE: str diff --git a/gunicorn/app/base.py b/gunicorn/app/base.py index 9bf7a4f0f..b11fff8f5 100644 --- a/gunicorn/app/base.py +++ b/gunicorn/app/base.py @@ -6,6 +6,7 @@ import os import sys import traceback +import logging from gunicorn import util from gunicorn.arbiter import Arbiter @@ -13,6 +14,8 @@ from gunicorn import debug +logger = logging.getLogger(__name__) + class BaseApplication: """ An application interface for configuring and loading @@ -101,8 +104,11 @@ def get_config_from_filename(self, filename): if ext in [".py", ".pyc"]: spec = importlib.util.spec_from_file_location(module_name, filename) else: - msg = "configuration file should have a valid Python extension.\n" - util.warn(msg) + if filename == os.devnull: + logger.debug("configuration file %r path matches os.devnull", filename) + else: + msg = "configuration file should have a valid Python extension.\n" + util.warn(msg) loader_ = importlib.machinery.SourceFileLoader(module_name, filename) spec = importlib.util.spec_from_file_location(module_name, filename, loader=loader_) mod = importlib.util.module_from_spec(spec) diff --git a/gunicorn/app/base.pyi b/gunicorn/app/base.pyi index 6e32d8ab9..436eaa50e 100644 --- a/gunicorn/app/base.pyi +++ b/gunicorn/app/base.pyi @@ -1,32 +1,37 @@ +from argparse import ArgumentParser, Namespace + from _typeshed import Incomplete +from _typeshed.wsgi import WSGIApplication as _WSGIApplication from gunicorn import debug as debug from gunicorn import util as util from gunicorn.arbiter import Arbiter as Arbiter -from gunicorn.config import Config as Config +from gunicorn.config import Config from gunicorn.config import get_default_config_file as get_default_config_file +# from abc import abstractmethod, ABCMeta + class BaseApplication: - usage: Incomplete - cfg: Incomplete - callable: Incomplete - prog: Incomplete + usage: str | None + cfg: Config + callable: None | _WSGIApplication + prog: str | None logger: Incomplete - def __init__(self, usage: Incomplete | None = ..., prog: Incomplete | None = ...) -> None: ... + def __init__(self, usage: str | None = ..., prog: str | None = ...) -> None: ... def do_load_config(self) -> None: ... def load_default_config(self) -> None: ... - def init(self, parser, opts, args) -> None: ... - def load(self) -> None: ... + def init(self, parser: ArgumentParser, opts: Namespace, args: list[str]) -> None: ... + def load(self) -> _WSGIApplication: ... def load_config(self) -> None: ... def reload(self) -> None: ... - def wsgi(self): ... + def wsgi(self) -> _WSGIApplication: ... def run(self) -> None: ... class Application(BaseApplication): def chdir(self) -> None: ... - def get_config_from_filename(self, filename): ... - def get_config_from_module_name(self, module_name): ... - def load_config_from_module_name_or_filename(self, location): ... - def load_config_from_file(self, filename): ... + def get_config_from_filename(self, filename: str) -> Config: ... + def get_config_from_module_name(self, module_name: str) -> Config: ... + def load_config_from_module_name_or_filename(self, location: str) -> Config: ... + def load_config_from_file(self, filename: str) -> Config: ... def load_config(self) -> None: ... def run(self) -> None: ... diff --git a/gunicorn/app/pasterapp.pyi b/gunicorn/app/pasterapp.pyi index fcd39f362..0673d1f36 100644 --- a/gunicorn/app/pasterapp.pyi +++ b/gunicorn/app/pasterapp.pyi @@ -1,8 +1,11 @@ from _typeshed import Incomplete +from _typeshed.wsgi import WSGIApplication as _WSGIApplication from gunicorn.app.wsgiapp import WSGIApplication as WSGIApplication from gunicorn.config import get_default_config_file as get_default_config_file -def get_wsgi_app(config_uri, name: Incomplete | None = ..., defaults: Incomplete | None = ...): ... -def has_logging_config(config_file): ... -def serve(app, global_conf, **local_conf): ... +def get_wsgi_app( + config_uri: str, name: Incomplete | None = ..., defaults: Incomplete | None = ... +) -> _WSGIApplication: ... +def has_logging_config(config_file: str) -> bool: ... +def serve(app: _WSGIApplication, global_conf: dict[str, Incomplete], **local_conf: Incomplete) -> None: ... diff --git a/gunicorn/app/wsgiapp.pyi b/gunicorn/app/wsgiapp.pyi index 61ce971c1..4cd943974 100644 --- a/gunicorn/app/wsgiapp.pyi +++ b/gunicorn/app/wsgiapp.pyi @@ -1,4 +1,7 @@ +from argparse import ArgumentParser, Namespace + from _typeshed import Incomplete +from _typeshed.wsgi import WSGIApplication as _WSGIApplication from gunicorn import util as util from gunicorn.app.base import Application as Application @@ -6,10 +9,10 @@ from gunicorn.errors import ConfigError as ConfigError class WSGIApplication(Application): app_uri: Incomplete - def init(self, parser, opts, args) -> None: ... + def init(self, parser: ArgumentParser, opts: Namespace, args: list[str]) -> None: ... def load_config(self) -> None: ... - def load_wsgiapp(self): ... - def load_pasteapp(self): ... - def load(self): ... + def load_wsgiapp(self) -> _WSGIApplication: ... + def load_pasteapp(self) -> _WSGIApplication: ... + def load(self) -> _WSGIApplication: ... -def run() -> None: ... +def run(prog: str | None = ...) -> None: ... diff --git a/gunicorn/arbiter.py b/gunicorn/arbiter.py index 2e97fbae5..fb4411036 100644 --- a/gunicorn/arbiter.py +++ b/gunicorn/arbiter.py @@ -40,17 +40,12 @@ class Arbiter: # I love dynamic languages SIG_QUEUE = [] - _want_signals_unix = set("HUP QUIT INT TERM TTIN TTOU USR1 USR2 WINCH".split()) + SIGNALS = [getattr(_signal, "SIG%s" % x) + for x in "HUP QUIT INT TERM TTIN TTOU USR1 USR2 WINCH".split()] SIG_NAMES = dict( - (getattr(_signal, name), name[3:]) for name in dir(_signal) + (getattr(_signal, name), name[3:].lower()) for name in dir(_signal) if name[:3] == "SIG" and name[3] != "_" ) - # FIXME: nonsense. just testing some unrelated stuff that IS available in wondows - SIGNALS = {getattr(_signal, "SIG%s" % x, None) for x in _want_signals_unix} - SIGNALS = [sig for sig in SIGNALS if sig is not None] - - # FIXME: nonsense. just testing some unrelated stuff that IS available in wondows - SIGKILL = getattr(_signal, "SIGKILL", _signal.SIGTERM) def __init__(self, app): os.environ["SERVER_SOFTWARE"] = SERVER_SOFTWARE @@ -396,7 +391,7 @@ def stop(self, graceful=True): while self.WORKERS and time.time() < limit: time.sleep(0.1) - self.kill_workers(self.SIGKILL) + self.kill_workers(_signal.SIGKILL) def reexec(self): """\ @@ -506,7 +501,7 @@ def murder_workers(self): worker.aborted = True self.kill_worker(pid, _signal.SIGABRT) else: - self.kill_worker(pid, self.SIGKILL) + self.kill_worker(pid, _signal.SIGKILL) def reap_workers(self): """\ @@ -550,7 +545,7 @@ def reap_workers(self): wpid, sig_name) # Additional hint for SIGKILL - if status == self.SIGKILL: + if status == _signal.SIGKILL: msg += " Perhaps out of memory?" self.log.error(msg) diff --git a/gunicorn/config.py b/gunicorn/config.py index 5f92270cd..1bc351c5d 100644 --- a/gunicorn/config.py +++ b/gunicorn/config.py @@ -2378,3 +2378,44 @@ class TolerateDangerousFraming(Setting): .. versionadded:: 22.0.0 """ + +def validate_fatal_behaviour(val): + # FIXME: refactor all of this subclassing stdlib argparse + + if val is None: + return + + if not isinstance(val, str): + raise TypeError("Invalid type for casting: %s" % val) + if val.lower().strip() == "world-readable": + return "world-readable" + elif val.lower().strip() == "refuse": + return "refuse" + elif val.lower().strip() == "quiet": + return "quiet" + elif val.lower().strip() == "guess": + return "guess" + else: + raise ValueError("Invalid header map behaviour: %s" % val) + + +class OnFatal(Setting): + name = "on_fatal" + section = "Server Mechanics" + cli = ["--on-fatal"] + validator = validate_fatal_behaviour + default = "guess" + desc = """\ + Configure what to do if loading the application failes + + If set to ``world-readable``, always send the traceback too the client. + + The defaule behaviour ``guess`` is to share the traceback with the world + if the reloader is enabled. If set to ``refuse``, stop processing requests. + If set to ``quiet``, respond with error status but do not share internals. + + The behaviour of ``world-readable`` (or, by extension ``guess``) risks exposing + sensitive data and is not recommended for production use. + + .. versionadded:: 22.0.0 + """ diff --git a/gunicorn/debug.pyi b/gunicorn/debug.pyi index 744a1437f..a9803a7cb 100644 --- a/gunicorn/debug.pyi +++ b/gunicorn/debug.pyi @@ -1,10 +1,17 @@ +from collections.abc import Container +from types import FrameType +from typing import Any, Literal + from _typeshed import Incomplete +from typing_extensions import Self class Spew: trace_names: Incomplete show_values: Incomplete - def __init__(self, trace_names: Incomplete | None = ..., show_values: bool = ...) -> None: ... - def __call__(self, frame, event, arg): ... + def __init__(self, trace_names: Container[str] | None = ..., show_values: bool = ...) -> None: ... + def __call__( + self, frame: FrameType, event: Literal["call", "line", "return", "exception", "opcode"], arg: Any + ) -> Self: ... -def spew(trace_names: Incomplete | None = ..., show_values: bool = ...) -> None: ... +def spew(trace_names: Container[str] | None = ..., show_values: bool = ...) -> None: ... def unspew() -> None: ... diff --git a/gunicorn/errors.pyi b/gunicorn/errors.pyi index 63c7c1f52..fb5f48b1a 100644 --- a/gunicorn/errors.pyi +++ b/gunicorn/errors.pyi @@ -1,8 +1,8 @@ from _typeshed import Incomplete class HaltServer(BaseException): - reason: Incomplete - exit_status: Incomplete + reason: str + exit_status: int def __init__(self, reason: str, exit_status: int = ...) -> None: ... class ConfigError(Exception): ... diff --git a/gunicorn/glogging.pyi b/gunicorn/glogging.pyi index 8a2cfd245..bec35fd23 100644 --- a/gunicorn/glogging.pyi +++ b/gunicorn/glogging.pyi @@ -1,17 +1,20 @@ -from _typeshed import Incomplete +import socket +from logging import Logger as _Logger +from logging.config import _DictConfigArgs +from typing import Literal -from gunicorn import util as util +from _typeshed import Incomplete -SYSLOG_FACILITIES: Incomplete -CONFIG_DEFAULTS: Incomplete +SYSLOG_FACILITIES: dict[str, int] +CONFIG_DEFAULTS: _DictConfigArgs -def loggers(): ... +def loggers() -> list[_Logger]: ... class SafeAtoms(dict): def __init__(self, atoms) -> None: ... def __getitem__(self, k): ... -def parse_syslog_address(addr): ... +def parse_syslog_address(addr: str) -> tuple[socket.SocketKind, tuple[str, int]]: ... class Logger: LOG_LEVELS: Incomplete @@ -39,6 +42,6 @@ class Logger: def log(self, lvl, msg, *args, **kwargs) -> None: ... def atoms(self, resp, req, environ, request_time): ... def access(self, resp, req, environ, request_time) -> None: ... - def now(self): ... + def now(self) -> str: ... def reopen_files(self) -> None: ... def close_on_exec(self) -> None: ... diff --git a/gunicorn/http/body.pyi b/gunicorn/http/body.pyi index 851363773..f1eeacd20 100644 --- a/gunicorn/http/body.pyi +++ b/gunicorn/http/body.pyi @@ -1,43 +1,48 @@ -from collections.abc import Generator +from collections.abc import Generator, Iterator +from io import BytesIO from _typeshed import Incomplete +from typing_extensions import Protocol, Self -from gunicorn.http.errors import ChunkMissingTerminator as ChunkMissingTerminator -from gunicorn.http.errors import InvalidChunkSize as InvalidChunkSize -from gunicorn.http.errors import NoMoreData as NoMoreData +from gunicorn.http.errors import ChunkMissingTerminator, InvalidChunkSize, NoMoreData +from gunicorn.http.message import Message +from gunicorn.http.unreader import Unreader + +class _Read(Protocol): + def read(self, size: int) -> bytes: ... class ChunkedReader: - req: Incomplete + req: Message parser: Incomplete - buf: Incomplete - def __init__(self, req, unreader) -> None: ... - def read(self, size): ... - def parse_trailers(self, unreader, data): ... - def parse_chunked(self, unreader) -> Generator[Incomplete, None, None]: ... - def parse_chunk_size(self, unreader, data: Incomplete | None = ...): ... - def get_data(self, unreader, buf) -> None: ... + buf: BytesIO + def __init__(self, req: Message, unreader: Unreader) -> None: ... + def read(self, size: int) -> bytes: ... + def parse_trailers(self, unreader: Unreader, data: bytes) -> bytes: ... + def parse_chunked(self, unreader: Unreader) -> Generator[bytes, None, None]: ... + def parse_chunk_size(self, unreader: Unreader, data: bytes | None = ...) -> tuple[int, bytes]: ... + def get_data(self, unreader: Unreader, buf: BytesIO) -> None: ... class LengthReader: - unreader: Incomplete - length: Incomplete - def __init__(self, unreader, length) -> None: ... - def read(self, size): ... + unreader: Unreader + length: int + def __init__(self, unreader: Unreader, length: int) -> None: ... + def read(self, size: int) -> bytes: ... class EOFReader: - unreader: Incomplete - buf: Incomplete + unreader: Unreader + buf: BytesIO finished: bool - def __init__(self, unreader) -> None: ... - def read(self, size): ... + def __init__(self, unreader: Unreader) -> None: ... + def read(self, size: int) -> bytes: ... class Body: - reader: Incomplete - buf: Incomplete - def __init__(self, reader) -> None: ... - def __iter__(self): ... - def __next__(self): ... + reader: _Read + buf: BytesIO + def __init__(self, reader: _Read) -> None: ... + def __iter__(self) -> Self: ... + def __next__(self) -> Iterator[bytes]: ... next = __next__ - def getsize(self, size): ... - def read(self, size: Incomplete | None = ...): ... - def readline(self, size: Incomplete | None = ...): ... - def readlines(self, size: Incomplete | None = ...): ... + def getsize(self, size: int) -> int: ... + def read(self, size: int | None = ...) -> bytes: ... + def readline(self, size: int | None = ...) -> bytes: ... + def readlines(self, size: int | None = ...) -> list[bytes]: ... diff --git a/gunicorn/http/errors.pyi b/gunicorn/http/errors.pyi index 6d31a235c..00602cd35 100644 --- a/gunicorn/http/errors.pyi +++ b/gunicorn/http/errors.pyi @@ -1,58 +1,65 @@ from _typeshed import Incomplete +from gunicorn.http.message import Message + class ParseException(Exception): ... class NoMoreData(IOError): buf: Incomplete def __init__(self, buf: bytes = ...) -> None: ... +class ConfigurationProblem(ParseException): + info: str + code: int + def __init__(self, info: str) -> None: ... + class InvalidRequestLine(ParseException): - req: Incomplete + req: str code: int def __init__(self, req: str) -> None: ... class InvalidRequestMethod(ParseException): - method: Incomplete + method: str def __init__(self, method: str) -> None: ... class InvalidHTTPVersion(ParseException): - version: Incomplete + version: str def __init__(self, version: str) -> None: ... class InvalidHeader(ParseException): - hdr: Incomplete - req: Incomplete - def __init__(self, hdr, req: Incomplete | None = ...) -> None: ... + hdr: str + req: Message | None + def __init__(self, hdr: str, req: Message | None = ...) -> None: ... class InvalidHeaderName(ParseException): - hdr: Incomplete - def __init__(self, hdr) -> None: ... + hdr: str + def __init__(self, hdr: str) -> None: ... class InvalidChunkSize(IOError): - data: Incomplete - def __init__(self, data) -> None: ... + data: bytes + def __init__(self, data: bytes) -> None: ... class ChunkMissingTerminator(IOError): - term: Incomplete - def __init__(self, term) -> None: ... + term: bytes + def __init__(self, term: bytes) -> None: ... class LimitRequestLine(ParseException): - size: Incomplete - max_size: Incomplete - def __init__(self, size, max_size) -> None: ... + size: int + max_size: int + def __init__(self, size: int, max_size: int) -> None: ... class LimitRequestHeaders(ParseException): - msg: Incomplete - def __init__(self, msg) -> None: ... + msg: str + def __init__(self, msg: str) -> None: ... class InvalidProxyLine(ParseException): - line: Incomplete + line: str code: int - def __init__(self, line) -> None: ... + def __init__(self, line: str) -> None: ... class ForbiddenProxyRequest(ParseException): - host: Incomplete + host: str code: int - def __init__(self, host) -> None: ... + def __init__(self, host: str) -> None: ... class InvalidSchemeHeaders(ParseException): ... diff --git a/gunicorn/http/message.pyi b/gunicorn/http/message.pyi index 9f83256e8..3364d5057 100644 --- a/gunicorn/http/message.pyi +++ b/gunicorn/http/message.pyi @@ -1,24 +1,23 @@ -from typing import BinaryIO +from typing import BinaryIO, Literal, TypeAlias from _typeshed import Incomplete -from gunicorn.http.body import Body as Body -from gunicorn.http.body import ChunkedReader as ChunkedReader -from gunicorn.http.body import EOFReader as EOFReader -from gunicorn.http.body import LengthReader as LengthReader -from gunicorn.http.errors import ForbiddenProxyRequest as ForbiddenProxyRequest -from gunicorn.http.errors import InvalidHeader as InvalidHeader -from gunicorn.http.errors import InvalidHeaderName as InvalidHeaderName -from gunicorn.http.errors import InvalidHTTPVersion as InvalidHTTPVersion -from gunicorn.http.errors import InvalidProxyLine as InvalidProxyLine -from gunicorn.http.errors import InvalidRequestLine as InvalidRequestLine -from gunicorn.http.errors import InvalidRequestMethod as InvalidRequestMethod -from gunicorn.http.errors import InvalidSchemeHeaders as InvalidSchemeHeaders -from gunicorn.http.errors import LimitRequestHeaders as LimitRequestHeaders -from gunicorn.http.errors import LimitRequestLine as LimitRequestLine -from gunicorn.http.errors import NoMoreData as NoMoreData -from gunicorn.util import bytes_to_str as bytes_to_str -from gunicorn.util import split_request_uri as split_request_uri +from gunicorn.config import Config +from gunicorn.http.body import Body, ChunkedReader, EOFReader, LengthReader +from gunicorn.http.errors import ( + ForbiddenProxyRequest, + InvalidHeader, + InvalidHeaderName, + InvalidHTTPVersion, + InvalidProxyLine, + InvalidRequestLine, + InvalidRequestMethod, + InvalidSchemeHeaders, + LimitRequestHeaders, + LimitRequestLine, + NoMoreData, +) +from gunicorn.http.unreader import Unreader MAX_REQUEST_LINE: int MAX_HEADERS: int @@ -27,24 +26,27 @@ HEADER_RE: Incomplete METH_RE: Incomplete VERSION_RE: Incomplete +PeerAddr: TypeAlias = tuple[str, int] | str + class Message: - cfg: Incomplete - unreader: Incomplete - peer_addr: Incomplete - remote_addr: Incomplete - version: Incomplete - headers: Incomplete - trailers: Incomplete - body: Incomplete - scheme: Incomplete + cfg: Config + unreader: Unreader + peer_addr: PeerAddr + remote_addr: PeerAddr + version: tuple[int, int] + headers: list[tuple[str, str]] + trailers: list[tuple[str, str]] + body: Body | None + scheme: Literal["https", "http"] + must_close: bool limit_request_fields: Incomplete limit_request_field_size: Incomplete max_buffer_headers: Incomplete - def __init__(self, cfg, unreader, peer_addr) -> None: ... - def parse(self, unreader) -> None: ... - def parse_headers(self, data): ... + def __init__(self, cfg: Config, unreader: Unreader, peer_addr: PeerAddr) -> None: ... + def parse(self, unreader: Unreader) -> None: ... + def parse_headers(self, data: bytes, from_trailers: bool) -> Incomplete: ... def set_body_reader(self) -> None: ... - def should_close(self): ... + def should_close(self) -> bool: ... class Request(Message): method: Incomplete @@ -55,15 +57,12 @@ class Request(Message): limit_request_line: Incomplete req_number: Incomplete proxy_protocol_info: Incomplete - def __init__(self, cfg, unreader, peer_addr, req_number: int = ...) -> None: ... - def get_data(self, unreader, buf: BinaryIO, stop: bool = ...): ... + def __init__(self, cfg: Config, unreader: Unreader, peer_addr: PeerAddr, req_number: int = ...) -> None: ... + def get_data(self, unreader: Config, buf: BinaryIO, stop: bool = ...) -> Incomplete: ... headers: Incomplete - def parse(self, unreader): ... - def read_line(self, unreader, buf, limit: int = ...): ... - def proxy_protocol(self, line): ... + def read_line(self, unreader: Unreader, buf: BinaryIO, limit: int = ...) -> Incomplete: ... + def proxy_protocol(self, line: str) -> Incomplete: ... def proxy_protocol_access_check(self) -> None: ... - def parse_proxy_protocol(self, line) -> None: ... - version: Incomplete - def parse_request_line(self, line_bytes) -> None: ... - body: Incomplete + def parse_proxy_protocol(self, line: str) -> None: ... + def parse_request_line(self, line_bytes: bytes) -> None: ... def set_body_reader(self) -> None: ... diff --git a/gunicorn/http/unreader.pyi b/gunicorn/http/unreader.pyi index 9267db394..55387d850 100644 --- a/gunicorn/http/unreader.pyi +++ b/gunicorn/http/unreader.pyi @@ -1,11 +1,11 @@ import socket -from collections.abc import Iterator -from typing import Optional +from collections.abc import Iterable, Iterator +from typing import IO, Optional from _typeshed import Incomplete class Unreader: - buf: Incomplete + buf: IO[bytes] def __init__(self) -> None: ... def chunk(self) -> bytes: ... def read(self, size: int | None = ...) -> bytes: ... @@ -18,6 +18,6 @@ class SocketUnreader(Unreader): def chunk(self) -> bytes: ... class IterUnreader(Unreader): - iter: Incomplete - def __init__(self, iterable: Iterator) -> None: ... + iter: Iterator[bytes] + def __init__(self, iterable: Iterable[bytes]) -> None: ... def chunk(self) -> bytes: ... diff --git a/gunicorn/http/wsgi.pyi b/gunicorn/http/wsgi.pyi index 1779a9744..6eaf6ea6a 100644 --- a/gunicorn/http/wsgi.pyi +++ b/gunicorn/http/wsgi.pyi @@ -1,59 +1,64 @@ import io +from collections.abc import Callable, Sequence +from logging import Logger +from re import Pattern from _typeshed import Incomplete +from _typeshed.wsgi import StartResponse, WSGIEnvironment +from typing_extensions import Never from gunicorn import SERVER as SERVER from gunicorn import SERVER_SOFTWARE as SERVER_SOFTWARE -from gunicorn import util as util -from gunicorn.http.errors import InvalidHeader as InvalidHeader -from gunicorn.http.errors import InvalidHeaderName as InvalidHeaderName -from gunicorn.http.message import HEADER_RE as HEADER_RE +from gunicorn.config import Config +from gunicorn.http.message import HEADER_RE, Message BLKSIZE: int -HEADER_VALUE_RE: Incomplete -log: Incomplete +HEADER_VALUE_RE: Pattern[str] +log: Logger class FileWrapper: filelike: Incomplete - blksize: Incomplete - close: Incomplete - def __init__(self, filelike, blksize: int = ...) -> None: ... - def __getitem__(self, key): ... + blksize: int + close: Callable[[], None] + def __init__(self, filelike: Incomplete, blksize: int = ...) -> None: ... + def __getitem__(self, key: Never) -> Incomplete: ... class WSGIErrorsWrapper(io.RawIOBase): streams: Incomplete - def __init__(self, cfg) -> None: ... - def write(self, data) -> None: ... + def __init__(self, cfg: Config) -> None: ... + def write(self, data: Incomplete) -> None: ... -def base_environ(cfg): ... -def default_environ(req, sock, cfg): ... -def proxy_environ(req): ... -def create(req, sock, client, server, cfg): ... +def base_environ(cfg: Config) -> WSGIEnvironment: ... +def default_environ(req: Message, sock: Incomplete, cfg: Config) -> WSGIEnvironment: ... +def proxy_environ(req: Message) -> WSGIEnvironment: ... +def create( + req: Message, sock: Incomplete, client: Incomplete, server: Incomplete, cfg: Config +) -> tuple[Response, WSGIEnvironment]: ... class Response: - req: Incomplete + req: Message sock: Incomplete - version: Incomplete - status: Incomplete + version: str + status: int chunked: bool must_close: bool - headers: Incomplete + headers: list[tuple[str, str]] headers_sent: bool - response_length: Incomplete + response_length: None | int sent: int upgrade: bool - cfg: Incomplete - def __init__(self, req, sock, cfg) -> None: ... + cfg: Config + def __init__(self, req: Message, sock: Incomplete, cfg: Config) -> None: ... def force_close(self) -> None: ... - def should_close(self): ... - status_code: Incomplete - def start_response(self, status, headers, exc_info: Incomplete | None = ...): ... - def process_headers(self, headers) -> None: ... - def is_chunked(self): ... - def default_headers(self): ... + def should_close(self) -> bool: ... + status_code: int + start_response: StartResponse + def process_headers(self, headers: Sequence[tuple[str, str]]) -> None: ... + def is_chunked(self) -> bool: ... + def default_headers(self) -> Incomplete: ... def send_headers(self) -> None: ... - def write(self, arg) -> None: ... - def can_sendfile(self): ... - def sendfile(self, respiter): ... - def write_file(self, respiter) -> None: ... + def write(self, arg: bytes) -> None: ... + def can_sendfile(self) -> bool: ... + def sendfile(self, respiter: Incomplete) -> Incomplete: ... + def write_file(self, respiter: Incomplete) -> None: ... def close(self) -> None: ... diff --git a/gunicorn/packaging_support.py b/gunicorn/packaging_support.py new file mode 100644 index 000000000..b8153ece2 --- /dev/null +++ b/gunicorn/packaging_support.py @@ -0,0 +1,145 @@ +import warnings +import packaging +import importlib +from packaging.version import parse as parse_version +from packaging.requirements import Requirement +from packaging.markers import Marker +from packaging.specifiers import SpecifierSet + +import sys + +# no importing of gunicorn package here +SERVER = "gunicorn" + +if sys.version_info >= (3, 8): + import importlib.metadata as importlib_metadata +else: + import importlib_metadata + + +def _minimum_version_of_installed(package_name, extras=set(), spew=True): + try: + # try to figure out what will crash in odd places + # take the lowest version mentioned and accepted by any extra + all_spec = [] + for req_str in importlib_metadata.requires("gunicorn"): + requirement = Requirement(req_str) + if requirement.name == package_name and requirement.extras == extras: + all_spec.append(requirement.specifier) + # simplication: this will fail on > operator (mentioned is not acceptable) + all_version = [spec.version for spec_list in all_spec for spec in spec_list] + acceptable_mentioned = sum( + (list(spec_list.filter(all_version)) for spec_list in all_spec), start=[] + ) + lowest_mentioned = min(acceptable_mentioned, default=None) + if lowest_mentioned: + return parse_version(lowest_mentioned) + else: + if spew: + warnings.warn( + "The installed version of %s has not been updated to specify and/or process requirements for %r package" + % ( + SERVER, + package_name, + ), + DeprecationWarning, + stacklevel=2, + ) + return None + except importlib_metadata.PackageNotFoundError: + # not installed, editable install, .. + return None + + +def _dependency_error(package_name): + want_version = _minimum_version_of_installed(package_name) + + def error_msg(): + if want_version: + return "%s %s worker requires %s %r" % ( + SERVER, + package_name, + package_name, + want_version, + ) + return "%s %s worker requires %s to be installed" % ( + SERVER, + package_name, + package_name, + ) + + try: + module = importlib.import_module(package_name) + dunder_version = parse_version(getattr(module, "__version__")) + except ImportError: + raise RuntimeError(error_msg()) + except AttributeError: + installed_version = parse_version(importlib_metadata.version(package_name)) + warnings.warn( + "The running version of %s has not been updated for API changes in %s (installed version: %r." + % ( + SERVER, + package_name, + installed_version, + ), + DeprecationWarning, + ) + return str(installed_version) + else: + if want_version is not None and dunder_version < want_version: + raise RuntimeError(error_msg()) + return str(dunder_version) + + +def requirements_min(): + installed_meta = importlib_metadata.requires("gunicorn") + + all_req = [] + fixed_req = [] + + for req_str in installed_meta: + requirement = Requirement(req_str) + if any ( ignored in str(requirement.marker) for ignored in ("lint", "style", "mindep") ): + # ignoring lint-docs extra + # print("ignored", requirement) + continue + minver = _minimum_version_of_installed(requirement.name, spew=False) + if minver is not None: + all_req.append("%s%s==%s" % (requirement.name, "[%s]" % ",".join(requirement.extras) if requirement.extras else "", minver)) + + for req_str in installed_meta: + requirement = Requirement(req_str) + if requirement.marker is None: + continue + if not requirement.marker.evaluate({"extra": "testing-mindep"}): + # print("ignored", requirement) + continue + minver = _minimum_version_of_installed(requirement.name, spew=False) + if minver is not None: + fixed_req.append( + "%s%s==%s" % ( + requirement.name, + "[%s]" % ",".join(requirement.extras) if requirement.extras else "", + minver, + ) + ) + + return set(all_req), set(fixed_req) + + +if __name__ == "__main__": + import argparse + parser = argparse.ArgumentParser() + parser.add_argument("--verbose", action="store_true") + parser.add_argument("--check", action="store_true") + args = parser.parse_args() + expected, installed = requirements_min() + if args.verbose: + print("expected:") + for req in expected: print(req) + print() + print("installed:") + for req in installed: print(req) + if expected != installed: + raise RuntimeError("optional-dependencies of testing-min extra installed %r, expected %r, difference: %r" %( installed, expected, installed ^ expected)) + sys.exit(0) diff --git a/gunicorn/pidfile.pyi b/gunicorn/pidfile.pyi index 1176f9b43..b7d081edd 100644 --- a/gunicorn/pidfile.pyi +++ b/gunicorn/pidfile.pyi @@ -1,10 +1,11 @@ from _typeshed import Incomplete class Pidfile: - fname: Incomplete - pid: Incomplete - def __init__(self, fname) -> None: ... - def create(self, pid) -> None: ... - def rename(self, path) -> None: ... + fname: str + pid: int | None + + def __init__(self, fname: str) -> None: ... + def create(self, pid: int) -> None: ... + def rename(self, path: str) -> None: ... def unlink(self) -> None: ... - def validate(self): ... + def validate(self) -> None: ... diff --git a/gunicorn/reloader.pyi b/gunicorn/reloader.pyi index b5377a0cb..de79d788a 100644 --- a/gunicorn/reloader.pyi +++ b/gunicorn/reloader.pyi @@ -1,8 +1,13 @@ import threading +from collections.abc import Callable, Iterable +from re import Pattern +from typing import TypeAlias from _typeshed import Incomplete -COMPILED_EXT_RE: Incomplete +COMPILED_EXT_RE: Pattern[str] + +reloader_cb: TypeAlias = Callable[[str], None] class Reloader(threading.Thread): daemon: bool @@ -12,20 +17,20 @@ class Reloader(threading.Thread): interval: int = ..., callback: Incomplete | None = ..., ) -> None: ... - def add_extra_file(self, filename) -> None: ... - def get_files(self): ... + def add_extra_file(self, filename: str) -> None: ... + def get_files(self) -> list[str]: ... def run(self) -> None: ... has_inotify: bool # ignoring duplicate - that one is guaranteed to error on actual use class InotifyReloader(threading.Thread): - event_mask: Incomplete + event_mask: int daemon: bool - def __init__(self, extra_files: Incomplete | None = ..., callback: Incomplete | None = ...) -> None: ... - def add_extra_file(self, filename) -> None: ... - def get_dirs(self): ... + def __init__(self, extra_files: Iterable[str] | None = ..., callback: reloader_cb | None = ...) -> None: ... + def add_extra_file(self, filename: str) -> None: ... + def get_dirs(self) -> list[str]: ... def run(self) -> None: ... -preferred_reloader: Incomplete -reloader_engines: Incomplete +preferred_reloader: type[InotifyReloader] | type[Reloader] +reloader_engines: dict[str, type[InotifyReloader] | type[Reloader]] diff --git a/gunicorn/sock.pyi b/gunicorn/sock.pyi index 381a029a7..a808a3d46 100644 --- a/gunicorn/sock.pyi +++ b/gunicorn/sock.pyi @@ -1,5 +1,6 @@ import socket from collections.abc import Callable +from ssl import SSLContext, SSLSocket from typing import Any from _typeshed import Incomplete @@ -14,7 +15,7 @@ class BaseSocket: cfg_addr: Incomplete sock: Incomplete def __init__(self, address: tuple[str, int], conf: Config, log: Logger, fd: None = ...) -> None: ... - def __getattr__(self, name: str) -> Callable: ... + def __getattr__(self, name: str) -> Incomplete: ... def set_options(self, sock: socket.socket, bound: bool = ...) -> socket.socket: ... def bind(self, sock: socket.socket) -> None: ... def close(self) -> None: ... @@ -28,10 +29,10 @@ class TCP6Socket(TCPSocket): class UnixSocket(BaseSocket): FAMILY: Incomplete - def __init__(self, addr, conf, log, fd: Incomplete | None = ...) -> None: ... - def bind(self, sock) -> None: ... + def __init__(self, addr: str, conf: Config, log: Logger, fd: list[int] | None = ...) -> None: ... + def bind(self, sock: socket.socket) -> None: ... def create_sockets(conf: Config, log: Logger, fds: None = ...) -> list[TCPSocket]: ... def close_sockets(listeners: list[TCPSocket | Any], unlink: bool = ...) -> None: ... -def ssl_context(conf): ... -def ssl_wrap_socket(sock, conf): ... +def ssl_context(conf: Config) -> SSLContext: ... +def ssl_wrap_socket(sock: socket.socket, conf: Config) -> SSLSocket: ... diff --git a/gunicorn/util.py b/gunicorn/util.py index 639c6cd64..45ccf2a9b 100644 --- a/gunicorn/util.py +++ b/gunicorn/util.py @@ -19,15 +19,18 @@ import traceback import warnings -try: +if sys.version_info >= (3, 8): import importlib.metadata as importlib_metadata -except (ModuleNotFoundError, ImportError): +else: import importlib_metadata from gunicorn.errors import AppImportError from gunicorn.workers import SUPPORTED_WORKERS import urllib.parse +# RFC9112 7.1 +REASON_PHRASE_RE = re.compile(r'[ \t\x21-\x7e\x80-\xff]*') + REDIRECT_TO = getattr(os, 'devnull', '/dev/null') # Server and Date aren't technically hop-by-hop @@ -54,6 +57,14 @@ def _setproctitle(title): pass +def get_dependency_version(package_name): + try: + from gunicorn.packaging_support import _dependency_error + return _dependency_error(package_name) + except Exception as ex: + warnings.warn("gunicorn.packaging_support unavailale - unmet dependencies will not produce nice tracebacks", RuntimeWarning) + return "" + def load_entry_point(distribution, group, name): dist_obj = importlib_metadata.distribution(distribution) eps = [ep for ep in dist_obj.entry_points @@ -236,7 +247,11 @@ def close(sock): try: - from os import closerange + from os import closerange as _closerange + + # mypy workaround + def closerange(fd_low, fd_high): + return _closerange(fd_low, fd_high) except ImportError: def closerange(fd_low, fd_high): # Iterate through and close all file descriptors. @@ -274,6 +289,13 @@ def write_nonblock(sock, data, chunked=False): def write_error(sock, status_int, reason, mesg): + # we may reflect malformed input in (content-type-appropriately escaped) mesg + # .. but we shall never send invalid HTTP status lines + assert REASON_PHRASE_RE.fullmatch(reason) + # we should not be sending error codes encoding special cases in our proxies + # RFC9110 15 + assert 100 <= reason <= 599 + html_error = textwrap.dedent("""\ @@ -428,7 +450,7 @@ def http_date(timestamp=None): def is_hoppish(header): - return header.lower().strip() in hop_headers + return header.lower() in hop_headers def daemonize(enable_stdio_inheritance=False): diff --git a/gunicorn/util.pyi b/gunicorn/util.pyi index aad5203b0..e006b1cea 100644 --- a/gunicorn/util.pyi +++ b/gunicorn/util.pyi @@ -1,44 +1,55 @@ +import importlib.metadata as importlib_metadata +import sys import typing -from typing import Union +from collections.abc import Callable, Iterable +from os import PathLike +from socket import socket +from types import TracebackType +from typing import Any, Protocol, Union from _typeshed import Incomplete +from _typeshed.wsgi import WSGIApplication +from typing_extensions import LiteralString from gunicorn.errors import AppImportError as AppImportError from gunicorn.glogging import Logger from gunicorn.workers import SUPPORTED_WORKERS as SUPPORTED_WORKERS from gunicorn.workers.sync import SyncWorker -REDIRECT_TO: Incomplete -hop_headers: Incomplete +REDIRECT_TO: str +hop_headers: set[str] -def load_entry_point(distribution, group, name): ... +def load_entry_point(distribution: str, group: str, name: str) -> type[Logger] | type[SyncWorker]: ... def load_class(uri: str, default: str = ..., section: str = ...) -> type[Logger] | type[SyncWorker]: ... positionals: Incomplete -def get_arity(f: typing.Callable) -> int: ... +def _setproctitle(title: str) -> None: ... +def better_dependency_error(package_name: str) -> None: ... +def get_arity(f: typing.Callable[..., None]) -> int: ... def get_username(uid: int) -> str: ... -def chown(path, uid, gid) -> None: ... +def chown(path: str | PathLike[str], uid: int, gid: int) -> None: ... def unlink(filename: str) -> None: ... def is_ipv6(addr: str) -> bool: ... def parse_address(netloc: str, default_port: str = ...) -> tuple[str, int]: ... -def close(sock) -> None: ... -def write_chunk(sock, data) -> None: ... -def write(sock, data, chunked: bool = ...): ... -def write_nonblock(sock, data, chunked: bool = ...): ... -def write_error(sock, status_int, reason, mesg) -> None: ... -def import_app(module: str) -> typing.Callable: ... +def close(sock: socket) -> None: ... +def closerange(fd_low: int, fd_high: int) -> None: ... +def write_chunk(sock: socket, data: bytes) -> None: ... +def write(sock: socket, data: bytes, chunked: bool = ...) -> None: ... +def write_nonblock(sock: socket, data: bytes, chunked: bool = ...) -> int: ... +def write_error(sock: socket, status_int: int, reason: LiteralString, mesg: str) -> None: ... +def import_app(module: str) -> WSGIApplication: ... def getcwd() -> str: ... -def http_date(timestamp: Incomplete | None = ...): ... -def is_hoppish(header): ... +def http_date(timestamp: Incomplete | None = ...) -> str: ... +def is_hoppish(header: str) -> bool: ... def daemonize(enable_stdio_inheritance: bool = ...) -> None: ... def seed() -> None: ... -def check_is_writable(path) -> None: ... -def to_bytestring(value, encoding: str = ...): ... -def has_fileno(obj): ... -def warn(msg) -> None: ... -def make_fail_app(msg): ... -def split_request_uri(uri): ... -def reraise(tp, value, tb: Incomplete | None = ...) -> None: ... +def check_is_writable(path: str | PathLike[str]) -> None: ... +def to_bytestring(value: Incomplete, encoding: LiteralString = ...) -> bytes: ... +def has_fileno(obj: Any) -> bool: ... +def warn(msg: LiteralString) -> None: ... +def make_fail_app(msg: str) -> WSGIApplication: ... +def split_request_uri(uri: str) -> tuple[str, str, str, str, str]: ... +def reraise(tp: type[BaseException], value: BaseException, tb: TracebackType | None = ...) -> None: ... def bytes_to_str(b: bytes | str) -> str: ... -def unquote_to_wsgi_str(string): ... +def unquote_to_wsgi_str(string: bytes) -> str: ... diff --git a/gunicorn/workers/base.py b/gunicorn/workers/base.py index 26f681e37..3c0f2d890 100644 --- a/gunicorn/workers/base.py +++ b/gunicorn/workers/base.py @@ -27,10 +27,9 @@ class Worker: - # FIXME: nonsense. just testing some unrelated stuff that IS available in wondows - _want_signals_unix = set("ABRT HUP QUIT INT TERM USR1 USR2 WINCH CHLD".split()) - SIGNALS = [getattr(signal, "SIG%s" % x, None) for x in _want_signals_unix] - SIGNALS = [sig for sig in SIGNALS if sig is not None] + SIGNALS = [getattr(signal, "SIG%s" % x) for x in ( + "ABRT HUP QUIT INT TERM USR1 USR2 WINCH CHLD".split() + )] PIPE = [] @@ -143,7 +142,16 @@ def load_wsgi(self): try: self.wsgi = self.app.wsgi() except SyntaxError as e: - if not self.cfg.reload: + if self.cfg.on_fatal == "world-readable": + pass + elif self.cfg.on_fatal == "quiet": + self.log.exception(e) + self.wsgi = util.make_fail_app("Internal Server Error") + return + elif self.cfg.on_fatal == "guess" and (self.cfg.reload or self.cfg.reload_extra_files): + pass + else: + # secure fallthrough: "refuse" raise self.log.exception(e) diff --git a/gunicorn/workers/base.pyi b/gunicorn/workers/base.pyi index adb68d564..fc0dc025c 100644 --- a/gunicorn/workers/base.pyi +++ b/gunicorn/workers/base.pyi @@ -1,27 +1,32 @@ -from typing import List +from signal import Signals +from socket import socket +from types import FrameType +from typing import List, TypeAlias from _typeshed import Incomplete -from gunicorn import util as util -from gunicorn.app.wsgiapp import WSGIApplication as WSGIApplication -from gunicorn.config import Config as Config -from gunicorn.glogging import Logger as Logger -from gunicorn.http.errors import ForbiddenProxyRequest as ForbiddenProxyRequest -from gunicorn.http.errors import InvalidHeader as InvalidHeader -from gunicorn.http.errors import InvalidHeaderName as InvalidHeaderName -from gunicorn.http.errors import InvalidHTTPVersion as InvalidHTTPVersion -from gunicorn.http.errors import InvalidProxyLine as InvalidProxyLine -from gunicorn.http.errors import InvalidRequestLine as InvalidRequestLine -from gunicorn.http.errors import InvalidRequestMethod as InvalidRequestMethod -from gunicorn.http.errors import InvalidSchemeHeaders as InvalidSchemeHeaders -from gunicorn.http.errors import LimitRequestHeaders as LimitRequestHeaders -from gunicorn.http.errors import LimitRequestLine as LimitRequestLine -from gunicorn.http.wsgi import Response as Response -from gunicorn.http.wsgi import default_environ as default_environ -from gunicorn.reloader import reloader_engines as reloader_engines -from gunicorn.sock import TCPSocket as TCPSocket -from gunicorn.workers.workertmp import WorkerTmp as WorkerTmp +from gunicorn import util +from gunicorn.app.wsgiapp import WSGIApplication +from gunicorn.config import Config +from gunicorn.glogging import Logger +from gunicorn.http.errors import ( + ForbiddenProxyRequest, + InvalidHeader, + InvalidHeaderName, + InvalidHTTPVersion, + InvalidProxyLine, + InvalidRequestLine, + InvalidRequestMethod, + InvalidSchemeHeaders, + LimitRequestHeaders, + LimitRequestLine, +) +from gunicorn.http.wsgi import Response, default_environ +from gunicorn.reloader import reloader_engines +from gunicorn.sock import TCPSocket +from gunicorn.workers.workertmp import WorkerTmp +peer_addr: TypeAlias = tuple[str, int] | str Pipe: Incomplete class Worker: @@ -59,9 +64,9 @@ class Worker: wsgi: Incomplete def load_wsgi(self) -> None: ... def init_signals(self) -> None: ... - def handle_usr1(self, sig, frame) -> None: ... - def handle_exit(self, sig, frame) -> None: ... - def handle_quit(self, sig, frame) -> None: ... - def handle_abort(self, sig, frame) -> None: ... - def handle_error(self, req, client, addr, exc) -> None: ... - def handle_winch(self, sig, fname) -> None: ... + def handle_usr1(self, sig: Signals, frame: FrameType | None) -> None: ... + def handle_exit(self, sig: Signals, frame: FrameType | None) -> None: ... + def handle_quit(self, sig: Signals, frame: FrameType | None) -> None: ... + def handle_abort(self, sig: Signals, frame: FrameType | None) -> None: ... + def handle_error(self, req, client, addr: peer_addr, exc: BaseException) -> None: ... + def handle_winch(self, sig: Signals, fname: FrameType | None) -> None: ... diff --git a/gunicorn/workers/base_async.pyi b/gunicorn/workers/base_async.pyi index f9e2498c9..6dbb001da 100644 --- a/gunicorn/workers/base_async.pyi +++ b/gunicorn/workers/base_async.pyi @@ -1,17 +1,24 @@ +from contextlib import AbstractAsyncContextManager +from socket import socket +from typing import TypeAlias + from _typeshed import Incomplete -from gunicorn import http as http -from gunicorn import util as util -from gunicorn.http import wsgi as wsgi -from gunicorn.workers import base as base +from gunicorn.http.message import Message +from gunicorn.sock import BaseSocket +from gunicorn.workers.base import Worker + +ListenInfo: TypeAlias = tuple[str, int] | str | bytes +peer_addr: TypeAlias = tuple[str, int] | str -ALREADY_HANDLED: Incomplete +ALREADY_HANDLED: object -class AsyncWorker(base.Worker): - worker_connections: Incomplete - def __init__(self, *args, **kwargs) -> None: ... - def timeout_ctx(self) -> None: ... - def is_already_handled(self, respiter): ... - def handle(self, listener, client, addr) -> None: ... +class AsyncWorker(Worker): + worker_connections: int + def timeout_ctx(self) -> AbstractAsyncContextManager[None]: ... + def is_already_handled(self, respiter: Incomplete) -> bool: ... + def handle(self, listener: BaseSocket, client: socket, addr: peer_addr) -> None: ... alive: bool - def handle_request(self, listener_name, req, sock, addr): ... + def handle_request( + self, listener_name: ListenInfo, req: Message, sock: BaseSocket, addr: peer_addr + ) -> bool | None: ... diff --git a/gunicorn/workers/geventlet.py b/gunicorn/workers/geventlet.py index 087eb61ec..6e5c6e173 100644 --- a/gunicorn/workers/geventlet.py +++ b/gunicorn/workers/geventlet.py @@ -3,17 +3,11 @@ # See the NOTICE for more information. from functools import partial +import warnings import sys - -try: - import eventlet -except ImportError: - raise RuntimeError("eventlet worker requires eventlet 0.24.1 or higher") -else: - from packaging.version import parse as parse_version - if parse_version(eventlet.__version__) < parse_version('0.24.1'): - raise RuntimeError("eventlet worker requires eventlet 0.24.1 or higher") - +from gunicorn import util +util.get_dependency_version("eventlet") +import eventlet from eventlet import hubs, greenthread from eventlet.greenio import GreenSocket import eventlet.wsgi diff --git a/gunicorn/workers/geventlet.pyi b/gunicorn/workers/geventlet.pyi index 4896a722f..fd8862d6a 100644 --- a/gunicorn/workers/geventlet.pyi +++ b/gunicorn/workers/geventlet.pyi @@ -1,7 +1,17 @@ +from contextlib import AbstractAsyncContextManager +from signal import Signals +from socket import socket +from types import FrameType +from typing import TypeAlias + from _typeshed import Incomplete -from gunicorn.sock import ssl_wrap_socket as ssl_wrap_socket -from gunicorn.workers.base_async import AsyncWorker as AsyncWorker +from gunicorn.http.message import Message +from gunicorn.sock import BaseSocket +from gunicorn.workers.base import Worker +from gunicorn.workers.base_async import AsyncWorker + +peer_addr: TypeAlias = tuple[str, int] | str EVENTLET_WSGI_LOCAL: Incomplete EVENTLET_ALREADY_HANDLED: Incomplete @@ -10,10 +20,10 @@ def patch_sendfile() -> None: ... class EventletWorker(AsyncWorker): def patch(self) -> None: ... - def is_already_handled(self, respiter): ... + def is_already_handled(self, respiter: Incomplete) -> bool: ... def init_process(self) -> None: ... - def handle_quit(self, sig, frame) -> None: ... - def handle_usr1(self, sig, frame) -> None: ... - def timeout_ctx(self): ... - def handle(self, listener, client, addr) -> None: ... + def handle_quit(self, sig: Signals, frame: FrameType | None) -> None: ... + def handle_usr1(self, sig: Signals, frame: FrameType | None) -> None: ... + def timeout_ctx(self) -> AbstractAsyncContextManager[None]: ... + def handle(self, listener: BaseSocket, client: socket, addr: peer_addr) -> None: ... def run(self) -> None: ... diff --git a/gunicorn/workers/ggevent.py b/gunicorn/workers/ggevent.py index 42a6f21bb..485bfb65c 100644 --- a/gunicorn/workers/ggevent.py +++ b/gunicorn/workers/ggevent.py @@ -7,15 +7,10 @@ from datetime import datetime from functools import partial import time - -try: - import gevent -except ImportError: - raise RuntimeError("gevent worker requires gevent 1.4 or higher") -else: - from packaging.version import parse as parse_version - if parse_version(gevent.__version__) < parse_version('1.4'): - raise RuntimeError("gevent worker requires gevent 1.4 or higher") +import warnings +from gunicorn import util +gevent_version = util.get_dependency_version("gevent") +import gevent from gevent.pool import Pool from gevent.server import StreamServer @@ -26,7 +21,7 @@ from gunicorn.sock import ssl_context from gunicorn.workers.base_async import AsyncWorker -VERSION = "gevent/%s gunicorn/%s" % (gevent.__version__, gunicorn.__version__) +VERSION = "gevent/%s gunicorn/%s" % (gevent_version, gunicorn.__version__) class GeventWorker(AsyncWorker): diff --git a/gunicorn/workers/ggevent.pyi b/gunicorn/workers/ggevent.pyi index 945a98e43..7c9563efb 100644 --- a/gunicorn/workers/ggevent.pyi +++ b/gunicorn/workers/ggevent.pyi @@ -1,13 +1,21 @@ -from typing import Type +from contextlib import AbstractAsyncContextManager +from signal import Signals +from socket import socket +from types import FrameType +from typing import Type, TypeAlias from _typeshed import Incomplete +from _typeshed.wsgi import StartResponse, WSGIEnvironment from gevent import pywsgi -from gunicorn.http.wsgi import base_environ as base_environ -from gunicorn.sock import ssl_context as ssl_context -from gunicorn.workers.base_async import AsyncWorker as AsyncWorker +from gunicorn.http.message import Message +from gunicorn.sock import BaseSocket +from gunicorn.workers.base_async import AsyncWorker -VERSION: Incomplete +ListenInfo: TypeAlias = tuple[str, int] | str | bytes +peer_addr: TypeAlias = tuple[str, int] | str + +VERSION: str class GeventWorker(AsyncWorker): server_class: type[PyWSGIServer] @@ -15,23 +23,23 @@ class GeventWorker(AsyncWorker): sockets: Incomplete def patch(self) -> None: ... def notify(self) -> None: ... - def timeout_ctx(self): ... + def timeout_ctx(self) -> AbstractAsyncContextManager[None]: ... def run(self) -> None: ... - def handle(self, listener, client, addr) -> None: ... - def handle_request(self, listener_name, req, sock, addr) -> None: ... - def handle_quit(self, sig, frame) -> None: ... - def handle_usr1(self, sig, frame) -> None: ... + def handle(self, listener: BaseSocket, client: socket, addr: peer_addr) -> None: ... + def handle_request(self, listener_name: ListenInfo, req: Message, sock: BaseSocket, addr: peer_addr) -> None: ... + def handle_quit(self, sig: Signals, frame: FrameType | None) -> None: ... + def handle_usr1(self, sig: Signals, frame: FrameType | None) -> None: ... def init_process(self) -> None: ... class GeventResponse: - status: Incomplete - headers: Incomplete - sent: Incomplete - def __init__(self, status, headers, clength) -> None: ... + status: int + headers: list[tuple[str, str]] + sent: int + def __init__(self, status: int, headers: list[tuple[str, str]], clength: int) -> None: ... class PyWSGIHandler(pywsgi.WSGIHandler): def log_request(self) -> None: ... - def get_environ(self): ... + def get_environ(self) -> WSGIEnvironment: ... class PyWSGIServer(pywsgi.WSGIServer): ... diff --git a/gunicorn/workers/gthread.pyi b/gunicorn/workers/gthread.pyi index 205a91e44..d5c98d56e 100644 --- a/gunicorn/workers/gthread.pyi +++ b/gunicorn/workers/gthread.pyi @@ -1,10 +1,21 @@ +from collections import deque +from concurrent.futures import Future, ThreadPoolExecutor +from selectors import BaseSelector +from signal import Signals +from types import FrameType +from typing import Type, TypeAlias + from _typeshed import Incomplete +from typing_extensions import Self + +from gunicorn.config import Config +from gunicorn.glogging import Logger +from gunicorn.http.message import Message +from gunicorn.sock import BaseSocket + +from .base import Worker -from .. import http as http -from .. import sock as sock -from .. import util as util -from ..http import wsgi as wsgi -from . import base as base +PeerAddr: TypeAlias = tuple[str, int] | str | bytes class TConn: cfg: Incomplete @@ -14,31 +25,32 @@ class TConn: timeout: Incomplete parser: Incomplete initialized: bool - def __init__(self, cfg, sock, client, server) -> None: ... + def __init__(self, cfg: Config, sock: BaseSocket, client: PeerAddr, server: PeerAddr) -> None: ... def init(self) -> None: ... def set_timeout(self) -> None: ... def close(self) -> None: ... -class ThreadWorker(base.Worker): +class ThreadWorker(Worker): worker_connections: Incomplete - max_keepalived: Incomplete - tpool: Incomplete - poller: Incomplete - futures: Incomplete + max_keepalived: int + tpool: ThreadPoolExecutor + poller: BaseSelector + futures: deque + _keep: deque nr_conns: int - def __init__(self, *args, **kwargs) -> None: ... @classmethod - def check_config(cls, cfg, log) -> None: ... + def check_config(cls: type[Self], cfg: Config, log: Logger) -> None: ... def init_process(self) -> None: ... - def get_thread_pool(self): ... + def get_thread_pool(self) -> ThreadPoolExecutor: ... alive: bool - def handle_quit(self, sig, frame) -> None: ... - def enqueue_req(self, conn) -> None: ... - def accept(self, server, listener) -> None: ... - def on_client_socket_readable(self, conn, client) -> None: ... + def handle_quit(self, sig: Signals, frame: FrameType | None) -> None: ... + def _wrap_future(self, fs: Future, conn: BaseSocket) -> None: ... + def enqueue_req(self, conn: BaseSocket) -> None: ... + def accept(self, server: PeerAddr, listener: BaseSocket) -> None: ... + def on_client_socket_readable(self, conn: BaseSocket, client: PeerAddr) -> None: ... def murder_keepalived(self) -> None: ... - def is_parent_alive(self): ... + def is_parent_alive(self) -> bool: ... def run(self) -> None: ... - def finish_request(self, fs) -> None: ... - def handle(self, conn): ... - def handle_request(self, req, conn): ... + def finish_request(self, fs: Future) -> None: ... + def handle(self, conn: BaseSocket) -> tuple[bool, BaseSocket]: ... + def handle_request(self, req: Message, conn: BaseSocket) -> bool: ... diff --git a/gunicorn/workers/gtornado.py b/gunicorn/workers/gtornado.py index 34664151c..44e84fd57 100644 --- a/gunicorn/workers/gtornado.py +++ b/gunicorn/workers/gtornado.py @@ -5,11 +5,10 @@ import os import sys import warnings +from gunicorn import util +util.better_dependency_error("tornado") -try: - import tornado -except ImportError: - raise RuntimeError("You need tornado installed to use this worker.") +import tornado import tornado.web import tornado.httpserver from tornado.ioloop import IOLoop, PeriodicCallback diff --git a/gunicorn/workers/gtornado.pyi b/gunicorn/workers/gtornado.pyi index a8d9348d5..1772171e4 100644 --- a/gunicorn/workers/gtornado.pyi +++ b/gunicorn/workers/gtornado.pyi @@ -1,14 +1,14 @@ -from _typeshed import Incomplete +from signal import Signals +from types import FrameType -from gunicorn.sock import ssl_context as ssl_context -from gunicorn.workers.base import Worker as Worker +from _typeshed import Incomplete -TORNADO5: Incomplete +from gunicorn.workers.base import Worker class TornadoWorker(Worker): @classmethod def setup(cls) -> None: ... - def handle_exit(self, sig, frame) -> None: ... + def handle_exit(self, sig: Signals, frame: FrameType | None) -> None: ... alive: bool def handle_request(self) -> None: ... def watchdog(self) -> None: ... diff --git a/gunicorn/workers/sync.pyi b/gunicorn/workers/sync.pyi index a519537cf..b42f0f63d 100644 --- a/gunicorn/workers/sync.pyi +++ b/gunicorn/workers/sync.pyi @@ -1,19 +1,19 @@ -from gunicorn import http as http -from gunicorn import sock as sock -from gunicorn import util as util -from gunicorn.http import wsgi as wsgi -from gunicorn.sock import TCPSocket as TCPSocket -from gunicorn.workers import base as base +from _typeshed import Incomplete + +from gunicorn import util +from gunicorn.http import Message +from gunicorn.sock import TCPSocket +from gunicorn.workers import base class StopWaiting(Exception): ... class SyncWorker(base.Worker): - def accept(self, listener: TCPSocket): ... - def wait(self, timeout: float): ... + def accept(self, listener: TCPSocket) -> Incomplete: ... + def wait(self, timeout: float) -> Incomplete: ... def is_parent_alive(self) -> bool: ... - def run_for_one(self, timeout: float): ... - def run_for_multiple(self, timeout) -> None: ... + def run_for_one(self, timeout: float) -> Incomplete: ... + def run_for_multiple(self, timeout: float) -> None: ... def run(self) -> None: ... - def handle(self, listener, client, addr) -> None: ... + def handle(self, listener: Incomplete, client: Incomplete, addr: Incomplete) -> None: ... alive: bool - def handle_request(self, listener, req, client, addr) -> None: ... + def handle_request(self, listener: Incomplete, req: Message, client: Incomplete, addr: Incomplete) -> None: ... diff --git a/pyproject.toml b/pyproject.toml index d436bec0a..8666de4eb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -37,8 +37,8 @@ classifiers = [ ] requires-python = ">=3.7" dependencies = [ - 'importlib_metadata; python_version<"3.8"', - "packaging", + "importlib_metadata; python_version<'3.8'", + "packaging>=21.0", ] dynamic = ["version"] @@ -50,16 +50,60 @@ Documentation = "https://docs.gunicorn.org" Changelog = "https://docs.gunicorn.org/en/stable/news.html" [project.optional-dependencies] -gevent = ["gevent>=1.4.0"] -eventlet = ["eventlet>=0.24.1"] -tornado = ["tornado>=0.2"] +gevent = [ + # gevent 1.4.0 does not build for 3.11 (incompat Cython) + # 21.8.0 is what Ubuntu 22.04 shipped + # 22.10 is the last version to support python 3.7 + "gevent>=21.8.0; python_version<'3.11'", + "gevent>=22.8.0; python_version>='3.11'", +] +eventlet = [ + # eventlet 0.34.1 known to break older gunicorn + # just missing toplevel __version__, resolved in both gunicorn and eventlet + "eventlet>=0.24.1", +] +tornado = ["tornado>=5.0.0"] gthread = [] -setproctitle = ["setproctitle"] +setproctitle = ["setproctitle>=1.2.0"] +# extras use - not _ +lint-types = [ + "mypy>=1.7.1", +] +# linting on fixed versions, upgrading to matrix-incompat pylint release is OK +lint-code = [ + "pylint==3.0.2", + "pycodestyle", +] +lint-docs = [ + "restructuredtext_lint", + "pygments", +] +# note we are styling stubs to work on 3.10 - this may not always match code +style-types = [ + "isort>=5.13.0", + "pyupgrade==3.15.0", + "black==23.11.0", +] +# if only pip supported --version-selectio=min +testing-mindep = [ + "setproctitle==1.2.0", + "coverage[toml]==5.2.1", + "gevent==21.8.0; python_version<'3.11'", + "gevent==22.8.0; python_version>='3.11'", + "eventlet==0.24.1", + "tornado==5.0.0", + "packaging==21.0", + "coverage[toml]==5.2.1", + "pytest==7.2.0", +] testing = [ "gevent", "eventlet", - "coverage", - "pytest", + # coverage 5.2.1 will install tomli if needed to parse pyproject.toml + "coverage[toml]>=5.2.1", + # pytest 8.0 intends to drop Python 3.7 + # pytest 7.2.0 starts using Python 3.11 stdlib tomllib + "pytest>=7.2.0", ] [project.scripts] @@ -80,7 +124,7 @@ omit = [ [tool.pytest.ini_options] minversion = "7.2.0" # seconds until still-running tests are dumped -faulthandler_timeout = 5 +faulthandler_timeout = 20 norecursedirs = ["examples", "lib", "local", "src"] testpaths = ["tests/"] # --assert=plain stops rewriting asserts for better expression info @@ -111,8 +155,15 @@ version = {attr = "gunicorn.__version__"} warn_unused_configs = true no_implicit_reexport = true warn_unused_ignores = true +warn_unreachable = true strict_equality = true +[[tool.mypy.overrides]] +# setproctitle known to not be typed +# misnomer, we actualy want ignore_untyped_imports +module = "setproctitle" +ignore_missing_imports = true + [[tool.mypy.overrides]] # gevent known to not be typed module = "eventlet" @@ -127,11 +178,11 @@ ignore_missing_imports = true # pyupgrade does not read this. specify --py37-plus yourself [tool.black] -line-length = 119 +line-length = 88 target-version = ["py37", "py38", "py39", "py310", "py311", "py312"] [tool.isort] -line_length = 119 +line_length = 88 # changes stdlib category - all or auto (=current) can produce inconsistent results # py_version = "py310" profile = "black" diff --git a/tests/requests/invalid/version_03.http b/tests/requests/invalid/version_03.http new file mode 100644 index 000000000..419f637d6 --- /dev/null +++ b/tests/requests/invalid/version_03.http @@ -0,0 +1,2 @@ +GET /foo HTTP/1.ยน\r\n +\r\n diff --git a/tests/requests/invalid/version_03.py b/tests/requests/invalid/version_03.py new file mode 100644 index 000000000..760840b69 --- /dev/null +++ b/tests/requests/invalid/version_03.py @@ -0,0 +1,2 @@ +from gunicorn.http.errors import InvalidHTTPVersion +request = InvalidHTTPVersion diff --git a/tests/test_e2e.py b/tests/test_e2e.py new file mode 100644 index 000000000..9dac2046e --- /dev/null +++ b/tests/test_e2e.py @@ -0,0 +1,211 @@ +import fcntl +import os +import re +import secrets +import signal +import subprocess +import sys +import time +from pathlib import Path +from tempfile import TemporaryDirectory + +import pytest + +from gunicorn import config, glogging +from gunicorn.app.base import Application +from gunicorn.app.wsgiapp import WSGIApplication +from gunicorn.errors import ConfigError +from gunicorn.instrument import statsd +from gunicorn.workers.sync import SyncWorker + +# pytest does not like exception from threads +# use subprocess.Popen only for now +# from threading import Thread, Event + + +GRACEFUL_TIMEOUT = 2 +SERVER_PORT = 2048 + secrets.randbelow(1024 * 14) +# FIXME: should also test inherited socket +SERVER_BIND = "[::1]:%d" % SERVER_PORT +APP_BASENAME = "testsyntax" + +PY_OK = """ +import sys +def app(environ_, start_response): + # print("stdout from app", file=sys.stdout) + print("stderr from app", file=sys.stderr) + body = b"response body from app" + response_head = [ + ("Content-Type", "text/plain"), + ("Content-Length", "%d" % len(body)), + ] + start_response("200 OK", response_head) + return iter([body]) +""" + +PY_BAD = """ +def app(environ_, start_response_): + syntax_error: + raise RuntimeError("The SyntaxError should raise") +""" + + +class Server: + def __init__(self, *args, temp_path, **kwargs): + super().__init__(*args, **kwargs) + # self.launched = Event() + self.p = None + assert isinstance(temp_path, Path) + self.temp_path = temp_path + self.py_path = (temp_path / ("%s.py" % APP_BASENAME)).absolute() + + def write_bad(self): + with open(self.py_path, "w+") as f: + f.write(PY_BAD) + + def write_ok(self): + with open(self.py_path, "w+") as f: + f.write(PY_OK) + + def __enter__(self): + self.write_bad() + # self.write_ok() + self.run() + return self + + def __exit__(self, *exc): + if self.p is None: + return + self.p.send_signal(signal.SIGKILL) + stdout, stderr = self.p.communicate(timeout=2) + ret = self.p.returncode + assert stdout == b"" + assert ret == 0, (ret, stdout, stderr) + + def run(self): + self.p = subprocess.Popen( + [ + sys.executable, + "-m", + "gunicorn", + "--config=%s" % os.devnull, + "--log-level=debug", + "--worker-class=sync", + "--enable-stdio-inheritance", + "--access-logfile=-", + "--disable-redirect-access-to-syslog", + "--graceful-timeout=%d" % GRACEFUL_TIMEOUT, + "--on-fatal=quiet", + # "--reload", + "--reload-extra=%s" % self.py_path, + "--bind=%s" % SERVER_BIND, + "--reuse-port", + "%s:app" % APP_BASENAME, + ], + bufsize=0, # allow read to return short + cwd=self.temp_path, + shell=False, + stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + # creationflags=subprocess.CREATE_NEW_PROCESS_GROUP, + ) + + def noblock(fd): + old = fcntl.fcntl(fd, fcntl.F_GETFL) + fcntl.fcntl(fd, fcntl.F_SETFL, old | os.O_NONBLOCK) + + noblock(self.p.stdout.fileno()) + noblock(self.p.stderr.fileno()) + # self.launched.set() + + def _graceful_quit(self): + self.p.send_signal(signal.SIGTERM) + # self.p.kill() + stdout, stderr = self.p.communicate(timeout=2 * GRACEFUL_TIMEOUT) + assert stdout == b"" + ret = self.p.poll() + assert ret == 0, (ret, stdout, stderr) + return stderr.decode("utf-8", "surrogateescape") + + def _read_stdio(self, *, key, timeout_sec, wait_for_keyword): + # try: + # stdout, stderr = self.p.communicate(timeout=timeout) + # except subprocess.TimeoutExpired: + buf = ["", ""] + extra = 0 + for i in range(timeout_sec * 10): + for fd, file in enumerate([self.p.stdout, self.p.stderr]): + read = file.read(64 * 1024) + if read is not None: + buf[fd] += read.decode("utf-8", "surrogateescape") + if extra or wait_for_keyword in buf[key]: + extra += 1 + # wait a bit *after* seeing the keyword to increase chance of reading context + if extra > 3: + break + time.sleep(0.1) + other = abs(key - 1) + # assert buf[other] == "" + assert wait_for_keyword in buf[key] + return buf[key] + + +class Client: + def run(self): + import http.client + + conn = http.client.HTTPConnection(SERVER_BIND, timeout=1) + conn.request("GET", "/", headers={"Host": "localhost"}, body="GETBODY!") + return conn.getresponse() + + +def test_process_request_after_syntax_error(): + client = Client() + + with TemporaryDirectory(suffix="_temp_py") as tempdir_name: + with Server(temp_path=Path(tempdir_name)) as server: + OUT = 0 + ERR = 1 + + boot_log = server._read_stdio( + key=ERR, wait_for_keyword="Arbiter booted", timeout_sec=5 + ) + assert "SyntaxError: invalid syntax" in boot_log + assert ('%s.py", line 3' % APP_BASENAME) in boot_log + + # worker could not load, request will fail + response = client.run() + assert response.status == 500 + assert response.reason == "Internal Server Error" + body = response.read(64 * 1024).decode("utf-8", "surrogateescape") + assert "error" in body.lower() + + access_log = server._read_stdio( + key=OUT, wait_for_keyword='GET / HTTP/1.1" 500 ', timeout_sec=3 + ) + # trigger reloader + server.write_ok() + # os.utime(editable_file) + + reload_log = server._read_stdio( + key=ERR, wait_for_keyword="reloading", timeout_sec=3 + ) + assert ("%s.py modified" % APP_BASENAME) in reload_log + assert "Booting worker" in reload_log + + # worker did boot now, request should work + response = client.run() + assert response.status == 200 + assert response.reason == "OK" + body = response.read(64 * 1024).decode("utf-8", "surrogateescape") + assert "response body from app" == body + + debug_log = server._read_stdio( + key=ERR, wait_for_keyword="stderr from app", timeout_sec=4 + ) + + shutdown_log = server._graceful_quit() + assert "Handling signal: term" in shutdown_log + assert "Worker exiting " in shutdown_log + assert "Shutting down: Master" in shutdown_log diff --git a/tests/test_http.py b/tests/test_http.py index a9333c5cf..f2d7fb3b2 100644 --- a/tests/test_http.py +++ b/tests/test_http.py @@ -11,6 +11,14 @@ from gunicorn.http.message import TOKEN_RE +def test_header_processing(): + assert util.is_hoppish("tRaNsFer-encoding") + assert not util.is_hoppish("host") + # the helper is working on header names, not environ + assert not util.is_hoppish(" upgrade") + assert not util.is_hoppish("TRANSFER_ENCODING") + + def test_method_pattern(): assert TOKEN_RE.fullmatch("GET") assert TOKEN_RE.fullmatch("MKCALENDAR") @@ -19,6 +27,10 @@ def test_method_pattern(): RFC9110_5_6_2_TOKEN_DELIM = r'"(),/:;<=>?@[\]{}' for bad_char in RFC9110_5_6_2_TOKEN_DELIM: assert not TOKEN_RE.match(bad_char) + assert not TOKEN_RE.match("\x00") + assert not TOKEN_RE.match("\xff") + assert not TOKEN_RE.match("\t") + assert not TOKEN_RE.match(" ") def assert_readline(payload, size, expected): diff --git a/tests/treq.py b/tests/treq.py index cbd9b50c4..1d889bf4e 100644 --- a/tests/treq.py +++ b/tests/treq.py @@ -19,7 +19,7 @@ # a) stdlib secrets module # will help expose more bad tests # may flip-flip between same test working and failing -# b) stdlib random (library name is a misnomer) with fixed seed +# b) stdlib random (misnomer) with fixed seed # will produce repeatably results # may miss bad tests aligning with some properties of the generator # c) stdlib random with arbitrary seed diff --git a/tox.ini b/tox.ini index 1e336dd45..8d6d1f093 100644 --- a/tox.ini +++ b/tox.ini @@ -1,49 +1,69 @@ [tox] -envlist = py{37,38,39,310,311,312,py3}, lint, docs-lint, pycodestyle, run-entrypoint, run-module -skipsdist = false -; Can't set skipsdist and use_develop in tox v4 to true due to https://github.com/tox-dev/tox/issues/2730 +envlist = + py{37,38,39,310,311,312,py3} + lint + docs-lint + pycodestyle + run-entrypoint + run-module [testenv] -use_develop = true +# tests must work either editable or installed, lets try *both* +package = editable +extras = testing +commands = + python -m coverage run -m pytest {posargs} + python -m coverage xml + +[testenv:py{37,38,39,310,311,312,py3}-mindep] +# tests must work either editable or installed, lets try *both* +package = wheel +extras = testing-mindep +# version checks, so we do not stop earyl for least important failure condition commands = python -m coverage run -m pytest {posargs} python -m coverage xml -deps = - -rrequirements_test.txt + python -m gunicorn.packaging_support --check [testenv:run-entrypoint] +# entry points need install +package = wheel # entry point: console script (provided by setuptools from pyproject.toml) commands = python -c 'import subprocess; cmd_out = subprocess.check_output(["gunicorn", "--version"])[:79].decode("utf-8", errors="replace"); print(cmd_out); assert cmd_out.startswith("gunicorn ")' [testenv:run-module] +# entry points need install +package = wheel # runpy (provided by module.__main__) commands = python -c 'import sys,subprocess; cmd_out = subprocess.check_output([sys.executable, "-m", "gunicorn", "--version"])[:79].decode("utf-8", errors="replace"); print(cmd_out); assert cmd_out.startswith("gunicorn ")' [testenv:format] +# type linting as consumers use it +package = sdist +extras = style-types allowlist_externals = git bash commands = - # note we are styling stubs to work on 3.10 - this may not always match code # isort --check-only will exit 0 on no change, 1 on change bash -c 'git ls-files -z "**.pyi" | xargs --null python -m isort --py=310 --check-only' # black --check will exit 0 on no change, 1 on change, and 123 on error bash -c 'git ls-files -z "**.pyi" | xargs --null python -m black --target-version=py310 --check' # pyupgrade has no readonly option - will set exit code nonzero if writing bash -c 'git ls-files -z "**.pyi" | xargs --null python -m pyupgrade --py310-plus' -deps = - isort>=5.13.0 - pyupgrade==3.15.0 - black==23.11.0 [testenv:mypy] -commands = python -m mypy -- gunicorn +package = wheel +# enhanced by installing extras # mypy wants to look at all imported modules (or their separately distributed stubs) -deps = - mypy>=1.7.1 - pytest>=7.2.0 +extras = + lint-types + testing +commands = python -m mypy -- gunicorn [testenv:lint] +no_package = true +extras = style-code commands = pylint -j0 \ --max-line-length=120 \ @@ -61,18 +81,13 @@ commands = tests/test_systemd.py \ tests/test_util.py \ tests/test_valid_requests.py -# linting on fixed versions, upgrading to matrix-incompat pylint release is OK -deps = - pylint==3.0.2 [testenv:docs-lint] allowlist_externals = rst-lint bash grep -deps = - restructuredtext_lint - pygments +extras = lint-docs commands = rst-lint README.rst docs/README.rst bash -c "(set -o pipefail; rst-lint --encoding utf-8 docs/source/*.rst | grep -v 'Unknown interpreted text role\|Unknown directive type'); test $? == 1" @@ -80,9 +95,9 @@ commands = [testenv:pycodestyle] commands = pycodestyle gunicorn -deps = - pycodestyle +extras = style-code +# yes, pycodestyle reads from here, or setup.cfg (but as of 2023, not pyproject.toml) [pycodestyle] max-line-length = 120 ignore = E129,W503,W504,W606