Merge remote-tracking branch 'benoitc/pr/3068' into unmaintained

gunicorn/config.py

@@ -209,6 +209,14 @@ def sendfile(self):
 
         return True
 
+    @property
+    def buf_read_size(self):
+        buf_read_size = self.settings['buf_read_size'].get()
+        if buf_read_size is None:
+            return 1024
+
+        return int(buf_read_size)
+
     @property
     def reuse_port(self):
         return self.settings['reuse_port'].get()
@@ -2143,6 +2151,16 @@ class CertReqs(Setting):
     """
 
 
+class BufReadSize(PosIntSetting):
+    name = "buf_read_size"
+    section = "Server Mechanics"
+    cli = ["--buf-read-size"]
+    default = 1024
+    desc = """\
+    Buffer read size from request data
+    """
+
+
 class CACerts(Setting):
     name = "ca_certs"
     section = "SSL"

gunicorn/config.pyi

@@ -92,6 +92,8 @@ class Config:
     @property
     def sendfile(self) -> bool: ...
     @property
+    def buf_read_size(self) -> int: ...
+    @property
     def reuse_port(self) -> bool: ...
     @property
     def paste_global_conf(self) -> dict[str, Incomplete]: ...
@@ -375,6 +377,7 @@ class KeyFile(Setting): ...
 class CertFile(Setting): ...
 class SSLVersion(Setting): ...
 class CertReqs(Setting): ...
+class BufReadSize(PosIntSetting): ...
 class CACerts(Setting): ...
 class SuppressRaggedEOFs(BoolSetting): ...
 class DoHandshakeOnConnect(BoolSetting): ...

gunicorn/http/body.py

@@ -177,7 +177,8 @@ def read(self, size):
 
 
 class Body(object):
-    def __init__(self, reader):
+    def __init__(self, cfg, reader):
+        self.cfg = cfg
         self.reader = reader
         self.buf = io.BytesIO()
 
@@ -214,7 +215,7 @@ def read(self, size=None):
             return ret
 
         while size > self.buf.tell():
-            data = self.reader.read(1024)
+            data = self.reader.read(self.cfg.buf_read_size)
             if not data:
                 break
             self.buf.write(data)

gunicorn/http/body.pyi

@@ -1,5 +1,6 @@
 from collections.abc import Generator, Iterator
 from io import BytesIO
+from typing import TYPE_CHECKING
 
 from _typeshed import Incomplete
 from typing_extensions import Protocol, Self
@@ -8,6 +9,9 @@ from gunicorn.http.errors import ChunkMissingTerminator, InvalidChunkSize, NoMor
 from gunicorn.http.message import Message
 from gunicorn.http.unreader import Unreader
 
+if TYPE_CHECKING:
+    from gunicorn.config import Config
+
 class _Read(Protocol):
     def read(self, size: int) -> bytes: ...
 
@@ -40,7 +44,7 @@ class EOFReader:
 class Body:
     reader: _Read
     buf: BytesIO
-    def __init__(self, reader: _Read) -> None: ...
+    def __init__(self, cfg: Config, reader: _Read) -> None: ...
     def __iter__(self) -> Self: ...
     def __next__(self) -> Iterator[bytes]: ...
     next = __next__

gunicorn/http/message.py

@@ -212,7 +212,7 @@ def set_body_reader(self):
                 # either processing or rejecting is permitted in RFC 9112 Section 6.1
                 if not self.cfg.tolerate_dangerous_framing:
                     raise InvalidHeader("CONTENT-LENGTH", req=self)
-            self.body = Body(ChunkedReader(self, self.unreader))
+            self.body = Body(self.cfg, ChunkedReader(self, self.unreader))
         elif content_length is not None:
             try:
                 if str(content_length).isnumeric():
@@ -225,9 +225,9 @@ def set_body_reader(self):
             if content_length < 0:
                 raise InvalidHeader("CONTENT-LENGTH", req=self)
 
-            self.body = Body(LengthReader(self.unreader, content_length))
+            self.body = Body(self.cfg, LengthReader(self.unreader, content_length))
         else:
-            self.body = Body(EOFReader(self.unreader))
+            self.body = Body(self.cfg, EOFReader(self.unreader))
 
     def should_close(self):
         if self.must_close:
@@ -452,4 +452,4 @@ def parse_request_line(self, line_bytes):
     def set_body_reader(self):
         super().set_body_reader()
         if isinstance(self.body.reader, EOFReader):
-            self.body = Body(LengthReader(self.unreader, 0))
+            self.body = Body(self.cfg, LengthReader(self.unreader, 0))

tests/test_config.py

@@ -534,6 +534,16 @@ def test_bind_fd():
     assert app.cfg.bind == ["fd://42"]
 
 
+def test_buf_read_size():
+    with AltArgs(["prog_name"]):
+        app = NoConfigApp()
+    assert app.cfg.buf_read_size == 1024
+
+    with AltArgs(["prog_name", "--buf-read-size", "1048576"]):
+        app = NoConfigApp()
+    assert app.cfg.buf_read_size == 1048576
+
+
 def test_repr():
     c = config.Config()
     c.set("workers", 5)

tests/test_e2e.py

@@ -182,6 +182,7 @@ def __init__(
             "--log-level=debug",
             "--worker-class=%s" % worker_class,
             "--workers=%d" % WORKER_COUNT,
+            "--buf-read-size=77",
             "--enable-stdio-inheritance",
             "--access-logfile=-",
             "--disable-redirect-access-to-syslog",

tests/test_http.py

@@ -5,7 +5,7 @@
 import pytest
 from unittest import mock
 
-from gunicorn import util
+from gunicorn import config, util
 from gunicorn.http.body import Body, LengthReader, EOFReader
 from gunicorn.http.wsgi import Response
 from gunicorn.http.unreader import Unreader, IterUnreader, SocketUnreader
@@ -24,7 +24,7 @@ def test_method_pattern():
 
 
 def assert_readline(payload, size, expected):
-    body = Body(io.BytesIO(payload))
+    body = Body(config.Config(), io.BytesIO(payload))
     assert body.readline(size) == expected
 
 
@@ -39,29 +39,29 @@ def test_readline_zero_size():
 
 
 def test_readline_new_line_before_size():
-    body = Body(io.BytesIO(b"abc\ndef"))
+    body = Body(config.Config(), io.BytesIO(b"abc\ndef"))
     assert body.readline(4) == b"abc\n"
     assert body.readline() == b"def"
 
 
 def test_readline_new_line_after_size():
-    body = Body(io.BytesIO(b"abc\ndef"))
+    body = Body(config.Config(), io.BytesIO(b"abc\ndef"))
     assert body.readline(2) == b"ab"
     assert body.readline() == b"c\n"
 
 
 def test_readline_no_new_line():
-    body = Body(io.BytesIO(b"abcdef"))
+    body = Body(config.Config(), io.BytesIO(b"abcdef"))
     assert body.readline() == b"abcdef"
-    body = Body(io.BytesIO(b"abcdef"))
+    body = Body(config.Config(), io.BytesIO(b"abcdef"))
     assert body.readline(2) == b"ab"
     assert body.readline(2) == b"cd"
     assert body.readline(2) == b"ef"
 
 
 def test_readline_buffer_loaded():
     reader = io.BytesIO(b"abc\ndef")
-    body = Body(reader)
+    body = Body(config.Config(), reader)
     body.read(1) # load internal buffer
     reader.write(b"g\nhi")
     reader.seek(7)
@@ -71,7 +71,7 @@ def test_readline_buffer_loaded():
 
 
 def test_readline_buffer_loaded_with_size():
-    body = Body(io.BytesIO(b"abc\ndef"))
+    body = Body(config.Config(), io.BytesIO(b"abc\ndef"))
     body.read(1)  # load internal buffer
     assert body.readline(2) == b"bc"
     assert body.readline(2) == b"\n"