From b1ff72ffbe993d6326ec3e34a4f562c9df6fa3e9 Mon Sep 17 00:00:00 2001 From: Vladimir Smitka Date: Sun, 18 Oct 2026 17:44:03 +0000 Subject: [PATCH] Add Response.raw for reading the body into a buffer Response.raw is a file-like object with read() and readinto(), as in CPython's requests. With stream=True, raw.readinto() reads a large body into a buffer that is reused, so it needs no further memory. json() already used this object internally. raw.read(size) now reads through the response like readinto() does. It read the socket directly, so it lost the start of the body that was read along with the headers, and ignored Content-Length and chunked encoding. The socket docstring now points to raw for the same reason. On a PicoPad (RP2040), 10 downloads of a 43 KB body into a 1 KB buffer left free memory and the largest free block unchanged. Reading the same body with .content fails with MemoryError on the first download. --- adafruit_requests.py | 21 ++++-- .../requests_wifi_stream_into_buffer.py | 32 +++++++++ tests/raw_test.py | 70 +++++++++++++++++++ 3 files changed, 118 insertions(+), 5 deletions(-) create mode 100644 examples/wifi/expanded/requests_wifi_stream_into_buffer.py create mode 100644 tests/raw_test.py diff --git a/adafruit_requests.py b/adafruit_requests.py index c21f69c..2cd7c7f 100644 --- a/adafruit_requests.py +++ b/adafruit_requests.py @@ -70,7 +70,9 @@ def read(self, size: int = -1) -> bytes: """ if size == -1: return self._response.content - return self._response.socket.recv(size) + buf = bytearray(size) + read = self._response._readinto(buf) + return bytes(memoryview(buf)[:read]) def readinto(self, buf: bytearray) -> int: """Read as much as available into buf or until it is full. Returns the number of bytes read @@ -99,6 +101,9 @@ class Response: Methods and properties that access response headers may be accessed. + The start of the body may already have been read along with the headers, so read the body + with `raw` rather than from the socket. + It is still necessary to ``close`` the response object for correct management of sockets, including doing so implicitly via ``with requests.get(...) as response``.""" @@ -325,6 +330,15 @@ def text(self) -> str: self._cached = str(self.content, self.encoding) return self._cached + @property + def raw(self) -> _RawResponse: + """The body as a file-like object with ``read()`` and ``readinto()``, as in CPython's + requests. With ``stream=True``, ``readinto()`` reads the body into a buffer you can reuse, + so a large response needs no memory beyond that buffer.""" + if not self._raw: + self._raw = _RawResponse(self) + return self._raw + def json(self) -> Any: """The HTTP content, parsed into a json dictionary""" # The cached JSON will be a list or dictionary. @@ -332,12 +346,9 @@ def json(self) -> Any: if isinstance(self._cached, (list, dict)): return self._cached raise RuntimeError("Cannot access json after getting text or content") - if not self._raw: - self._raw = _RawResponse(self) - self._validate_not_gzip() - obj = json_module.load(self._raw) + obj = json_module.load(self.raw) if not self._cached: self._cached = obj diff --git a/examples/wifi/expanded/requests_wifi_stream_into_buffer.py b/examples/wifi/expanded/requests_wifi_stream_into_buffer.py new file mode 100644 index 0000000..dce2464 --- /dev/null +++ b/examples/wifi/expanded/requests_wifi_stream_into_buffer.py @@ -0,0 +1,32 @@ +# SPDX-FileCopyrightText: 2026 Vladimir Smitka +# SPDX-License-Identifier: MIT +"""Read a large response into a reused buffer, so it needs no memory beyond the buffer""" + +import gc +import os + +import adafruit_connection_manager +import wifi + +import adafruit_requests + +URL = "https://httpbin.org/bytes/40000" + +wifi.radio.connect(os.getenv("CIRCUITPY_WIFI_SSID"), os.getenv("CIRCUITPY_WIFI_PASSWORD")) +pool = adafruit_connection_manager.get_radio_socketpool(wifi.radio) +ssl_context = adafruit_connection_manager.get_radio_ssl_context(wifi.radio) +requests = adafruit_requests.Session(pool, ssl_context) + +buffer = bytearray(1024) +for attempt in range(5): + gc.collect() + total = 0 + with requests.get(URL, stream=True) as response: + while True: + read = response.raw.readinto(buffer) + if read == 0: + break + # Use buffer[:read] here, for example write it to a file or a Bitmap. + total += read + gc.collect() + print(f"Download {attempt}: {total} bytes, {gc.mem_free()} bytes free") diff --git a/tests/raw_test.py b/tests/raw_test.py new file mode 100644 index 0000000..201f79e --- /dev/null +++ b/tests/raw_test.py @@ -0,0 +1,70 @@ +# SPDX-FileCopyrightText: 2026 Vladimir Smitka +# +# SPDX-License-Identifier: Unlicense + +"""Raw Tests""" + +import mocket + +import adafruit_requests + +BODY = bytes(range(256)) * 4 +LENGTH_RESPONSE = b"HTTP/1.1 200 OK\r\nContent-Length: %d\r\n\r\n" % len(BODY) + BODY + + +def _chunked(body, split): + out = b"" + for i in range(0, len(body), split): + piece = body[i : i + split] + out += hex(len(piece))[2:].encode() + b"\r\n" + piece + b"\r\n" + return b"HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n" + out + b"0\r\n\r\n" + + +def _get(response, **kw): + pool = mocket.MocketPool() + pool.socket.return_value = mocket.Mocket(response) + requests_session = adafruit_requests.Session(pool) + return requests_session.get("http://" + mocket.MOCK_ENDPOINT_1, **kw) + + +def _read_all(raw, size): + buf = bytearray(size) + out = b"" + while True: + read = raw.readinto(buf) + if read == 0: + return out + out += buf[:read] + + +def test_readinto_content_length(): + with _get(LENGTH_RESPONSE, stream=True) as response: + assert _read_all(response.raw, 100) == BODY + + +def test_readinto_chunked(): + with _get(_chunked(BODY, 300), stream=True) as response: + assert _read_all(response.raw, 64) == BODY + + +def test_read_includes_bytes_read_with_the_headers(): + with _get(LENGTH_RESPONSE, stream=True) as response: + # The headers are parsed from 32-byte reads, so the start of the body is buffered. + buffered = response._received_length + assert buffered > 0 + first = response.raw.read(100) + assert first == BODY[:buffered] + assert first + _read_all(response.raw, 100) == BODY + + +def test_read_stops_at_content_length(): + with _get(LENGTH_RESPONSE + b"trailing", stream=True) as response: + assert _read_all(response.raw, 100) == BODY + assert response.raw.read(10) == b"" + + +def test_json_uses_raw(): + body = b'{"a": 1}' + response = b"HTTP/1.1 200 OK\r\nContent-Length: %d\r\n\r\n" % len(body) + body + with _get(response) as response: + assert response.json() == {"a": 1}