diff --git a/adafruit_requests.py b/adafruit_requests.py index c21f69c..2cd7c7f 100644 --- a/adafruit_requests.py +++ b/adafruit_requests.py @@ -70,7 +70,9 @@ def read(self, size: int = -1) -> bytes: """ if size == -1: return self._response.content - return self._response.socket.recv(size) + buf = bytearray(size) + read = self._response._readinto(buf) + return bytes(memoryview(buf)[:read]) def readinto(self, buf: bytearray) -> int: """Read as much as available into buf or until it is full. Returns the number of bytes read @@ -99,6 +101,9 @@ class Response: Methods and properties that access response headers may be accessed. + The start of the body may already have been read along with the headers, so read the body + with `raw` rather than from the socket. + It is still necessary to ``close`` the response object for correct management of sockets, including doing so implicitly via ``with requests.get(...) as response``.""" @@ -325,6 +330,15 @@ def text(self) -> str: self._cached = str(self.content, self.encoding) return self._cached + @property + def raw(self) -> _RawResponse: + """The body as a file-like object with ``read()`` and ``readinto()``, as in CPython's + requests. With ``stream=True``, ``readinto()`` reads the body into a buffer you can reuse, + so a large response needs no memory beyond that buffer.""" + if not self._raw: + self._raw = _RawResponse(self) + return self._raw + def json(self) -> Any: """The HTTP content, parsed into a json dictionary""" # The cached JSON will be a list or dictionary. @@ -332,12 +346,9 @@ def json(self) -> Any: if isinstance(self._cached, (list, dict)): return self._cached raise RuntimeError("Cannot access json after getting text or content") - if not self._raw: - self._raw = _RawResponse(self) - self._validate_not_gzip() - obj = json_module.load(self._raw) + obj = json_module.load(self.raw) if not self._cached: self._cached = obj diff --git a/examples/wifi/expanded/requests_wifi_stream_into_buffer.py b/examples/wifi/expanded/requests_wifi_stream_into_buffer.py new file mode 100644 index 0000000..dce2464 --- /dev/null +++ b/examples/wifi/expanded/requests_wifi_stream_into_buffer.py @@ -0,0 +1,32 @@ +# SPDX-FileCopyrightText: 2026 Vladimir Smitka +# SPDX-License-Identifier: MIT +"""Read a large response into a reused buffer, so it needs no memory beyond the buffer""" + +import gc +import os + +import adafruit_connection_manager +import wifi + +import adafruit_requests + +URL = "https://httpbin.org/bytes/40000" + +wifi.radio.connect(os.getenv("CIRCUITPY_WIFI_SSID"), os.getenv("CIRCUITPY_WIFI_PASSWORD")) +pool = adafruit_connection_manager.get_radio_socketpool(wifi.radio) +ssl_context = adafruit_connection_manager.get_radio_ssl_context(wifi.radio) +requests = adafruit_requests.Session(pool, ssl_context) + +buffer = bytearray(1024) +for attempt in range(5): + gc.collect() + total = 0 + with requests.get(URL, stream=True) as response: + while True: + read = response.raw.readinto(buffer) + if read == 0: + break + # Use buffer[:read] here, for example write it to a file or a Bitmap. + total += read + gc.collect() + print(f"Download {attempt}: {total} bytes, {gc.mem_free()} bytes free") diff --git a/tests/raw_test.py b/tests/raw_test.py new file mode 100644 index 0000000..201f79e --- /dev/null +++ b/tests/raw_test.py @@ -0,0 +1,70 @@ +# SPDX-FileCopyrightText: 2026 Vladimir Smitka +# +# SPDX-License-Identifier: Unlicense + +"""Raw Tests""" + +import mocket + +import adafruit_requests + +BODY = bytes(range(256)) * 4 +LENGTH_RESPONSE = b"HTTP/1.1 200 OK\r\nContent-Length: %d\r\n\r\n" % len(BODY) + BODY + + +def _chunked(body, split): + out = b"" + for i in range(0, len(body), split): + piece = body[i : i + split] + out += hex(len(piece))[2:].encode() + b"\r\n" + piece + b"\r\n" + return b"HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n" + out + b"0\r\n\r\n" + + +def _get(response, **kw): + pool = mocket.MocketPool() + pool.socket.return_value = mocket.Mocket(response) + requests_session = adafruit_requests.Session(pool) + return requests_session.get("http://" + mocket.MOCK_ENDPOINT_1, **kw) + + +def _read_all(raw, size): + buf = bytearray(size) + out = b"" + while True: + read = raw.readinto(buf) + if read == 0: + return out + out += buf[:read] + + +def test_readinto_content_length(): + with _get(LENGTH_RESPONSE, stream=True) as response: + assert _read_all(response.raw, 100) == BODY + + +def test_readinto_chunked(): + with _get(_chunked(BODY, 300), stream=True) as response: + assert _read_all(response.raw, 64) == BODY + + +def test_read_includes_bytes_read_with_the_headers(): + with _get(LENGTH_RESPONSE, stream=True) as response: + # The headers are parsed from 32-byte reads, so the start of the body is buffered. + buffered = response._received_length + assert buffered > 0 + first = response.raw.read(100) + assert first == BODY[:buffered] + assert first + _read_all(response.raw, 100) == BODY + + +def test_read_stops_at_content_length(): + with _get(LENGTH_RESPONSE + b"trailing", stream=True) as response: + assert _read_all(response.raw, 100) == BODY + assert response.raw.read(10) == b"" + + +def test_json_uses_raw(): + body = b'{"a": 1}' + response = b"HTTP/1.1 200 OK\r\nContent-Length: %d\r\n\r\n" % len(body) + body + with _get(response) as response: + assert response.json() == {"a": 1}