From 08fe78a835acb5a6f1392ec3b8cec7bb2a274a5c Mon Sep 17 00:00:00 2001 From: Dhairya Jangir <183471878+dhairyajangir@users.noreply.github.com> Date: Thu, 12 Nov 2026 00:59:28 +0530 Subject: [PATCH] Handle non-UTF8 response header values --- adafruit_requests.py | 19 ++++++++++- tests/header_test.py | 81 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 99 insertions(+), 1 deletion(-) diff --git a/adafruit_requests.py b/adafruit_requests.py index 3e197ca..868a1ef 100644 --- a/adafruit_requests.py +++ b/adafruit_requests.py @@ -257,6 +257,18 @@ def close(self) -> None: self.socket = None + @staticmethod + def _decode_opaque_header_value(value: bytes) -> str: + """Map opaque octets without requiring a Latin-1 codec.""" + encoded = bytearray() + for octet in value: + if octet < 0x80: + encoded.append(octet) + else: + encoded.append(0xC0 | (octet >> 6)) + encoded.append(0x80 | (octet & 0x3F)) + return str(encoded, "utf-8") + def _parse_headers(self) -> None: """ Parses the header portion of an HTTP request/response from the socket. @@ -271,7 +283,12 @@ def _parse_headers(self) -> None: if title and content: # enforce that all headers are lowercase title = str(title, "utf-8").lower() - content = str(content, "utf-8") + try: + content = str(content, "utf-8") + except UnicodeError: + if title in ("content-length", "transfer-encoding"): + raise + content = self._decode_opaque_header_value(content) if title == "content-length": self._remaining = int(content) if title == "transfer-encoding": diff --git a/tests/header_test.py b/tests/header_test.py index bbcc43c..a14f9c3 100644 --- a/tests/header_test.py +++ b/tests/header_test.py @@ -7,6 +7,87 @@ import mocket import pytest +import adafruit_requests + + +@pytest.mark.parametrize( + ("value", "expected"), + ( + (b"Motor", "Motor"), + (b"M\xc3\xb6tor", "M\u00f6tor"), + (b"\xe2\x82\xac", "\u20ac"), + (b"M\xf6tor", "M\u00f6tor"), + (b"\xff", "\u00ff"), + (b"\xc3\xb6\xff", "\u00c3\u00b6\u00ff"), + (bytes(range(0x80, 0x100)), bytes(range(0x80, 0x100)).decode("latin-1")), + (b"M\xf6tor:west", "M\u00f6tor:west"), + ), +) +def test_response_header_value_octets(pool, value, expected): + sock = mocket.Mocket( + b"HTTP/1.1 200 OK\r\nX-Sensor-Name: " + value + b"\r\nContent-Length: 2\r\n\r\nok" + ) + pool.socket.return_value = sock + with adafruit_requests.Session(pool).get("http://" + mocket.MOCK_HOST_1 + "/get") as response: + assert response.headers["x-sensor-name"] == expected + assert response.headers["content-length"] == "2" + assert response.content == b"ok" + + +def test_response_opaque_header_with_chunked_body(pool): + sock = mocket.Mocket( + b"HTTP/1.1 200 OK\r\nX-Sensor-Name: M\xf6tor\r\n" + b"Transfer-Encoding: chunked\r\n\r\n2\r\nhe\r\n3\r\nllo\r\n0\r\n\r\n" + ) + pool.socket.return_value = sock + with adafruit_requests.Session(pool).get("http://" + mocket.MOCK_HOST_1 + "/get") as response: + assert response.headers["x-sensor-name"] == "M\u00f6tor" + assert response.content == b"hello" + + +def test_response_opaque_duplicate_cookies(pool): + sock = mocket.Mocket( + b"HTTP/1.1 200 OK\r\nSet-Cookie: a=M\xf6tor\r\nSet-Cookie: b=2\r\n" + b"Content-Length: 2\r\n\r\nok" + ) + pool.socket.return_value = sock + with adafruit_requests.Session(pool).get("http://" + mocket.MOCK_HOST_1 + "/get") as response: + assert response.headers["set-cookie"] == "a=M\u00f6tor, b=2" + assert response.content == b"ok" + + +def test_response_opaque_header_preserves_utf8_body(pool): + sock = mocket.Mocket( + b"HTTP/1.1 200 OK\r\nX-Sensor-Name: \xff\r\nContent-Length: 5\r\n\r\ncaf\xc3\xa9" + ) + pool.socket.return_value = sock + with adafruit_requests.Session(pool).get("http://" + mocket.MOCK_HOST_1 + "/get") as response: + assert response.headers["x-sensor-name"] == "\u00ff" + assert response.text == "caf\u00e9" + + +def test_response_invalid_utf8_header_name_is_still_rejected(pool): + sock = mocket.Mocket(b"HTTP/1.1 200 OK\r\nX-\xff: value\r\nContent-Length: 2\r\n\r\nok") + pool.socket.return_value = sock + with pytest.raises(UnicodeError): + adafruit_requests.Session(pool).get("http://" + mocket.MOCK_HOST_1 + "/get") + + +@pytest.mark.parametrize( + "header", + ( + b"Content-Length: \xa03", + b"Content-Length: 3\xff", + b"Transfer-Encoding: \xa0chunked", + b"Transfer-Encoding: chunked\xff", + ), +) +def test_response_non_utf8_framing_is_still_rejected(pool, header): + sock = mocket.Mocket(b"HTTP/1.1 200 OK\r\n" + header + b"\r\n\r\nabc") + pool.socket.return_value = sock + with pytest.raises(UnicodeError): + adafruit_requests.Session(pool).get("http://" + mocket.MOCK_HOST_1 + "/get") + def test_check_headers_not_dict(requests): with pytest.raises(TypeError) as context: