Add, test support for chunk extensions

The chunk line is read up to the terminating "\r\n", then any extensions are
ignored by only using the string up to the first ";" (if any) when determining
the length of the chunk.

Resources:
https://tools.ietf.org/html/rfc7230#section-4.1.1
https://www.boost.org/doc/libs/1_74_0/libs/beast/doc/html/beast/using_http/chunked_encoding.html

Author: jepler

adafruit_requests.py

@@ -212,7 +212,7 @@ def _readinto(self, buf):
                 # Consume trailing \r\n for chunks 2+
                 if self._remaining == 0:
                     self._throw_away(2)
-                chunk_header = self._readto(b";", b"\r\n")
+                chunk_header = self._readto(b"\r\n").split(b";", 1)[0]
                 http_chunk_size = int(bytes(chunk_header), 16)
                 if http_chunk_size == 0:
                     self._chunked = False
@@ -253,7 +253,7 @@ def close(self):
                 self._throw_away(self._remaining)
             elif self._chunked:
                 while True:
-                    chunk_header = self._readto(b";", b"\r\n")
+                    chunk_header = self._readto(b"\r\n").split(b";", 1)[0]
                     chunk_size = int(bytes(chunk_header), 16)
                     if chunk_size == 0:
                         break

tests/chunk_test.py

@@ -9,7 +9,7 @@
 headers = b"HTTP/1.0 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n"
 
 
-def _chunk(response, split):
+def _chunk(response, split, extra=b''):
     i = 0
     chunked = b""
     while i < len(response):
@@ -19,7 +19,7 @@ def _chunk(response, split):
             chunk_size = remaining
         new_i = i + chunk_size
         chunked += (
-            hex(chunk_size)[2:].encode("ascii") + b"\r\n" + response[i:new_i] + b"\r\n"
+            hex(chunk_size)[2:].encode("ascii") + extra + b"\r\n" + response[i:new_i] + b"\r\n"
         )
         i = new_i
     # The final chunk is zero length.
@@ -28,56 +28,58 @@ def _chunk(response, split):
 
 
 def test_get_text():
-    pool = mocket.MocketPool()
-    pool.getaddrinfo.return_value = ((None, None, None, None, (ip, 80)),)
-    c = _chunk(text, 33)
-    print(c)
-    sock = mocket.Mocket(headers + c)
-    pool.socket.return_value = sock
+    for extra in (b'', b';blahblah; blah'):
+        pool = mocket.MocketPool()
+        pool.getaddrinfo.return_value = ((None, None, None, None, (ip, 80)),)
+        c = _chunk(text, 33, extra)
+        print(c)
+        sock = mocket.Mocket(headers + c)
+        pool.socket.return_value = sock
 
-    s = adafruit_requests.Session(pool)
-    r = s.get("http://" + host + path)
+        s = adafruit_requests.Session(pool)
+        r = s.get("http://" + host + path)
 
-    sock.connect.assert_called_once_with((ip, 80))
+        sock.connect.assert_called_once_with((ip, 80))
 
-    sock.send.assert_has_calls(
-        [
-            mock.call(b"GET"),
-            mock.call(b" /"),
-            mock.call(b"testwifi/index.html"),
-            mock.call(b" HTTP/1.1\r\n"),
-        ]
-    )
-    sock.send.assert_has_calls(
-        [mock.call(b"Host: "), mock.call(b"wifitest.adafruit.com"),]
-    )
-    assert r.text == str(text, "utf-8")
+        sock.send.assert_has_calls(
+            [
+                mock.call(b"GET"),
+                mock.call(b" /"),
+                mock.call(b"testwifi/index.html"),
+                mock.call(b" HTTP/1.1\r\n"),
+            ]
+        )
+        sock.send.assert_has_calls(
+            [mock.call(b"Host: "), mock.call(b"wifitest.adafruit.com"),]
+        )
+        assert r.text == str(text, "utf-8")
 
 
 def test_close_flush():
     """Test that a chunked response can be closed even when the request contents were not accessed."""
-    pool = mocket.MocketPool()
-    pool.getaddrinfo.return_value = ((None, None, None, None, (ip, 80)),)
-    c = _chunk(text, 33)
-    print(c)
-    sock = mocket.Mocket(headers + c)
-    pool.socket.return_value = sock
+    for extra in (b'', b';blahblah; blah'):
+        pool = mocket.MocketPool()
+        pool.getaddrinfo.return_value = ((None, None, None, None, (ip, 80)),)
+        c = _chunk(text, 33, extra)
+        print(c)
+        sock = mocket.Mocket(headers + c)
+        pool.socket.return_value = sock
 
-    s = adafruit_requests.Session(pool)
-    r = s.get("http://" + host + path)
+        s = adafruit_requests.Session(pool)
+        r = s.get("http://" + host + path)
 
-    sock.connect.assert_called_once_with((ip, 80))
+        sock.connect.assert_called_once_with((ip, 80))
 
-    sock.send.assert_has_calls(
-        [
-            mock.call(b"GET"),
-            mock.call(b" /"),
-            mock.call(b"testwifi/index.html"),
-            mock.call(b" HTTP/1.1\r\n"),
-        ]
-    )
-    sock.send.assert_has_calls(
-        [mock.call(b"Host: "), mock.call(b"wifitest.adafruit.com"),]
-    )
+        sock.send.assert_has_calls(
+            [
+                mock.call(b"GET"),
+                mock.call(b" /"),
+                mock.call(b"testwifi/index.html"),
+                mock.call(b" HTTP/1.1\r\n"),
+            ]
+        )
+        sock.send.assert_has_calls(
+            [mock.call(b"Host: "), mock.call(b"wifitest.adafruit.com"),]
+        )
 
-    r.close()
+        r.close()