fix(codecs)!: restrict decompression to max size

README.md

@@ -22,20 +22,10 @@ On top of the object oriented abstraction of HTTP httoop provides an easy way to
 
 HTTP and extensions are defined in the following RFC's:
 
-* HTTP/1.1 RFC 7230 [Message Syntax and Routing](https://datatracker.ietf.org/doc/html/rfc7230)
-
-* HTTP/1.1 RFC 7231 [Semantics and Content](https://datatracker.ietf.org/doc/html/rfc7231)
-
-* HTTP/1.1 RFC 7232 [Conditional Requests](https://datatracker.ietf.org/doc/html/rfc7232)
-
-* HTTP/1.1 RFC 7233 [Range Requests](https://datatracker.ietf.org/doc/html/rfc7233)
+* [RFC 9110 HTTP Semantics](https://datatracker.ietf.org/doc/html/rfc9110)
 
 * HTTP/1.1 RFC 7234 [Caching](https://datatracker.ietf.org/doc/html/rfc7234)
 
-* HTTP/1.1 RFC 7235 [Authentication](https://datatracker.ietf.org/doc/html/rfc7235)
-
-* <s>Hypertext Transfer Protocol -- HTTP/1.1 ([RFC 2616](https://datatracker.ietf.org/doc/html/rfc2616)) </s>
-
 * HTTP/2 RFC 7540 [Hypertext Transfer Protocol Version 2](https://datatracker.ietf.org/doc/html/rfc7540)
 
 * HTTP/2 RFC 7541 [HPACK: Header Compression for HTTP/2](https://datatracker.ietf.org/doc/html/rfc7541)
@@ -58,10 +48,6 @@ HTTP and extensions are defined in the following RFC's:
 
 * Internet Message Format ([RFC 822](https://datatracker.ietf.org/doc/html/rfc822), [2822](https://datatracker.ietf.org/doc/html/rfc2822), [5322](https://datatracker.ietf.org/doc/html/rfc5322))
 
-* <s>HTTP Authentication: Basic and Digest Access Authentication ([RFC 2617](https://datatracker.ietf.org/doc/html/rfc2617))</s>
-
-* HTTP Authentication-Info and Proxy-Authentication-Info Response Header Fields ([RFC 7615](https://datatracker.ietf.org/doc/html/rfc7615))
-
 * HTTP Digest Access Authentication ([RFC 7616](https://datatracker.ietf.org/doc/html/rfc7616))
 
 * The 'Basic' HTTP Authentication Scheme ([RFC 7617](https://datatracker.ietf.org/doc/html/rfc7617))
@@ -159,3 +145,14 @@ OAuth 2.0:
 * [JSON Web Token (JWT)](https://datatracker.ietf.org/doc/html/rfc7519)
 * [JSON Web Token (JWT) Profile for OAuth 2.0 Access Tokens](https://datatracker.ietf.org/doc/html/rfc9068)
 * [Resource Indicators for OAuth 2.0](https://datatracker.ietf.org/doc/html/rfc8707)
+
+Obsolete:
+
+* <s>HTTP/1.1 RFC 7230 [Message Syntax and Routing](https://datatracker.ietf.org/doc/html/rfc7230)</s>
+* <s>HTTP/1.1 RFC 7231 [Semantics and Content](https://datatracker.ietf.org/doc/html/rfc7231)</s>
+* <s>HTTP/1.1 RFC 7232 [Conditional Requests](https://datatracker.ietf.org/doc/html/rfc7232)</s>
+* <s>HTTP/1.1 RFC 7233 [Range Requests](https://datatracker.ietf.org/doc/html/rfc7233)</s>
+* <s>HTTP/1.1 RFC 7235 [Authentication](https://datatracker.ietf.org/doc/html/rfc7235)</s>
+* <s>Hypertext Transfer Protocol -- HTTP/1.1 ([RFC 2616](https://datatracker.ietf.org/doc/html/rfc2616)) </s>
+* <s>HTTP Authentication: Basic and Digest Access Authentication ([RFC 2617](https://datatracker.ietf.org/doc/html/rfc2617))</s>
+* <s>HTTP Authentication-Info and Proxy-Authentication-Info Response Header Fields ([RFC 7615](https://datatracker.ietf.org/doc/html/rfc7615))</s>

httoop/__init__.py

@@ -18,6 +18,7 @@
     BAD_GATEWAY,
     BAD_REQUEST,
     CONFLICT,
+    CONTENT_TOO_LARGE,
     CONTINUE,
     CREATED,
     EXPECTATION_FAILED,
@@ -40,7 +41,6 @@
     NOT_MODIFIED,
     OK,
     PARTIAL_CONTENT,
-    PAYLOAD_TOO_LARGE,
     PAYMENT_REQUIRED,
     PRECONDITION_FAILED,
     PROXY_AUTHENTICATION_REQUIRED,
@@ -67,7 +67,7 @@
     'ACCEPTED', 'BAD_GATEWAY', 'BAD_REQUEST', 'CONFLICT', 'CONTINUE', 'CREATED', 'EXPECTATION_FAILED', 'FORBIDDEN', 'FOUND', 'GATEWAY_TIMEOUT',
     'GONE', 'HTTP_VERSION_NOT_SUPPORTED', 'INTERNAL_SERVER_ERROR', 'I_AM_A_TEAPOT', 'LENGTH_REQUIRED', 'METHOD_NOT_ALLOWED', 'MOVED_PERMANENTLY',
     'MULTIPLE_CHOICES', 'NON_AUTHORITATIVE_INFORMATION', 'NOT_ACCEPTABLE', 'NOT_FOUND', 'NOT_IMPLEMENTED', 'NOT_MODIFIED', 'NO_CONTENT', 'OK',
-    'PARTIAL_CONTENT', 'PAYLOAD_TOO_LARGE', 'PAYMENT_REQUIRED', 'PRECONDITION_FAILED', 'PROXY_AUTHENTICATION_REQUIRED', 'RANGE_NOT_SATISFIABLE',
+    'PARTIAL_CONTENT', 'CONTENT_TOO_LARGE', 'PAYMENT_REQUIRED', 'PRECONDITION_FAILED', 'PROXY_AUTHENTICATION_REQUIRED', 'RANGE_NOT_SATISFIABLE',
     'REQUEST_TIMEOUT', 'RESET_CONTENT', 'SEE_OTHER', 'SERVICE_UNAVAILABLE', 'SWITCHING_PROTOCOLS', 'TEMPORARY_REDIRECT', 'UNAUTHORIZED',
     'UNPROCESSABLE_ENTITY', 'UNSUPPORTED_MEDIA_TYPE', 'URI', 'URI_TOO_LONG', 'USE_PROXY',
     'Body', 'ClientStateMachine', 'ComposedRequest', 'ComposedResponse', 'Date', 'DecodeError', 'EncodeError', 'Headers',

httoop/client/__init__.py

@@ -1,16 +1,18 @@
 from __future__ import annotations
 
 from httoop.exceptions import InvalidLine
-from httoop.messages import Response
+from httoop.messages import Request, Response
 from httoop.parser import NOT_RECEIVED_YET, StateMachine
 
 
 class ClientStateMachine(StateMachine):
 
     Message = Response
+    request: Request
+    message: Response
 
-    def __init__(self, *, strict: bool = True, max_status_line_length: int | float = 256) -> None:
-        super().__init__(strict=strict)
+    def __init__(self, *, strict: bool = True, max_status_line_length: float = 256, **kwargs) -> None:
+        super().__init__(strict=strict, **kwargs)
         self.max_status_line_length = max_status_line_length
 
     def parse_startline(self) -> bool | None:

httoop/codecs/application/gzip.py

@@ -1,9 +1,11 @@
+from __future__ import annotations
+
 import gzip
 import io
 import zlib
 
 from httoop.codecs.codec import Codec
-from httoop.exceptions import DecodeError, EncodeError
+from httoop.exceptions import DecodeError, EncodeError, InvalidBodySize
 from httoop.util import _
 
 
@@ -20,46 +22,48 @@ def encode(cls, data: bytes, charset: None = None, mimetype: None = None) -> byt
                 fd.write(Codec.encode(data, charset))
             return out.getvalue()
         except zlib.error:  # pragma: no cover
-            raise EncodeError(_('Invalid gzip data.'))
+            raise EncodeError(_('Invalid gzip data.')) from None
 
     @classmethod
-    def decode(cls, data: bytes, charset: None = None, mimetype: None = None) -> str:
-        try:
-            with gzip.GzipFile(fileobj=io.BytesIO(data)) as fd:
-                data = fd.read()
-        except (zlib.error, OSError, EOFError):
-            raise DecodeError(_('Invalid gzip data.'))
-        return Codec.decode(data, charset)
+    def decode(cls, data: bytes, charset: str | None = None, mimetype: None = None, max_size: int = -1) -> str:
+        with gzip.GzipFile(fileobj=io.BytesIO(data)) as fd:
+            try:
+                result = fd.read(max_size + 1 if max_size > 0 else -1)
+                if max_size > 0 and len(result) > max_size:
+                    raise InvalidBodySize(_('Maximum content size (%d) reached'), max_size)
+            except (zlib.error, OSError, EOFError):
+                raise DecodeError(_('Invalid gzip data.')) from None
+        return Codec.decode(result, charset)
 
     @classmethod
     def iterencode(cls, data, charset=None, mimetype=None):
+        out = io.BytesIO()
         try:
-            out = io.BytesIO()
             with gzip.GzipFile(fileobj=out, mode='w', compresslevel=cls.compression_level) as fd:
                 for part in data:
                     fd.write(Codec.encode(part, charset))
                     yield out.getvalue()
                     out.seek(0)
                     out.truncate()
-            yield out.getvalue()
         except zlib.error:  # pragma: no cover
-            raise EncodeError(_('Invalid gzip data.'))
+            raise EncodeError(_('Invalid gzip data.')) from None
+        yield out.getvalue()
 
-    @classmethod
-    def iterdecode(cls, data, charset=None, mimetype=None):
-        try:
-            fd = io.BytesIO()
-            with gzip.GzipFile(fileobj=fd) as gzfd:
-                # FIXME: the gzip module cannot handle partial data
-                # for part in data:
-                #    fd.write(part)
-                #    fd.seek(fd.tell() - length)
-                #    fd.seek(fd.tell() - length)
-                #    yield Codec.decode(gzfd.read(), charset)
-                # yield Codec.decode(gzfd.read(), charset)
-                for part in data:
-                    fd.write(part)
-                fd.seek(0)
-                yield Codec.decode(gzfd.read(), charset)
-        except (zlib.error, OSError, EOFError):
-            raise DecodeError(_('Invalid gzip data.'))
+    # @classmethod
+    # def iterdecode(cls, data, charset=None, mimetype=None):
+    #     try:
+    #         fd = io.BytesIO()
+    #         with gzip.GzipFile(fileobj=fd) as gzfd:
+    #             # FIXME: the gzip module cannot handle partial data
+    #             # for part in data:
+    #             #    fd.write(part)
+    #             #    fd.seek(fd.tell() - length)
+    #             #    fd.seek(fd.tell() - length)
+    #             #    yield Codec.decode(gzfd.read(), charset)
+    #             # yield Codec.decode(gzfd.read(), charset)
+    #             for part in data:
+    #                 fd.write(part)
+    #             fd.seek(0)
+    #             yield Codec.decode(gzfd.read(), charset)
+    #     except (zlib.error, OSError, EOFError):
+    #         raise DecodeError(_('Invalid gzip data.')) from None

httoop/codecs/application/zlib.py

@@ -3,7 +3,7 @@
 import zlib
 
 from httoop.codecs.codec import Codec
-from httoop.exceptions import DecodeError, EncodeError
+from httoop.exceptions import DecodeError, EncodeError, InvalidBodySize
 from httoop.util import _
 
 
@@ -15,12 +15,42 @@ def encode(cls, data: bytes, charset: None = None, mimetype: None = None) -> byt
         try:
             return zlib.compress(Codec.encode(data, charset))
         except zlib.error:  # pragma: no cover
-            raise EncodeError(_('Invalid zlib/deflate data.'))
+            raise EncodeError(_('Invalid zlib/deflate data.')) from None
 
     @classmethod
-    def decode(cls, data: bytes, charset: str | None = None, mimetype: None = None) -> str:
+    def decode(cls, data: bytes, charset: str | None = None, mimetype: None = None, max_size: int = -1) -> str:
         try:
-            data = zlib.decompress(data)
+            result = _decompress_limited(data, max_size) if max_size > 0 else zlib.decompress(data)
         except zlib.error:
-            raise DecodeError(_('Invalid zlib/deflate data.'))
-        return Codec.decode(data, charset)
+            raise DecodeError(_('Invalid zlib/deflate data.')) from None
+
+        return Codec.decode(result, charset)
+
+
+def _decompress_limited(data: bytes, max_size: int) -> bytes:
+    decompressor = zlib.decompressobj()
+    chunks = []
+    size = 0
+    pending = data
+
+    while True:
+        remaining = max_size - size
+
+        chunk = decompressor.decompress(pending, remaining + 1)
+        size += len(chunk)
+
+        if size > max_size:
+            raise InvalidBodySize(_('Maximum content size (%d) reached'), max_size)
+
+        chunks.append(chunk)
+
+        if decompressor.eof:
+            break
+
+        pending = decompressor.unconsumed_tail
+
+        if not pending:
+            # if zlib still hasn't reached EOF the compressed representation is truncated
+            raise zlib.error()
+
+    return b''.join(chunks)

httoop/codecs/codec.py

@@ -9,14 +9,16 @@
 
 class Codec:
 
+    mimetype: str
+
     @classmethod
-    def decode(cls, data: bytes, charset: str | None = None, mimetype: None = None) -> str:  # pragma: no cover
+    def decode(cls, data: bytes, charset: str | None = None, mimetype: ContentType | None = None, **kwargs) -> str:  # pragma: no cover
         if isinstance(data, bytes):
             data = data.decode(charset or 'ascii')
         return data
 
     @classmethod
-    def encode(cls, data: bytes, charset: None = None, mimetype: None = None) -> bytes:  # pragma: no cover
+    def encode(cls, data: bytes, charset: None = None, mimetype: ContentType | None = None) -> bytes:  # pragma: no cover
         if isinstance(data, str):
             data = data.encode(charset or 'ascii')
         return data
@@ -26,5 +28,5 @@ def iterencode(cls, data: Any, charset: str | None = None, mimetype: ContentType
         yield cls.encode(data, charset, mimetype)
 
     @classmethod
-    def iterdecode(cls, data: Any, charset=None, mimetype=None):  # pragma: no cover
+    def iterdecode(cls, data: Any, charset=None, mimetype: ContentType | None = None):  # pragma: no cover
         yield cls.decode(data, charset, mimetype)

httoop/messages/body.py

@@ -156,11 +156,11 @@ def compress(self) -> None:
             self.set(codec.encode(self.__content_bytes()))
             self.content_encoding = None
 
-    def decompress(self) -> None:
+    def decompress(self, *, max_size: int = -1) -> None:
         """Applies the Content-Encoding codec to the content."""
         codec = self.content_codec
         if codec:
-            self.set(codec.decode(self.__content_bytes()))
+            self.set(codec.decode(self.__content_bytes(), max_size=max_size))
             self.content_encoding = None
 
     def set(self, content: Any) -> None:

httoop/parser.py

@@ -1,5 +1,4 @@
 """Implements a state machine for the parsing process."""
-# TODO: translation API
 
 from __future__ import annotations
 
@@ -9,7 +8,7 @@
 from httoop.header import Headers
 from httoop.header.messaging import Trailer
 from httoop.messages import Message
-from httoop.status import BAD_REQUEST, NOT_IMPLEMENTED, PAYLOAD_TOO_LARGE, REQUEST_HEADER_FIELDS_TOO_LARGE
+from httoop.status import BAD_REQUEST, CONTENT_TOO_LARGE, NOT_IMPLEMENTED, REQUEST_HEADER_FIELDS_TOO_LARGE
 from httoop.util import _, integer
 
 
@@ -22,6 +21,7 @@
 LF = b'\n'
 CRLF = CR + LF
 NOT_RECEIVED_YET = True
+DEFAULT_MAX_BODY_SIZE = 104857600  # 100 MB
 
 
 class StateMachine:
@@ -32,14 +32,16 @@ class StateMachine:
 
     Message = Message  # subclass provides the type
 
-    def __init__(
+    def __init__(  # noqa: PLR0913
         self,
         *,
         strict: bool = True,
         max_header_count: int = 1024,
         max_header_line_length: int = 8192,
         max_header_section_size: int = 65536,
-        max_body_size: float = float('inf'),
+        max_body_size: float = DEFAULT_MAX_BODY_SIZE,
+        decompress_body: bool = False,
+        max_decompressed_body_size: float | None = None,
     ) -> None:
         self.buffer = bytearray()
         self.message = None
@@ -48,6 +50,10 @@ def __init__(
         self.max_header_line_length = max_header_line_length
         self.max_header_section_size = max_header_section_size
         self.max_body_size = max_body_size
+        self.decompress_body = decompress_body
+        if max_decompressed_body_size is None:
+            max_decompressed_body_size = max_body_size
+        self.max_decompressed_body_size = max_decompressed_body_size
 
     def _reset_state(self) -> None:
         self.message = self.Message()
@@ -90,8 +96,12 @@ def on_headers_complete(self) -> None:
 
     def on_body_complete(self) -> None:
         self.message.body.seek(0)
-        self.message.body.decompress()
-        self.message.body.seek(0)
+        if self.decompress_body:
+            try:
+                self.message.body.decompress(max_size=-1 if self.max_decompressed_body_size == float('inf') else self.max_decompressed_body_size)
+            except InvalidBodySize as exc:
+                raise CONTENT_TOO_LARGE(str(exc)) from exc
+            self.message.body.seek(0)
         self.set_content_length()
 
     def on_message_complete(self) -> Response | Request:
@@ -225,14 +235,14 @@ def determine_message_length(self) -> None:
             except ValueError:
                 raise BAD_REQUEST(_('Invalid Content-Length header.'))
             if self.message_length > self.max_body_size:
-                raise PAYLOAD_TOO_LARGE(_('Maximum content size (%d) reached') % (self.max_body_size,))
+                raise CONTENT_TOO_LARGE(_('Maximum content size (%d) reached') % (self.max_body_size,))
 
     def parse_body_with_message_length(self) -> bool | None:
         body, self.buffer = self.buffer[:self.message_length], self.buffer[self.message_length:]
         try:
             self.message.body.parse(bytes(body))
         except InvalidBodySize as exc:
-            raise PAYLOAD_TOO_LARGE(str(exc))
+            raise CONTENT_TOO_LARGE(str(exc))
 
         blen = len(body)
         unfinished = blen < self.message_length
@@ -260,7 +270,7 @@ def parse_chunked_body(self) -> bool:
         try:
             self.message.body.parse(bytes(body_part))
         except InvalidBodySize as exc:
-            raise PAYLOAD_TOO_LARGE(str(exc))
+            raise CONTENT_TOO_LARGE(str(exc))
         self.buffer = rest_chunk
 
         if chunk_size == 0: