From 4941f34557fcc589cb7c4618e9e83b31da51d749 Mon Sep 17 00:00:00 2001 From: Ray Walker Date: Wed, 2 Sep 2026 18:06:15 +1000 Subject: [PATCH 1/9] fix(serializers): bound untrusted msgpack decode depth and header allocation (LAB-2503) All four backend-bytes decode sites (auto, standard, interop, and the DataFrame/Series branches) now go through unpackb_bounded: a header-only Unpacker.skip() walk first (allocation-free, ~1/4 the cost of decode) rejects nesting past the pinned 1024 ceiling and any header claiming more than the input can back, then unpackb runs with every max_*_len passed explicitly. Before: msgpack-python's defaults allowed ~8 x 1024 x len(input) bytes of transient heap (measured 10 KB -> 67 MB). Also fail closed when a checksum-verified envelope carries an undecodable payload: AutoSerializer used to fall through and return the ENVELOPE's positional fields as the cached value. Regression-guarded by the protocol decode-bounds vectors on every path. --- README.md | 1 + src/cachekit/interop.py | 6 +- src/cachekit/serializers/auto_serializer.py | 64 +++--- src/cachekit/serializers/base.py | 59 +++++ .../serializers/standard_serializer.py | 4 +- .../unit/protocol/fixtures/decode-bounds.json | 210 ++++++++++++++++++ tests/unit/protocol/test_decode_bounds.py | 183 +++++++++++++++ 7 files changed, 491 insertions(+), 36 deletions(-) create mode 100644 tests/unit/protocol/fixtures/decode-bounds.json create mode 100644 tests/unit/protocol/test_decode_bounds.py diff --git a/README.md b/README.md index 3f6fae3e..94f5defb 100644 --- a/README.md +++ b/README.md @@ -235,6 +235,7 @@ def test_cached_function(): - Connection pooling with thread affinity (+28% throughput) - Distributed locking prevents cache stampedes - Pluggable backend abstraction (Redis, CachekitIO, File, Memcached, custom) +- Untrusted-decode bounds: every cache read is a MessagePack decode of bytes the backend controls, so nesting depth and header-declared allocation are capped as cachekit-owned invariants (a forged nested-header entry is a bounded cache miss, not a memory blow-up) — verified against the protocol's shared [`decode-bounds.json`](https://github.com/cachekit-io/protocol/blob/main/test-vectors/decode-bounds.json) vectors > [!NOTE] > All reliability features are **enabled by default** with `@cache.production`. Use `@cache.minimal` to disable them for maximum throughput. diff --git a/src/cachekit/interop.py b/src/cachekit/interop.py index 6be28ee9..d15a9bc4 100644 --- a/src/cachekit/interop.py +++ b/src/cachekit/interop.py @@ -36,9 +36,7 @@ from typing import Any from uuid import UUID -import msgpack - -from .serializers.base import SerializationError +from .serializers.base import SerializationError, unpackb_bounded # Full-string match REQUIRED: re.match with a $ anchor still accepts a # trailing newline. Pinned by the reject_trailing_newline error vector. @@ -440,7 +438,7 @@ def decode_interop_value(data: bytes | bytearray | memoryview) -> Any: "check that every writer for this key uses @cache(interop=...)." ) try: - return msgpack.unpackb(raw, raw=False, strict_map_key=True, object_hook=_revive_sentinels) + return unpackb_bounded(raw, raw=False, strict_map_key=True, object_hook=_revive_sentinels) except Exception as e: raise InteropDecodeError(f"stored value is not a single well-formed MessagePack document: {e}") from e diff --git a/src/cachekit/serializers/auto_serializer.py b/src/cachekit/serializers/auto_serializer.py index 1dcd25ce..a445d021 100644 --- a/src/cachekit/serializers/auto_serializer.py +++ b/src/cachekit/serializers/auto_serializer.py @@ -60,7 +60,7 @@ from cachekit._rust_serializer import ByteStorage -from .base import SerializationError, SerializationFormat, SerializationMetadata +from .base import SerializationError, SerializationFormat, SerializationMetadata, unpackb_bounded logger = logging.getLogger(__name__) @@ -550,10 +550,10 @@ def deserialize(self, data: bytes | memoryview, metadata: Optional[Serialization original_data, _ = self._byte_storage.retrieve(data) except (ValueError, SerializationError) as e: raise SerializationError(f"DataFrame integrity check failed (corrupted cache entry): {e}") from e - unpacked_data = msgpack.unpackb(original_data, **self._msgpack_unpack_opts) + unpacked_data = unpackb_bounded(original_data, **self._msgpack_unpack_opts) return self._deserialize_dataframe(unpacked_data) # Integrity off: data is direct msgpack (no envelope) - unpacked_data = msgpack.unpackb(data, **self._msgpack_unpack_opts) + unpacked_data = unpackb_bounded(data, **self._msgpack_unpack_opts) return self._deserialize_dataframe(unpacked_data) elif detected_format == "series": if self.enable_integrity_checking and len(data) > 4: @@ -562,10 +562,10 @@ def deserialize(self, data: bytes | memoryview, metadata: Optional[Serialization original_data, _ = self._byte_storage.retrieve(data) except (ValueError, SerializationError) as e: raise SerializationError(f"Series integrity check failed (corrupted cache entry): {e}") from e - unpacked_data = msgpack.unpackb(original_data, **self._msgpack_unpack_opts) + unpacked_data = unpackb_bounded(original_data, **self._msgpack_unpack_opts) return self._deserialize_series(unpacked_data) # Integrity off: data is direct msgpack (no envelope) - unpacked_data = msgpack.unpackb(data, **self._msgpack_unpack_opts) + unpacked_data = unpackb_bounded(data, **self._msgpack_unpack_opts) return self._deserialize_series(unpacked_data) # For Rust-envelope formats, use the Rust layer @@ -573,32 +573,36 @@ def deserialize(self, data: bytes | memoryview, metadata: Optional[Serialization try: # Use Rust layer for decompression and validation original_data, format_id = self._byte_storage.retrieve(data) - - # Use metadata if available, otherwise fall back to format_id from envelope - if metadata and hasattr(metadata, "original_type"): - detected_format = metadata.original_type - else: - detected_format = format_id - - # Deserialize based on detected format - if detected_format == "numpy": - return self._deserialize_numpy(original_data) - elif detected_format == "dataframe": - # Unpack the msgpack data first, then pass to DataFrame deserializer - unpacked_data = msgpack.unpackb(original_data, **self._msgpack_unpack_opts) - return self._deserialize_dataframe(unpacked_data) - elif detected_format == "series": - # Unpack the msgpack data first, then pass to Series deserializer - unpacked_data = msgpack.unpackb(original_data, **self._msgpack_unpack_opts) - return self._deserialize_series(unpacked_data) - else: # msgpack - return msgpack.unpackb(original_data, **self._msgpack_unpack_opts) except SerializationError: # Re-raise SerializationError (corruption detection) without swallowing raise except Exception as e: - # If Rust envelope parsing fails for other reasons, try Python-only deserialization + # Not a ByteStorage envelope (e.g. written with integrity checking off): + # fall through to the Python-only paths below. logger.debug(f"Rust envelope parsing failed, falling back to Python-only deserialization: {e}") + else: + # The envelope verified (checksum matched), so its payload is exactly what was + # stored; a payload that then fails to decode is corruption or a forged entry + # (LAB-2503 decode bomb) and MUST fail closed. Falling through here used to + # re-decode the ENVELOPE bytes as plain MessagePack and return its positional + # fields as the cached value — wrong data, silently. + # Use metadata if available, otherwise fall back to format_id from envelope + detected_format = metadata.original_type if metadata and hasattr(metadata, "original_type") else format_id + try: + if detected_format == "numpy": + return self._deserialize_numpy(original_data) + if detected_format in ("dataframe", "series"): + unpacked_data = unpackb_bounded(original_data, **self._msgpack_unpack_opts) + if detected_format == "dataframe": + return self._deserialize_dataframe(unpacked_data) + return self._deserialize_series(unpacked_data) + return unpackb_bounded(original_data, **self._msgpack_unpack_opts) + except SerializationError: + raise + except Exception as e: + raise SerializationError( + f"Cache entry payload failed to decode inside a verified envelope (format={detected_format!r}): {e}" + ) from e # Check for Arrow IPC format before msgpack fall-through # Arrow data may have xxHash3-64 checksum prefix (8 bytes) or be direct Arrow IPC @@ -621,7 +625,7 @@ def deserialize(self, data: bytes | memoryview, metadata: Optional[Serialization # Python-only path (no Rust compression) - direct msgpack deserialization try: - return msgpack.unpackb(data, **self._msgpack_unpack_opts) + return unpackb_bounded(data, **self._msgpack_unpack_opts) except SerializationError: # Re-raise SerializationError (corruption detection) without swallowing raise @@ -776,7 +780,7 @@ def _deserialize_dataframe(self, data) -> pd.DataFrame: serialized = data else: # Otherwise unpack msgpack - serialized = msgpack.unpackb(data, **self._msgpack_unpack_opts) + serialized = unpackb_bounded(data, **self._msgpack_unpack_opts) # Reconstruct DataFrame column by column columns_data = {} @@ -843,7 +847,7 @@ def _deserialize_series(self, data) -> pd.Series: serialized = data else: # Otherwise unpack msgpack - serialized = msgpack.unpackb(data, **self._msgpack_unpack_opts) + serialized = unpackb_bounded(data, **self._msgpack_unpack_opts) if serialized["type"] == "numeric": # .copy() → writable Series values that do not alias the source buffer (#157). @@ -908,7 +912,7 @@ def validate_data(self, data: bytes) -> bool: else: # Python-only mode validation try: - msgpack.unpackb(data, **self._msgpack_unpack_opts) + unpackb_bounded(data, **self._msgpack_unpack_opts) return True except (msgpack.exceptions.UnpackException, ValueError, TypeError, AttributeError): # AttributeError can occur when datetime_object_hook tries to restore invalid data diff --git a/src/cachekit/serializers/base.py b/src/cachekit/serializers/base.py index 6b44cbfb..727b5598 100644 --- a/src/cachekit/serializers/base.py +++ b/src/cachekit/serializers/base.py @@ -8,6 +8,8 @@ from enum import Enum from typing import Any, ClassVar, Protocol, runtime_checkable +import msgpack + @runtime_checkable class SerializerProtocol(Protocol): @@ -320,3 +322,60 @@ class SuspiciousCacheEntryError(SerializationError): """ pass + + +# --------------------------------------------------------------------------- +# Owned untrusted-decode bounds (LAB-2503; protocol spec/interop-mode.md → Decode bounds) +# --------------------------------------------------------------------------- + +#: Nesting depth msgpack-python's C unpacker accepts before raising ``StackError``. +#: Not configurable through its API — pinned here and regression-tested +#: (tests/unit/protocol/test_decode_bounds.py) so a dependency bump that moves it +#: fails a test instead of silently changing the decode ceiling. The protocol +#: requires every SDK's bound to sit in 32..=1024. +MSGPACK_MAX_NESTING = 1024 + + +def unpackb_bounded(data: bytes | bytearray | memoryview, **unpack_opts: Any) -> Any: + """Decode one untrusted MessagePack document under cachekit-owned bounds. + + Why not plain ``msgpack.unpackb``: a collection header costs 1-5 bytes but may + declare up to 2**32-1 elements, and the C unpacker pre-allocates the container + (``PyList_New(n)``) *before* decoding the children. Nested headers stack those + allocations depth-first, so the library's per-collection default cap + (``max_*_len = len(data)``) still permits ~8 x 1024 x len(data) bytes of + transient heap — measured 10 KB -> 67 MB. Two bounds close it: + + 1. A header-only structural walk (``Unpacker.skip``) runs first. It allocates + nothing, costs a fraction of the decode, and rejects a document that nests + deeper than :data:`MSGPACK_MAX_NESTING` (``StackError``) or declares more + elements/bytes than the input can back (``OutOfData``). Every element that + survives is backed by >= 1 input byte, so the real decode's pre-allocation + is bounded by ~8 x len(data). + 2. The collection/str/bin/ext caps are passed explicitly as ``len(data)`` — + the library's current default, made an owned invariant so a msgpack-python + change cannot silently lift it. + + Every rejection is a ``ValueError`` (``StackError``, ``FormatError``, + ``ExtraData``, or the over-claim ``ValueError`` raised here), which the read + paths already turn into a controlled cache miss. Trailing bytes are still + rejected by ``unpackb`` itself. + + Examples: + >>> unpackb_bounded(msgpack.packb({"a": [1, 2]}), raw=False) + {'a': [1, 2]} + >>> unpackb_bounded(b"\\xdc\\x27\\x10" * 5000) # 15 KB nested-header bomb + Traceback (most recent call last): + ... + msgpack.exceptions.StackError + """ + n = len(data) + walker = msgpack.Unpacker(max_buffer_size=n) + walker.feed(data) + try: + walker.skip() + except msgpack.exceptions.OutOfData as e: + # OutOfData is the one Unpacker error that is not a ValueError; normalise it to + # the same contract unpackb uses for truncated input ("Unpack failed: incomplete input"). + raise ValueError("Unpack failed: MessagePack document declares more elements/bytes than the input can back") from e + return msgpack.unpackb(data, max_str_len=n, max_bin_len=n, max_array_len=n, max_map_len=n, max_ext_len=n, **unpack_opts) diff --git a/src/cachekit/serializers/standard_serializer.py b/src/cachekit/serializers/standard_serializer.py index 404c7797..e69f88fa 100644 --- a/src/cachekit/serializers/standard_serializer.py +++ b/src/cachekit/serializers/standard_serializer.py @@ -27,7 +27,7 @@ from cachekit._rust_serializer import ByteStorage -from .base import SerializationError, SerializationFormat, SerializationMetadata +from .base import SerializationError, SerializationFormat, SerializationMetadata, unpackb_bounded # Error message constants for unsupported types (Task 2) NUMPY_ERROR_MESSAGE = ( @@ -339,7 +339,7 @@ def deserialize(self, data: bytes | memoryview, metadata: SerializationMetadata msgpack_data = data # Deserialize MessagePack - return msgpack.unpackb(msgpack_data, **self._msgpack_unpack_opts) + return unpackb_bounded(msgpack_data, **self._msgpack_unpack_opts) except SerializationError: # Re-raise SerializationError (integrity check failure) without swallowing raise diff --git a/tests/unit/protocol/fixtures/decode-bounds.json b/tests/unit/protocol/fixtures/decode-bounds.json new file mode 100644 index 00000000..91ab717c --- /dev/null +++ b/tests/unit/protocol/fixtures/decode-bounds.json @@ -0,0 +1,210 @@ +{ + "version": "1.0.0", + "spec": "spec/interop-mode.md#decode-bounds", + "generator": "tools/decode-bounds-reference.py generate (CPython stdlib)", + "scope": "Any untrusted MessagePack decode in any SDK: interop/v1 values, auto-mode payloads after the ByteStorage envelope is unwrapped, invalidation events. The bytes are plain MessagePack with no envelope.", + "rules": { + "depth": "Readers MUST bound nesting depth. The bound MUST be >= 32 and MUST be <= 1024; every reject vector tagged 'depth' nests deeper than 1024.", + "overclaim": "Readers MUST NOT pre-allocate for a collection/str/bin header more than the remaining input can back (each element or byte needs >= 1 input byte), and MUST reject a structurally incomplete document. Every reject vector tagged 'overclaim' has declared_slots > input_len - 1 (the root header is the only byte that is not an element).", + "failure_mode": "Rejection MUST surface as a catchable decode error that the SDK read path turns into a cache miss (fail-closed), never an uncaught crash or an OOM abort." + }, + "field_notes": { + "construction": "input = bytes.fromhex(repeat_hex) * count + bytes.fromhex(suffix_hex)", + "nesting_depth": "collection headers along the deepest spine (str/bin count as 0)", + "declared_slots": "sum of every header's declared element/byte count (a nested header counts as one element of its parent)", + "reject_reasons": "which rule(s) the vector violates; a maintainer note, not a normative message" + }, + "reject_vectors": [ + { + "name": "nested_array16_depth_2048", + "description": "2048 nested array16 headers each claiming 10 000 elements, 0 backing bytes. The LAB-2487 amplifier shape: an eager decoder pre-allocates 10 000 slots per level before hitting EOF.", + "construction": { + "repeat_hex": "dc2710", + "count": 2048, + "suffix_hex": "" + }, + "input_hex": "dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710", + "input_len": 6144, + "nesting_depth": 2048, + "declared_slots": 20480000, + "reject_reasons": [ + "depth", + "overclaim" + ] + }, + { + "name": "nested_array32_input_len_depth_1100", + "description": "1100 nested array32 headers each claiming exactly len(input)=5500 elements. Defeats a per-collection cap of len(input): peak pre-allocation is depth x len(input) x slot size.", + "construction": { + "repeat_hex": "dd0000157c", + "count": 1100, + "suffix_hex": "" + }, + "input_hex": "dd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157c", + "input_len": 5500, + "nesting_depth": 1100, + "declared_slots": 6050000, + "reject_reasons": [ + "depth", + "overclaim" + ] + }, + { + "name": "nested_map16_depth_2048", + "description": "Map twin of nested_array16_depth_2048 (map pre-allocation is typically larger per slot).", + "construction": { + "repeat_hex": "de2710", + "count": 2048, + "suffix_hex": "" + }, + "input_hex": "de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710", + "input_len": 6144, + "nesting_depth": 2048, + "declared_slots": 20480000, + "reject_reasons": [ + "depth", + "overclaim" + ] + }, + { + "name": "nested_fixarray_depth_2048_complete", + "description": "Structurally COMPLETE document ([[...[null]...]]) nested 2048 deep: every header is backed, so only the depth bound rejects it. Isolates the depth rule from the allocation rule.", + "construction": { + "repeat_hex": "91", + "count": 2048, + "suffix_hex": "c0" + }, + "input_hex": "9191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191c0", + "input_len": 2049, + "nesting_depth": 2048, + "declared_slots": 2048, + "reject_reasons": [ + "depth" + ] + }, + { + "name": "array16_overclaim_shallow", + "description": "One array16 header claiming 10 000 elements with 3 backing bytes.", + "construction": { + "repeat_hex": "dc2710", + "count": 1, + "suffix_hex": "010203" + }, + "input_hex": "dc2710010203", + "input_len": 6, + "nesting_depth": 1, + "declared_slots": 10000, + "reject_reasons": [ + "overclaim" + ] + }, + { + "name": "array32_max_claim_alone", + "description": "A lone 5-byte array32 header claiming 2^32-1 elements.", + "construction": { + "repeat_hex": "ddffffffff", + "count": 1, + "suffix_hex": "" + }, + "input_hex": "ddffffffff", + "input_len": 5, + "nesting_depth": 1, + "declared_slots": 4294967295, + "reject_reasons": [ + "overclaim" + ] + }, + { + "name": "map32_max_claim_alone", + "description": "A lone 5-byte map32 header claiming 2^32-1 pairs.", + "construction": { + "repeat_hex": "dfffffffff", + "count": 1, + "suffix_hex": "" + }, + "input_hex": "dfffffffff", + "input_len": 5, + "nesting_depth": 1, + "declared_slots": 4294967295, + "reject_reasons": [ + "overclaim" + ] + }, + { + "name": "bin32_overclaim", + "description": "bin32 header claiming 2^32-1 bytes with 1 backing byte (a 6-byte document declaring a 4 GiB buffer).", + "construction": { + "repeat_hex": "c6ffffffff", + "count": 1, + "suffix_hex": "41" + }, + "input_hex": "c6ffffffff41", + "input_len": 6, + "nesting_depth": 0, + "declared_slots": 4294967295, + "reject_reasons": [ + "overclaim" + ] + }, + { + "name": "str32_overclaim", + "description": "str32 twin of bin32_overclaim.", + "construction": { + "repeat_hex": "dbffffffff", + "count": 1, + "suffix_hex": "41" + }, + "input_hex": "dbffffffff41", + "input_len": 6, + "nesting_depth": 0, + "declared_slots": 4294967295, + "reject_reasons": [ + "overclaim" + ] + }, + { + "name": "fixarray_short_by_one", + "description": "fixarray claiming 5 elements with 4 present: the minimal truncated document.", + "construction": { + "repeat_hex": "95", + "count": 1, + "suffix_hex": "c0c0c0c0" + }, + "input_hex": "95c0c0c0c0", + "input_len": 5, + "nesting_depth": 1, + "declared_slots": 5, + "reject_reasons": [ + "overclaim" + ] + } + ], + "accept_vectors": [ + { + "name": "nested_fixarray_depth_32", + "description": "[[...[null]...]] nested 32 deep, complete. A conforming reader MUST accept it: the depth bound may not be tighter than 32.", + "construction": { + "repeat_hex": "91", + "count": 32, + "suffix_hex": "c0" + }, + "input_hex": "9191919191919191919191919191919191919191919191919191919191919191c0", + "input_len": 33, + "nesting_depth": 32, + "declared_slots": 32 + }, + { + "name": "array16_256_backed_nils", + "description": "array16 header claiming 256 elements with all 256 present. A *16 header that is fully backed by input is legitimate; the allocation rule is about backing, not header width.", + "construction": { + "repeat_hex": "dc0100", + "count": 1, + "suffix_hex": "c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0" + }, + "input_hex": "dc0100c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0", + "input_len": 259, + "nesting_depth": 1, + "declared_slots": 256 + } + ] +} diff --git a/tests/unit/protocol/test_decode_bounds.py b/tests/unit/protocol/test_decode_bounds.py new file mode 100644 index 00000000..6134b52c --- /dev/null +++ b/tests/unit/protocol/test_decode_bounds.py @@ -0,0 +1,183 @@ +"""Untrusted-decode bounds (LAB-2503): protocol vectors + the SDK-local regression guard. + +Every cache read is a MessagePack decode of bytes the backend controls. A +collection header costs 1-5 bytes but may declare up to 2**32-1 elements, and +msgpack-python's C unpacker pre-allocates the container before decoding the +children, so nested headers stack allocations depth-first. Before this guard the +decoder was bounded only by library defaults, and those defaults still allowed +~8 x 1024 x len(data) bytes of transient heap (measured 10 KB -> 67 MB; the +"82 MB hard ceiling" once reported was an artifact of the array16(10000) probe). + +Fixture: tests/unit/protocol/fixtures/decode-bounds.json, vendored from +cachekit-io/protocol test-vectors/decode-bounds.json +(sha256 864b7126986e9a2bd0dd50358018eda34fe2f70bca06ae9763e8ce6321f34b0a). +Regenerate ONLY by re-copying from the protocol repo — never by hand. + +What is pinned here, so a msgpack-python bump cannot silently move it: +- every reject vector is rejected on every decode path, with a bounded peak; +- every accept vector decodes on every path (the bound cannot over-tighten); +- the nesting ceiling is exactly MSGPACK_MAX_NESTING; +- the read path turns a bomb into SerializationError (a controlled miss), not a crash. +""" + +from __future__ import annotations + +import functools +import hashlib +import json +import tracemalloc +from collections.abc import Callable +from pathlib import Path +from typing import Any + +import msgpack +import pytest + +from cachekit._rust_serializer import ByteStorage +from cachekit.cache_handler import CacheSerializationHandler +from cachekit.interop import decode_interop_value +from cachekit.serializers.auto_serializer import AutoSerializer +from cachekit.serializers.base import MSGPACK_MAX_NESTING, SerializationError, unpackb_bounded +from cachekit.serializers.standard_serializer import StandardSerializer +from cachekit.serializers.wrapper import SerializationWrapper + +FIXTURE_PATH = Path(__file__).parent / "fixtures" / "decode-bounds.json" +FIXTURE_SHA256 = "864b7126986e9a2bd0dd50358018eda34fe2f70bca06ae9763e8ce6321f34b0a" # pragma: allowlist secret +VECTORS = json.loads(FIXTURE_PATH.read_text(encoding="utf-8")) +EXPECTED_COUNTS = {"reject_vectors": 10, "accept_vectors": 2} + +# Peak transient heap a rejected decode may cost: a small constant (Unpacker buffer) +# plus a few multiples of the input. The unguarded decoder peaks at ~5000x for the +# 15 KB bomb below, so this discriminates by three orders of magnitude. +PEAK_BUDGET = 2 * 1024 * 1024 +PEAK_PER_INPUT_BYTE = 4 + + +def _envelope(payload: bytes) -> bytes: + return bytes(ByteStorage("msgpack").store(payload, "msgpack")) + + +CACHE_KEY = "ns:decode:bounds" + + +@functools.lru_cache(maxsize=1) +def _frame_template() -> tuple[dict[str, Any], str]: + _, metadata, serializer_name = SerializationWrapper.unwrap( + CacheSerializationHandler().serialize_data({"t": 1}, cache_key=CACHE_KEY) + ) + return metadata, serializer_name + + +def _forged_entry(payload: bytes) -> bytes: + """A genuine CK v3 frame with its payload swapped — the backend-write attacker's move.""" + metadata, serializer_name = _frame_template() + return SerializationWrapper.wrap(_envelope(payload), metadata, serializer_name) + + +# Every path that decodes backend-supplied MessagePack. Each must reach unpackb_bounded. +DECODE_PATHS: dict[str, Callable[[bytes], Any]] = { + "unpackb_bounded": lambda b: unpackb_bounded(b, raw=False), + "interop": decode_interop_value, + "standard/plain": StandardSerializer(enable_integrity_checking=False).deserialize, + "standard/envelope": lambda b: StandardSerializer().deserialize(_envelope(b)), + "auto/plain": AutoSerializer(enable_integrity_checking=False).deserialize, + "auto/envelope": lambda b: AutoSerializer().deserialize(_envelope(b)), + "handler.deserialize_data": lambda b: CacheSerializationHandler().deserialize_data(_forged_entry(b), cache_key=CACHE_KEY), +} + + +def _peak_of(fn: Callable[..., Any], *args: Any) -> tuple[Any, BaseException | None, int]: + tracemalloc.start() + try: + return fn(*args), None, tracemalloc.get_traced_memory()[1] + except Exception as e: # noqa: BLE001 — the exception type is what we assert on + return None, e, tracemalloc.get_traced_memory()[1] + finally: + tracemalloc.stop() + + +def _vector_ids(group: str) -> list[str]: + return [v["name"] for v in VECTORS[group]] + + +class TestFixtureIsTheVendoredProtocolFile: + def test_sha256_and_counts(self) -> None: + assert hashlib.sha256(FIXTURE_PATH.read_bytes()).hexdigest() == FIXTURE_SHA256 + assert {g: len(VECTORS[g]) for g in EXPECTED_COUNTS} == EXPECTED_COUNTS + assert VECTORS["spec"] == "spec/interop-mode.md#decode-bounds" + + +@pytest.mark.parametrize("path", DECODE_PATHS) +class TestProtocolVectors: + @pytest.mark.parametrize("vector", VECTORS["reject_vectors"], ids=_vector_ids("reject_vectors")) + def test_reject_vector_is_rejected_with_bounded_peak(self, path: str, vector: dict[str, Any]) -> None: + data = bytes.fromhex(vector["input_hex"]) + assert len(data) == vector["input_len"] + _, err, peak = _peak_of(DECODE_PATHS[path], data) + assert err is not None, f"{vector['name']}: {path} decoded a reject vector" + # Every rejection is a controlled error the read path maps to a cache miss. + assert isinstance(err, (ValueError, SerializationError)), f"{vector['name']}: {path} raised {type(err).__name__}" + assert peak < PEAK_BUDGET + PEAK_PER_INPUT_BYTE * len(data), f"{vector['name']}: {path} peaked at {peak} bytes" + + @pytest.mark.parametrize("vector", VECTORS["accept_vectors"], ids=_vector_ids("accept_vectors")) + def test_accept_vector_decodes(self, path: str, vector: dict[str, Any]) -> None: + data = bytes.fromhex(vector["input_hex"]) + value = DECODE_PATHS[path](data) + depth = 0 + while isinstance(value, list): + depth, value = depth + 1, value[0] if value else None + assert depth == vector["nesting_depth"] + + +class TestOwnedBounds: + """SDK-local guards that go beyond the shared vectors.""" + + @pytest.mark.parametrize("path", DECODE_PATHS) + def test_worst_case_bombs_stay_bounded(self, path: str) -> None: + # (a) the LAB-2487 probe: 5000 x array16(10000) = 15 KB, 82 MB unguarded. + # (b) array32 headers claiming exactly len(data): defeats a per-collection cap of + # len(data); 10 KB -> 67 MB unguarded (depth x len x 8). + a16 = b"\xdc\x27\x10" * 5000 + a32 = (b"\xdd" + (8192).to_bytes(4, "big")) * 2048 + for bomb in (a16, a32): + _, err, peak = _peak_of(DECODE_PATHS[path], bomb) + assert isinstance(err, (ValueError, SerializationError)), f"{path}: {err!r}" + assert peak < PEAK_BUDGET + PEAK_PER_INPUT_BYTE * len(bomb), f"{path}: peak {peak}" + + def test_nesting_ceiling_is_exactly_the_pinned_constant(self) -> None: + # msgpack-python exposes no depth option; the C unpacker's fixed stack is the + # bound. If a release moves it, this fails and the constant + protocol note + # (spec/interop-mode.md → Decode bounds, 32 <= bound <= 1024) must be revisited. + assert MSGPACK_MAX_NESTING == 1024 + at_bound = b"\x91" * MSGPACK_MAX_NESTING + b"\xc0" + assert unpackb_bounded(at_bound) == json.loads("[" * MSGPACK_MAX_NESTING + "null" + "]" * MSGPACK_MAX_NESTING) + with pytest.raises(msgpack.exceptions.StackError): + unpackb_bounded(b"\x91" * (MSGPACK_MAX_NESTING + 1) + b"\xc0") + + def test_collection_caps_are_explicit_not_defaults(self) -> None: + # A header may not declare more than the input can back — even when the walk is + # bypassed by a structurally complete but oversize claim, unpackb's explicit caps hold. + with pytest.raises(ValueError, match="max_array_len"): + msgpack.unpackb(b"\xdc\x27\x10" + b"\xc0" * 3, max_array_len=6) + # and the real thing: over-claim is caught by the walk before any allocation + _, err, peak = _peak_of(unpackb_bounded, b"\xdc\x27\x10" + b"\xc0" * 3) + assert isinstance(err, ValueError) and "more elements/bytes than the input can back" in str(err) + assert peak < PEAK_BUDGET + + def test_legitimate_payloads_round_trip_unchanged(self) -> None: + # The bound must not touch real values: large flat collections, long str/bin, + # nested maps, and the columnar shapes the DataFrame path emits. + big = { + "s": "x" * 200_000, + "b": b"y" * 200_000, + "l": list(range(100_000)), + "m": {str(i): [i, {"i": i}] for i in range(1000)}, + } + packed = msgpack.packb(big, use_bin_type=True) + assert unpackb_bounded(packed, raw=False) == big + assert StandardSerializer().deserialize(StandardSerializer().serialize(big)[0]) == big + assert AutoSerializer().deserialize(AutoSerializer().serialize(big)[0]) == big + + def test_trailing_bytes_still_rejected(self) -> None: + with pytest.raises(msgpack.exceptions.ExtraData): + unpackb_bounded(b"\xc0\xc0") From d9eee872f0ab1cfc1708cb7a42fb612aa67f9bd5 Mon Sep 17 00:00:00 2001 From: Ray Walker Date: Wed, 2 Sep 2026 18:54:02 +1000 Subject: [PATCH 2/9] fix(serializers): apply LAB-2503 panel findings to the decode bound - StackError carries an empty message: normalise depth rejections to a ValueError naming MSGPACK_MAX_NESTING (StandardSerializer previously reported 'Failed to deserialize MessagePack data: ' with nothing after). - AutoSerializer's plain path no longer hides the decode-bound rejection behind the NumPy header error: the final SerializationError carries the envelope, msgpack and numpy reasons. - Docstring stops selling the explicit max_*_len caps as an independent bound (unreachable once the walk passes; defence in depth) and records the +1x transient copy Unpacker.feed costs. - Vendored vectors re-synced (array16/map16 bombs now claim 2000 < len so they discriminate for msgpack-python); redundant SDK-local tests cut. --- README.md | 2 +- src/cachekit/serializers/auto_serializer.py | 21 ++++-- src/cachekit/serializers/base.py | 40 ++++++----- .../unit/protocol/fixtures/decode-bounds.json | 14 ++-- tests/unit/protocol/test_decode_bounds.py | 66 ++++--------------- 5 files changed, 58 insertions(+), 85 deletions(-) diff --git a/README.md b/README.md index 94f5defb..be8b32e8 100644 --- a/README.md +++ b/README.md @@ -235,7 +235,7 @@ def test_cached_function(): - Connection pooling with thread affinity (+28% throughput) - Distributed locking prevents cache stampedes - Pluggable backend abstraction (Redis, CachekitIO, File, Memcached, custom) -- Untrusted-decode bounds: every cache read is a MessagePack decode of bytes the backend controls, so nesting depth and header-declared allocation are capped as cachekit-owned invariants (a forged nested-header entry is a bounded cache miss, not a memory blow-up) — verified against the protocol's shared [`decode-bounds.json`](https://github.com/cachekit-io/protocol/blob/main/test-vectors/decode-bounds.json) vectors +- Untrusted-decode bounds: nesting depth and header-declared allocation are capped on every cache read (a forged entry is a bounded cache miss), verified against the protocol's shared [`decode-bounds.json`](https://github.com/cachekit-io/protocol/blob/main/test-vectors/decode-bounds.json) vectors > [!NOTE] > All reliability features are **enabled by default** with `@cache.production`. Use `@cache.minimal` to disable them for maximum throughput. diff --git a/src/cachekit/serializers/auto_serializer.py b/src/cachekit/serializers/auto_serializer.py index a445d021..6daab12b 100644 --- a/src/cachekit/serializers/auto_serializer.py +++ b/src/cachekit/serializers/auto_serializer.py @@ -569,6 +569,7 @@ def deserialize(self, data: bytes | memoryview, metadata: Optional[Serialization return self._deserialize_series(unpacked_data) # For Rust-envelope formats, use the Rust layer + envelope_error: Exception | None = None if self.enable_integrity_checking: try: # Use Rust layer for decompression and validation @@ -578,7 +579,10 @@ def deserialize(self, data: bytes | memoryview, metadata: Optional[Serialization raise except Exception as e: # Not a ByteStorage envelope (e.g. written with integrity checking off): - # fall through to the Python-only paths below. + # fall through to the Python-only paths below, keeping the reason for the + # final error (a checksum mismatch also lands here — retrieve raises a plain + # ValueError for both; distinguishing them is a Rust-extension follow-up). + envelope_error = e logger.debug(f"Rust envelope parsing failed, falling back to Python-only deserialization: {e}") else: # The envelope verified (checksum matched), so its payload is exactly what was @@ -629,9 +633,18 @@ def deserialize(self, data: bytes | memoryview, metadata: Optional[Serialization except SerializationError: # Re-raise SerializationError (corruption detection) without swallowing raise - except Exception: - # If msgpack fails for other reasons, try NumPy-specific deserialization - return self._deserialize_numpy(data) + except Exception as msgpack_error: + # If msgpack fails for other reasons, try NumPy-specific deserialization — and if + # that fails too, report every reason: the msgpack one is the decode-bound + # rejection for a forged entry and must not vanish behind the NumPy header error. + try: + return self._deserialize_numpy(data) + except Exception as numpy_error: + raise SerializationError( + "Cache entry is not a decodable MessagePack or NumPy payload" + f"{f' (envelope: {envelope_error})' if envelope_error else ''}" + f" (msgpack: {msgpack_error}) (numpy: {numpy_error})" + ) from msgpack_error def _serialize_numpy(self, arr: np.ndarray) -> bytes: # type: ignore[name-defined] """Serialize a NumPy array into the ``NUMPY_RAW`` binary format. diff --git a/src/cachekit/serializers/base.py b/src/cachekit/serializers/base.py index 727b5598..90d2d3fa 100644 --- a/src/cachekit/serializers/base.py +++ b/src/cachekit/serializers/base.py @@ -344,36 +344,40 @@ def unpackb_bounded(data: bytes | bytearray | memoryview, **unpack_opts: Any) -> (``PyList_New(n)``) *before* decoding the children. Nested headers stack those allocations depth-first, so the library's per-collection default cap (``max_*_len = len(data)``) still permits ~8 x 1024 x len(data) bytes of - transient heap — measured 10 KB -> 67 MB. Two bounds close it: - - 1. A header-only structural walk (``Unpacker.skip``) runs first. It allocates - nothing, costs a fraction of the decode, and rejects a document that nests - deeper than :data:`MSGPACK_MAX_NESTING` (``StackError``) or declares more - elements/bytes than the input can back (``OutOfData``). Every element that - survives is backed by >= 1 input byte, so the real decode's pre-allocation - is bounded by ~8 x len(data). - 2. The collection/str/bin/ext caps are passed explicitly as ``len(data)`` — - the library's current default, made an owned invariant so a msgpack-python - change cannot silently lift it. - - Every rejection is a ``ValueError`` (``StackError``, ``FormatError``, - ``ExtraData``, or the over-claim ``ValueError`` raised here), which the read - paths already turn into a controlled cache miss. Trailing bytes are still - rejected by ``unpackb`` itself. + transient heap — measured 10 KB -> 67 MB. + + The bound is a header-only structural walk (``Unpacker.skip``) run before the + decode. It allocates nothing beyond one copy of the input (``feed`` copies into + the Unpacker's buffer — a known +1x transient; a zero-copy walk in the Rust + extension is the follow-up), costs a fraction of the decode, and rejects a + document that nests deeper than :data:`MSGPACK_MAX_NESTING` or declares more + elements/bytes than the input can back. Every element that survives is backed + by >= 1 input byte, so the real decode's pre-allocation is bounded by + ~8 x len(data). The explicit ``max_*_len=len(data)`` caps on ``unpackb`` are + unreachable once the walk passes; they are defence in depth against a + ``skip`` regression, not an independent bound. + + Every rejection is a ``ValueError`` (``FormatError``, ``ExtraData``, or the + depth / over-claim ``ValueError`` raised here), which the read paths already + turn into a controlled cache miss. Trailing bytes are still rejected by + ``unpackb`` itself. Examples: >>> unpackb_bounded(msgpack.packb({"a": [1, 2]}), raw=False) {'a': [1, 2]} - >>> unpackb_bounded(b"\\xdc\\x27\\x10" * 5000) # 15 KB nested-header bomb + >>> unpackb_bounded(b"\\xdc\\x07\\xd0" * 5000) # 15 KB nested-header bomb Traceback (most recent call last): ... - msgpack.exceptions.StackError + ValueError: Unpack failed: MessagePack document nests deeper than 1024 levels """ n = len(data) walker = msgpack.Unpacker(max_buffer_size=n) walker.feed(data) try: walker.skip() + except msgpack.exceptions.StackError as e: + # StackError carries an empty message; say what the bound is. + raise ValueError(f"Unpack failed: MessagePack document nests deeper than {MSGPACK_MAX_NESTING} levels") from e except msgpack.exceptions.OutOfData as e: # OutOfData is the one Unpacker error that is not a ValueError; normalise it to # the same contract unpackb uses for truncated input ("Unpack failed: incomplete input"). diff --git a/tests/unit/protocol/fixtures/decode-bounds.json b/tests/unit/protocol/fixtures/decode-bounds.json index 91ab717c..6a58520d 100644 --- a/tests/unit/protocol/fixtures/decode-bounds.json +++ b/tests/unit/protocol/fixtures/decode-bounds.json @@ -17,16 +17,16 @@ "reject_vectors": [ { "name": "nested_array16_depth_2048", - "description": "2048 nested array16 headers each claiming 10 000 elements, 0 backing bytes. The LAB-2487 amplifier shape: an eager decoder pre-allocates 10 000 slots per level before hitting EOF.", + "description": "2048 nested array16 headers each claiming 2000 elements, 0 backing bytes. The LAB-2487 amplifier shape: an eager decoder pre-allocates 2000 slots per level before hitting EOF. 2000 < input_len, so a per-collection cap of len(input) does NOT reject it.", "construction": { - "repeat_hex": "dc2710", + "repeat_hex": "dc07d0", "count": 2048, "suffix_hex": "" }, - "input_hex": "dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710dc2710", + "input_hex": "dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0", "input_len": 6144, "nesting_depth": 2048, - "declared_slots": 20480000, + "declared_slots": 4096000, "reject_reasons": [ "depth", "overclaim" @@ -53,14 +53,14 @@ "name": "nested_map16_depth_2048", "description": "Map twin of nested_array16_depth_2048 (map pre-allocation is typically larger per slot).", "construction": { - "repeat_hex": "de2710", + "repeat_hex": "de07d0", "count": 2048, "suffix_hex": "" }, - "input_hex": "de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710de2710", + "input_hex": "de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0", "input_len": 6144, "nesting_depth": 2048, - "declared_slots": 20480000, + "declared_slots": 8192000, "reject_reasons": [ "depth", "overclaim" diff --git a/tests/unit/protocol/test_decode_bounds.py b/tests/unit/protocol/test_decode_bounds.py index 6134b52c..d9187d45 100644 --- a/tests/unit/protocol/test_decode_bounds.py +++ b/tests/unit/protocol/test_decode_bounds.py @@ -1,23 +1,16 @@ """Untrusted-decode bounds (LAB-2503): protocol vectors + the SDK-local regression guard. -Every cache read is a MessagePack decode of bytes the backend controls. A -collection header costs 1-5 bytes but may declare up to 2**32-1 elements, and -msgpack-python's C unpacker pre-allocates the container before decoding the -children, so nested headers stack allocations depth-first. Before this guard the -decoder was bounded only by library defaults, and those defaults still allowed -~8 x 1024 x len(data) bytes of transient heap (measured 10 KB -> 67 MB; the -"82 MB hard ceiling" once reported was an artifact of the array16(10000) probe). - -Fixture: tests/unit/protocol/fixtures/decode-bounds.json, vendored from -cachekit-io/protocol test-vectors/decode-bounds.json -(sha256 864b7126986e9a2bd0dd50358018eda34fe2f70bca06ae9763e8ce6321f34b0a). -Regenerate ONLY by re-copying from the protocol repo — never by hand. - -What is pinned here, so a msgpack-python bump cannot silently move it: +Why the bound exists and how it works: the ``unpackb_bounded`` docstring in +``cachekit.serializers.base`` (the canonical home). This file pins, so a +msgpack-python bump cannot silently move it: - every reject vector is rejected on every decode path, with a bounded peak; - every accept vector decodes on every path (the bound cannot over-tighten); - the nesting ceiling is exactly MSGPACK_MAX_NESTING; - the read path turns a bomb into SerializationError (a controlled miss), not a crash. + +Fixture: tests/unit/protocol/fixtures/decode-bounds.json, vendored from +cachekit-io/protocol test-vectors/decode-bounds.json (sha256 pinned below). +Regenerate ONLY by re-copying from the protocol repo — never by hand. """ from __future__ import annotations @@ -42,13 +35,13 @@ from cachekit.serializers.wrapper import SerializationWrapper FIXTURE_PATH = Path(__file__).parent / "fixtures" / "decode-bounds.json" -FIXTURE_SHA256 = "864b7126986e9a2bd0dd50358018eda34fe2f70bca06ae9763e8ce6321f34b0a" # pragma: allowlist secret +FIXTURE_SHA256 = "fa8bc750a4911fe3663b9ab68f13438a3e924b6bc742e9f6763ca35ad2407476" # pragma: allowlist secret VECTORS = json.loads(FIXTURE_PATH.read_text(encoding="utf-8")) EXPECTED_COUNTS = {"reject_vectors": 10, "accept_vectors": 2} # Peak transient heap a rejected decode may cost: a small constant (Unpacker buffer) -# plus a few multiples of the input. The unguarded decoder peaks at ~5000x for the -# 15 KB bomb below, so this discriminates by three orders of magnitude. +# plus a few multiples of the input. Unguarded, the nested_array32_input_len vector +# peaks at ~8000x its input, so this discriminates by three orders of magnitude. PEAK_BUDGET = 2 * 1024 * 1024 PEAK_PER_INPUT_BYTE = 4 @@ -112,7 +105,6 @@ class TestProtocolVectors: @pytest.mark.parametrize("vector", VECTORS["reject_vectors"], ids=_vector_ids("reject_vectors")) def test_reject_vector_is_rejected_with_bounded_peak(self, path: str, vector: dict[str, Any]) -> None: data = bytes.fromhex(vector["input_hex"]) - assert len(data) == vector["input_len"] _, err, peak = _peak_of(DECODE_PATHS[path], data) assert err is not None, f"{vector['name']}: {path} decoded a reject vector" # Every rejection is a controlled error the read path maps to a cache miss. @@ -132,18 +124,6 @@ def test_accept_vector_decodes(self, path: str, vector: dict[str, Any]) -> None: class TestOwnedBounds: """SDK-local guards that go beyond the shared vectors.""" - @pytest.mark.parametrize("path", DECODE_PATHS) - def test_worst_case_bombs_stay_bounded(self, path: str) -> None: - # (a) the LAB-2487 probe: 5000 x array16(10000) = 15 KB, 82 MB unguarded. - # (b) array32 headers claiming exactly len(data): defeats a per-collection cap of - # len(data); 10 KB -> 67 MB unguarded (depth x len x 8). - a16 = b"\xdc\x27\x10" * 5000 - a32 = (b"\xdd" + (8192).to_bytes(4, "big")) * 2048 - for bomb in (a16, a32): - _, err, peak = _peak_of(DECODE_PATHS[path], bomb) - assert isinstance(err, (ValueError, SerializationError)), f"{path}: {err!r}" - assert peak < PEAK_BUDGET + PEAK_PER_INPUT_BYTE * len(bomb), f"{path}: peak {peak}" - def test_nesting_ceiling_is_exactly_the_pinned_constant(self) -> None: # msgpack-python exposes no depth option; the C unpacker's fixed stack is the # bound. If a release moves it, this fails and the constant + protocol note @@ -151,33 +131,9 @@ def test_nesting_ceiling_is_exactly_the_pinned_constant(self) -> None: assert MSGPACK_MAX_NESTING == 1024 at_bound = b"\x91" * MSGPACK_MAX_NESTING + b"\xc0" assert unpackb_bounded(at_bound) == json.loads("[" * MSGPACK_MAX_NESTING + "null" + "]" * MSGPACK_MAX_NESTING) - with pytest.raises(msgpack.exceptions.StackError): + with pytest.raises(ValueError, match=f"nests deeper than {MSGPACK_MAX_NESTING} levels"): unpackb_bounded(b"\x91" * (MSGPACK_MAX_NESTING + 1) + b"\xc0") - def test_collection_caps_are_explicit_not_defaults(self) -> None: - # A header may not declare more than the input can back — even when the walk is - # bypassed by a structurally complete but oversize claim, unpackb's explicit caps hold. - with pytest.raises(ValueError, match="max_array_len"): - msgpack.unpackb(b"\xdc\x27\x10" + b"\xc0" * 3, max_array_len=6) - # and the real thing: over-claim is caught by the walk before any allocation - _, err, peak = _peak_of(unpackb_bounded, b"\xdc\x27\x10" + b"\xc0" * 3) - assert isinstance(err, ValueError) and "more elements/bytes than the input can back" in str(err) - assert peak < PEAK_BUDGET - - def test_legitimate_payloads_round_trip_unchanged(self) -> None: - # The bound must not touch real values: large flat collections, long str/bin, - # nested maps, and the columnar shapes the DataFrame path emits. - big = { - "s": "x" * 200_000, - "b": b"y" * 200_000, - "l": list(range(100_000)), - "m": {str(i): [i, {"i": i}] for i in range(1000)}, - } - packed = msgpack.packb(big, use_bin_type=True) - assert unpackb_bounded(packed, raw=False) == big - assert StandardSerializer().deserialize(StandardSerializer().serialize(big)[0]) == big - assert AutoSerializer().deserialize(AutoSerializer().serialize(big)[0]) == big - def test_trailing_bytes_still_rejected(self) -> None: with pytest.raises(msgpack.exceptions.ExtraData): unpackb_bounded(b"\xc0\xc0") From 385798ab6aeb7a9d906d05eaf1a56be4624f57b3 Mon Sep 17 00:00:00 2001 From: Ray Walker Date: Thu, 3 Sep 2026 09:34:16 +1000 Subject: [PATCH 3/9] perf(serializers): zero-copy Rust header walk for the msgpack decode bound (LAB-2503) unpackb_bounded ran the structural check with msgpack.Unpacker.skip(), and Unpacker.feed() copies the whole input into its buffer first: a +1x transient on every cache read, which is what tripped the File-backend 3.5x allocation bound (4.00x) in CI. The walk now lives in the Rust extension as check_msgpack_structure: header-only, str/bin/ext payloads skipped by offset, zero-copy for bytes and for the read-only memoryview-of-bytes the read path carries, one u64 per open collection. It also tracks the global element budget (pending elements <= remaining bytes) alongside depth, so a 15 KB array16(2000) bomb is rejected at depth 8 instead of after a 1024-level walk. Measured: walk is 2-13% of decode time on collection-heavy payloads, ~0 on a 50 MiB bin, 0 B Python-heap peak. retrieve() and the walk share one bytes_view() borrow helper so the containment proof is written once. Kody: the broad excepts in AutoSerializer.deserialize now catch one named tuple of decode failures (_PAYLOAD_DECODE_ERRORS); RuntimeError for a missing optional dependency bubbles instead of reading as a corrupt entry. --- rust/src/lib.rs | 7 + rust/src/python_bindings.rs | 181 ++++++++++++++++---- src/cachekit/serializers/auto_serializer.py | 17 +- src/cachekit/serializers/base.py | 46 +++-- tests/unit/protocol/test_decode_bounds.py | 5 +- 5 files changed, 189 insertions(+), 67 deletions(-) diff --git a/rust/src/lib.rs b/rust/src/lib.rs index 9c451d0c..c1611dda 100644 --- a/rust/src/lib.rs +++ b/rust/src/lib.rs @@ -32,6 +32,13 @@ fn _rust_serializer(_py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_function(wrap_pyfunction!(python_bindings::checksum_py, m)?)?; m.add_function(wrap_pyfunction!(python_bindings::verify_checksum_py, m)?)?; + // Untrusted-decode structural bound (LAB-2503) — zero-copy header walk that + // serializers/base.py::unpackb_bounded runs before every msgpack.unpackb + m.add_function(wrap_pyfunction!( + python_bindings::check_msgpack_structure_py, + m + )?)?; + // Add encryption functionality if feature is enabled #[cfg(feature = "encryption")] { diff --git a/rust/src/python_bindings.rs b/rust/src/python_bindings.rs index d89cf553..5a1deeb4 100644 --- a/rust/src/python_bindings.rs +++ b/rust/src/python_bindings.rs @@ -44,6 +44,148 @@ fn borrowable_offset(buf: &PyBuffer, base: &Bound<'_, PyBytes>) -> Option { + /// A `bytes` object: immutable and kept alive by the Bound — zero-copy. + Bytes(Bound<'py, PyBytes>), + /// A read-only, C-contiguous window onto a `bytes` object (the `memoryview` + /// `SerializationWrapper.unwrap` produces), proven by `borrowable_offset` — zero-copy. + Window(Bound<'py, PyBytes>, usize, usize), + /// Mutable, non-`bytes`-backed, strided, or empty exporter: the only safe answer is a copy. + Owned(Vec), +} + +impl BytesView<'_> { + fn as_slice(&self) -> &[u8] { + match self { + BytesView::Bytes(b) => b.as_bytes(), + BytesView::Window(base, off, len) => &base.as_bytes()[*off..*off + *len], + BytesView::Owned(v) => v, + } + } +} + +/// Borrow `obj`'s bytes zero-copy when the BACKING STORAGE is provably immutable, else copy. +/// +/// `readonly()` describes the view, not the exporter (`memoryview(bytearray).toreadonly()` +/// passes it while another thread can still mutate the bytearray), and a PEP 688 +/// `__buffer__` exporter can name a decoy `bytes` in `.obj` — so the gate is the containment +/// proof in `borrowable_offset`, whose payoff is that the borrow is an ORDINARY SLICE of that +/// `bytes`: bounds-checked by Rust, no `unsafe`, nothing for a stale comment to misstate. +fn bytes_view<'py>(py: Python<'py>, obj: &Bound<'py, PyAny>) -> PyResult> { + if let Ok(b) = obj.cast::() { + return Ok(BytesView::Bytes(b.clone())); + } + let buf = PyBuffer::::get(obj)?; + let base = obj + .getattr("obj") + .ok() + .and_then(|base| base.cast_into::().ok()); + if let Some(base) = base { + if let Some(off) = borrowable_offset(&buf, &base) { + return Ok(BytesView::Window(base, off, buf.item_count())); + } + } + Ok(BytesView::Owned(buf.to_vec(py)?)) +} + +/// Structural bound for one untrusted MessagePack document (LAB-2503; protocol +/// spec/interop-mode.md → Decode bounds). Header-only: str/bin/ext payloads are skipped +/// by offset, never read, and nothing is allocated beyond one `u64` per open collection. +/// +/// Rejects, before any decoder pre-allocates a container: +/// - nesting deeper than `max_depth`; +/// - a header declaring more payload bytes than the input holds; +/// - more pending elements (across every open collection) than remaining bytes can back — +/// every element costs >= 1 byte, so a decoder's total container pre-allocation is then +/// bounded by the input length instead of by `depth × declared_len`; +/// - the reserved marker 0xc1 and input that ends mid-document. +/// +/// Trailing bytes after the root element are left to the decoder (`ExtraData`). +pub fn check_msgpack_structure(bytes: &[u8], max_depth: usize) -> Result<(), String> { + fn be(bytes: &[u8], pos: usize, width: usize) -> Result { + let end = pos + .checked_add(width) + .filter(|e| *e <= bytes.len()) + .ok_or_else(|| "ends inside a length prefix".to_owned())?; + Ok(bytes[pos..end] + .iter() + .fold(0u64, |acc, b| (acc << 8) | u64::from(*b))) + } + + let mut pos = 0usize; + let mut pending: u64 = 1; // elements owed across all open collections (the root is one) + let mut open: Vec = Vec::new(); // elements still owed per open collection = depth + while pending > 0 { + while open.last() == Some(&0) { + open.pop(); + } + let marker = *bytes + .get(pos) + .ok_or_else(|| "ends before the document is complete".to_owned())?; + pos += 1; + pending -= 1; + if let Some(innermost) = open.last_mut() { + *innermost -= 1; + } + // (length-prefix bytes, payload bytes after the prefix, child elements) + let (prefix, payload, children): (usize, u64, u64) = match marker { + 0x00..=0x7f | 0xc0 | 0xc2 | 0xc3 | 0xe0..=0xff => (0, 0, 0), + 0x80..=0x8f => (0, 0, 2 * u64::from(marker & 0x0f)), + 0x90..=0x9f => (0, 0, u64::from(marker & 0x0f)), + 0xa0..=0xbf => (0, u64::from(marker & 0x1f), 0), + 0xc1 => return Err("contains the reserved marker 0xc1".to_owned()), + 0xc4 | 0xd9 => (1, be(bytes, pos, 1)?, 0), + 0xc5 | 0xda => (2, be(bytes, pos, 2)?, 0), + 0xc6 | 0xdb => (4, be(bytes, pos, 4)?, 0), + 0xc7 => (1, be(bytes, pos, 1)? + 1, 0), // ext: length prefix, then type byte + data + 0xc8 => (2, be(bytes, pos, 2)? + 1, 0), + 0xc9 => (4, be(bytes, pos, 4)? + 1, 0), + 0xca..=0xd3 => (0, 1u64 << (marker & 0x03), 0), // f32/f64/u8..u64/i8..i64: 4,8,1,2,4,8,1,2,4,8 + 0xd4..=0xd8 => (0, 1 + (1u64 << (marker - 0xd4)), 0), // fixext: type byte + 1/2/4/8/16 + 0xdc => (2, 0, be(bytes, pos, 2)?), + 0xdd => (4, 0, be(bytes, pos, 4)?), + 0xde => (2, 0, 2 * be(bytes, pos, 2)?), + 0xdf => (4, 0, 2 * be(bytes, pos, 4)?), + }; + pos += prefix; + let remaining = (bytes.len() - pos) as u64; + if payload > remaining { + return Err("declares more bytes than the input holds".to_owned()); + } + pos += payload as usize; // <= remaining, so it fits usize + if children > 0 { + if open.len() >= max_depth { + return Err(format!("nests deeper than {max_depth} levels")); + } + open.push(children); + } + pending += children; + if pending > remaining - payload { + return Err("declares more elements than the input can back".to_owned()); + } + } + Ok(()) +} + +/// Reject a MessagePack document whose headers would make decoding it allocate out of +/// proportion to its size — see `check_msgpack_structure`. Zero-copy for `bytes` and for +/// read-only `memoryview`s of `bytes`; raises ValueError naming the violated bound. +#[pyfunction] +#[pyo3(name = "check_msgpack_structure")] +pub fn check_msgpack_structure_py( + py: Python<'_>, + data: &Bound<'_, PyAny>, + max_depth: usize, +) -> PyResult<()> { + let view = bytes_view(py, data)?; + check_msgpack_structure(view.as_slice(), max_depth).map_err(|what| { + PyValueError::new_err(format!("Unpack failed: MessagePack document {what}")) + }) +} + #[pymethods] impl PyByteStorage { #[new] @@ -86,41 +228,10 @@ impl PyByteStorage { py: Python, envelope_bytes: &Bound<'_, PyAny>, ) -> PyResult<(Vec, String)> { - let owned: Vec; - let buf: PyBuffer; - let base_bytes: Option>; - let data: &[u8] = if let Ok(b) = envelope_bytes.cast::() { - // `bytes` is immutable and kept alive by the Bound for the whole call: - // a zero-copy borrow with no data-race exposure. - b.as_bytes() - } else { - buf = PyBuffer::get(envelope_bytes)?; - // Borrowing across the GIL release below is only sound when the BACKING - // STORAGE is immutable — readonly() describes the view, not the exporter - // (memoryview(bytearray).toreadonly() passes it while another thread can - // still mutate the bytearray). Attribute trust is not enough either: a - // PEP 688 __buffer__ exporter can name a decoy `bytes` in `.obj`. So the - // gate is a containment proof (borrowable_offset), and its payoff is that - // the borrow becomes expressible as an ORDINARY SLICE of that `bytes` — - // bounds-checked by Rust, no `unsafe`, nothing for a stale comment to - // misstate. Anything unproven falls back to a copy. - base_bytes = envelope_bytes - .getattr("obj") - .ok() - .and_then(|base| base.cast_into::().ok()); - let borrowed = base_bytes.as_ref().and_then(|base| { - borrowable_offset(&buf, base) - .map(|off| &base.as_bytes()[off..off + buf.item_count()]) - }); - match borrowed { - Some(slice) => slice, - None => { - // Mutable, non-bytes-backed, non-contiguous, or empty exporter. - owned = buf.to_vec(py)?; - &owned - } - } - }; + // Borrowing across the GIL release below is only sound when the backing storage + // is immutable — bytes_view proves that or copies (see its doc). + let view = bytes_view(py, envelope_bytes)?; + let data = view.as_slice(); // Detach from the GIL for decompression + checksum (see store()). py.detach(|| self.inner.retrieve(data)) .map_err(|e| PyValueError::new_err(format!("Retrieval failed: {}", e))) diff --git a/src/cachekit/serializers/auto_serializer.py b/src/cachekit/serializers/auto_serializer.py index 6daab12b..136e93b8 100644 --- a/src/cachekit/serializers/auto_serializer.py +++ b/src/cachekit/serializers/auto_serializer.py @@ -64,6 +64,14 @@ logger = logging.getLogger(__name__) +# What a corrupted or forged payload can make the decode helpers raise: msgpack's own +# errors (ValueError subclasses, plus the UnpackException family), the object-hook +# restorers on a malformed marker (ValueError/TypeError/AttributeError), and the +# DataFrame/Series reconstructors indexing a dict that is not the shape they wrote +# (KeyError/TypeError/ValueError). Anything else — above all RuntimeError for a missing +# optional dependency — is an environment fault, not a bad cache entry, and must bubble. +_PAYLOAD_DECODE_ERRORS = (msgpack.exceptions.UnpackException, ValueError, TypeError, KeyError, AttributeError) + # Error message constants for unsupported types PYDANTIC_ERROR_MESSAGE = ( "AutoSerializer does not support Pydantic models. Use .model_dump() to convert to dict: result = model.model_dump()" @@ -603,7 +611,7 @@ def deserialize(self, data: bytes | memoryview, metadata: Optional[Serialization return unpackb_bounded(original_data, **self._msgpack_unpack_opts) except SerializationError: raise - except Exception as e: + except _PAYLOAD_DECODE_ERRORS as e: raise SerializationError( f"Cache entry payload failed to decode inside a verified envelope (format={detected_format!r}): {e}" ) from e @@ -633,13 +641,13 @@ def deserialize(self, data: bytes | memoryview, metadata: Optional[Serialization except SerializationError: # Re-raise SerializationError (corruption detection) without swallowing raise - except Exception as msgpack_error: + except _PAYLOAD_DECODE_ERRORS as msgpack_error: # If msgpack fails for other reasons, try NumPy-specific deserialization — and if # that fails too, report every reason: the msgpack one is the decode-bound # rejection for a forged entry and must not vanish behind the NumPy header error. try: return self._deserialize_numpy(data) - except Exception as numpy_error: + except (SerializationError, *_PAYLOAD_DECODE_ERRORS) as numpy_error: raise SerializationError( "Cache entry is not a decodable MessagePack or NumPy payload" f"{f' (envelope: {envelope_error})' if envelope_error else ''}" @@ -927,8 +935,7 @@ def validate_data(self, data: bytes) -> bool: try: unpackb_bounded(data, **self._msgpack_unpack_opts) return True - except (msgpack.exceptions.UnpackException, ValueError, TypeError, AttributeError): - # AttributeError can occur when datetime_object_hook tries to restore invalid data + except _PAYLOAD_DECODE_ERRORS: return False diff --git a/src/cachekit/serializers/base.py b/src/cachekit/serializers/base.py index 90d2d3fa..b8246edc 100644 --- a/src/cachekit/serializers/base.py +++ b/src/cachekit/serializers/base.py @@ -10,6 +10,8 @@ import msgpack +from cachekit._rust_serializer import check_msgpack_structure + @runtime_checkable class SerializerProtocol(Protocol): @@ -346,21 +348,23 @@ def unpackb_bounded(data: bytes | bytearray | memoryview, **unpack_opts: Any) -> (``max_*_len = len(data)``) still permits ~8 x 1024 x len(data) bytes of transient heap — measured 10 KB -> 67 MB. - The bound is a header-only structural walk (``Unpacker.skip``) run before the - decode. It allocates nothing beyond one copy of the input (``feed`` copies into - the Unpacker's buffer — a known +1x transient; a zero-copy walk in the Rust - extension is the follow-up), costs a fraction of the decode, and rejects a - document that nests deeper than :data:`MSGPACK_MAX_NESTING` or declares more - elements/bytes than the input can back. Every element that survives is backed - by >= 1 input byte, so the real decode's pre-allocation is bounded by - ~8 x len(data). The explicit ``max_*_len=len(data)`` caps on ``unpackb`` are - unreachable once the walk passes; they are defence in depth against a - ``skip`` regression, not an independent bound. + The bound is a header-only structural walk in the Rust extension + (``check_msgpack_structure``) run before the decode. It reads the input in + place — zero-copy for ``bytes`` and for the read-only ``memoryview`` of + ``bytes`` the read path carries — skips str/bin/ext payloads by offset, and + allocates one integer per open collection. It rejects a document that nests + deeper than :data:`MSGPACK_MAX_NESTING` or whose open headers declare more + elements or bytes than the remaining input can back. Every element that + survives is backed by >= 1 input byte, so the real decode's total container + pre-allocation is bounded by len(data) rather than by depth x declared length. + The explicit ``max_*_len=len(data)`` caps on ``unpackb`` are unreachable once + the walk passes; they are defence in depth against a walk regression, not an + independent bound. Every rejection is a ``ValueError`` (``FormatError``, ``ExtraData``, or the - depth / over-claim ``ValueError`` raised here), which the read paths already - turn into a controlled cache miss. Trailing bytes are still rejected by - ``unpackb`` itself. + walk's own ``ValueError`` naming the violated bound), which the read paths + already turn into a controlled cache miss. Trailing bytes are still rejected + by ``unpackb`` itself. Examples: >>> unpackb_bounded(msgpack.packb({"a": [1, 2]}), raw=False) @@ -368,18 +372,12 @@ def unpackb_bounded(data: bytes | bytearray | memoryview, **unpack_opts: Any) -> >>> unpackb_bounded(b"\\xdc\\x07\\xd0" * 5000) # 15 KB nested-header bomb Traceback (most recent call last): ... + ValueError: Unpack failed: MessagePack document declares more elements than the input can back + >>> unpackb_bounded(b"\\x91" * 1025 + b"\\xc0") # one level past the ceiling + Traceback (most recent call last): + ... ValueError: Unpack failed: MessagePack document nests deeper than 1024 levels """ n = len(data) - walker = msgpack.Unpacker(max_buffer_size=n) - walker.feed(data) - try: - walker.skip() - except msgpack.exceptions.StackError as e: - # StackError carries an empty message; say what the bound is. - raise ValueError(f"Unpack failed: MessagePack document nests deeper than {MSGPACK_MAX_NESTING} levels") from e - except msgpack.exceptions.OutOfData as e: - # OutOfData is the one Unpacker error that is not a ValueError; normalise it to - # the same contract unpackb uses for truncated input ("Unpack failed: incomplete input"). - raise ValueError("Unpack failed: MessagePack document declares more elements/bytes than the input can back") from e + check_msgpack_structure(data, MSGPACK_MAX_NESTING) return msgpack.unpackb(data, max_str_len=n, max_bin_len=n, max_array_len=n, max_map_len=n, max_ext_len=n, **unpack_opts) diff --git a/tests/unit/protocol/test_decode_bounds.py b/tests/unit/protocol/test_decode_bounds.py index d9187d45..728641ee 100644 --- a/tests/unit/protocol/test_decode_bounds.py +++ b/tests/unit/protocol/test_decode_bounds.py @@ -83,7 +83,8 @@ def _peak_of(fn: Callable[..., Any], *args: Any) -> tuple[Any, BaseException | N tracemalloc.start() try: return fn(*args), None, tracemalloc.get_traced_memory()[1] - except Exception as e: # noqa: BLE001 — the exception type is what we assert on + except (ValueError, SerializationError) as e: + # The only rejections the read path maps to a controlled miss; any other type propagates. return None, e, tracemalloc.get_traced_memory()[1] finally: tracemalloc.stop() @@ -107,8 +108,6 @@ def test_reject_vector_is_rejected_with_bounded_peak(self, path: str, vector: di data = bytes.fromhex(vector["input_hex"]) _, err, peak = _peak_of(DECODE_PATHS[path], data) assert err is not None, f"{vector['name']}: {path} decoded a reject vector" - # Every rejection is a controlled error the read path maps to a cache miss. - assert isinstance(err, (ValueError, SerializationError)), f"{vector['name']}: {path} raised {type(err).__name__}" assert peak < PEAK_BUDGET + PEAK_PER_INPUT_BYTE * len(data), f"{vector['name']}: {path} peaked at {peak} bytes" @pytest.mark.parametrize("vector", VECTORS["accept_vectors"], ids=_vector_ids("accept_vectors")) From 0ba205bfb462c1511aeb3d9c303b67c144a409ac Mon Sep 17 00:00:00 2001 From: Ray Walker Date: Thu, 3 Sep 2026 09:49:50 +1000 Subject: [PATCH 4/9] fix(serializers): apply LAB-2503 panel round 2 to the Rust decode walk MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Move the pure check_msgpack_structure into rust/src/msgpack_bounds.rs (not gated on the python feature) and stop the crate headers claiming all logic lives in cachekit-core. - PAYLOAD_DECODE_ERRORS now lives in serializers/base.py beside the function that raises them and is shared by AutoSerializer and StandardSerializer. Adds OverflowError (np.frombuffer on a forged ndarray itemsize escaped deserialize as a bare exception — reproduced) and BufferError (non-u8 exporter at the PyO3 boundary, LAB-770); drops UnpackException, which unpackb never raises. _deserialize_numpy also catches the TypeError a forged dtype string produces. - BytesView folded to Borrowed/Owned: a bytes object is a window at offset 0. - MSGPACK_MAX_NESTING comment and the at-bound test comment now say what the constant is (cachekit's ceiling enforced by the walk, bounded above by the C unpacker stack) instead of the pre-walk StackError story. - Regression test: a forged ndarray payload is a SerializationError on both the plain and verified-envelope paths. --- rust/src/lib.rs | 6 +- rust/src/msgpack_bounds.rs | 82 ++++++++++++++ rust/src/python_bindings.rs | 101 ++---------------- src/cachekit/serializers/auto_serializer.py | 21 ++-- src/cachekit/serializers/base.py | 41 ++++--- .../serializers/standard_serializer.py | 7 +- tests/unit/protocol/test_decode_bounds.py | 10 +- tests/unit/test_auto_serializer_new_types.py | 22 ++++ 8 files changed, 157 insertions(+), 133 deletions(-) create mode 100644 rust/src/msgpack_bounds.rs diff --git a/rust/src/lib.rs b/rust/src/lib.rs index c1611dda..65c3aea8 100644 --- a/rust/src/lib.rs +++ b/rust/src/lib.rs @@ -1,11 +1,15 @@ //! `PyO3` bindings for `cachekit-core` //! //! This crate provides thin Python wrappers around the cachekit-core library. -//! All business logic lives in cachekit-core; this crate only handles Python FFI. +//! Business logic lives in cachekit-core, with one SDK-owned exception: the untrusted +//! msgpack decode bound in `msgpack_bounds` (LAB-2503), pending a core-shared walk. // Re-export core types for use in Python bindings pub use cachekit_core::{ByteStorage, OperationMetrics, StorageEnvelope}; +/// Untrusted msgpack structural bound — pure Rust, not gated on `python` +pub mod msgpack_bounds; + #[cfg(feature = "encryption")] pub use cachekit_core::{ derive_domain_key, diff --git a/rust/src/msgpack_bounds.rs b/rust/src/msgpack_bounds.rs new file mode 100644 index 00000000..e791a42f --- /dev/null +++ b/rust/src/msgpack_bounds.rs @@ -0,0 +1,82 @@ +//! Structural bound for untrusted MessagePack (LAB-2503; protocol spec/interop-mode.md → +//! Decode bounds). The one algorithm this crate owns rather than delegates to cachekit-core; +//! a core-shared walk usable from py/rs/wasm is the follow-up. Mirrors the opcode table of +//! cachekit-rs `check_structure` so the two SDKs reject the same documents. + +/// Header-only walk over one MessagePack document: str/bin/ext payloads are skipped by +/// offset, never read, and nothing is allocated beyond one `u64` per open collection. +/// +/// Rejects, before any decoder pre-allocates a container: +/// - nesting deeper than `max_depth`; +/// - a header declaring more payload bytes than the input holds; +/// - more pending elements (across every open collection) than remaining bytes can back — +/// every element costs >= 1 byte, so a decoder's total container pre-allocation is then +/// bounded by the input length instead of by `depth × declared_len`; +/// - the reserved marker 0xc1 and input that ends mid-document. +/// +/// Trailing bytes after the root element are left to the decoder (`ExtraData`). +pub fn check_msgpack_structure(bytes: &[u8], max_depth: usize) -> Result<(), String> { + fn be(bytes: &[u8], pos: usize, width: usize) -> Result { + let end = pos + .checked_add(width) + .filter(|e| *e <= bytes.len()) + .ok_or_else(|| "ends inside a length prefix".to_owned())?; + Ok(bytes[pos..end] + .iter() + .fold(0u64, |acc, b| (acc << 8) | u64::from(*b))) + } + + let mut pos = 0usize; + let mut pending: u64 = 1; // elements owed across all open collections (the root is one) + let mut open: Vec = Vec::new(); // elements still owed per open collection = depth + while pending > 0 { + while open.last() == Some(&0) { + open.pop(); + } + let marker = *bytes + .get(pos) + .ok_or_else(|| "ends before the document is complete".to_owned())?; + pos += 1; + pending -= 1; + if let Some(innermost) = open.last_mut() { + *innermost -= 1; + } + // (length-prefix bytes, payload bytes after the prefix, child elements) + let (prefix, payload, children): (usize, u64, u64) = match marker { + 0x00..=0x7f | 0xc0 | 0xc2 | 0xc3 | 0xe0..=0xff => (0, 0, 0), + 0x80..=0x8f => (0, 0, 2 * u64::from(marker & 0x0f)), + 0x90..=0x9f => (0, 0, u64::from(marker & 0x0f)), + 0xa0..=0xbf => (0, u64::from(marker & 0x1f), 0), + 0xc1 => return Err("contains the reserved marker 0xc1".to_owned()), + 0xc4 | 0xd9 => (1, be(bytes, pos, 1)?, 0), + 0xc5 | 0xda => (2, be(bytes, pos, 2)?, 0), + 0xc6 | 0xdb => (4, be(bytes, pos, 4)?, 0), + 0xc7 => (1, be(bytes, pos, 1)? + 1, 0), // ext: length prefix, then type byte + data + 0xc8 => (2, be(bytes, pos, 2)? + 1, 0), + 0xc9 => (4, be(bytes, pos, 4)? + 1, 0), + 0xca..=0xd3 => (0, 1u64 << (marker & 0x03), 0), // f32/f64/u8..u64/i8..i64: 4,8,1,2,4,8,1,2,4,8 + 0xd4..=0xd8 => (0, 1 + (1u64 << (marker - 0xd4)), 0), // fixext: type byte + 1/2/4/8/16 + 0xdc => (2, 0, be(bytes, pos, 2)?), + 0xdd => (4, 0, be(bytes, pos, 4)?), + 0xde => (2, 0, 2 * be(bytes, pos, 2)?), + 0xdf => (4, 0, 2 * be(bytes, pos, 4)?), + }; + pos += prefix; + let remaining = (bytes.len() - pos) as u64; + if payload > remaining { + return Err("declares more bytes than the input holds".to_owned()); + } + pos += payload as usize; // <= remaining, so it fits usize + if children > 0 { + if open.len() >= max_depth { + return Err(format!("nests deeper than {max_depth} levels")); + } + open.push(children); + } + pending += children; + if pending > remaining - payload { + return Err("declares more elements than the input can back".to_owned()); + } + } + Ok(()) +} diff --git a/rust/src/python_bindings.rs b/rust/src/python_bindings.rs index 5a1deeb4..d15c96e7 100644 --- a/rust/src/python_bindings.rs +++ b/rust/src/python_bindings.rs @@ -1,8 +1,10 @@ //! Python bindings for cachekit-core //! -//! This module provides thin PyO3 wrappers around cachekit-core functionality. -//! All business logic is delegated to cachekit-core. +//! This module provides thin PyO3 wrappers around cachekit-core functionality, plus the +//! buffer-borrow helper they share. Business logic lives in cachekit-core, except the +//! SDK-owned msgpack decode bound in `crate::msgpack_bounds`. +use crate::msgpack_bounds::check_msgpack_structure; use cachekit_core::ByteStorage; use pyo3::buffer::PyBuffer; use pyo3::exceptions::PyValueError; @@ -48,11 +50,10 @@ fn borrowable_offset(buf: &PyBuffer, base: &Bound<'_, PyBytes>) -> Option { - /// A `bytes` object: immutable and kept alive by the Bound — zero-copy. - Bytes(Bound<'py, PyBytes>), - /// A read-only, C-contiguous window onto a `bytes` object (the `memoryview` - /// `SerializationWrapper.unwrap` produces), proven by `borrowable_offset` — zero-copy. - Window(Bound<'py, PyBytes>, usize, usize), + /// `(base, offset, len)`: a window onto an immutable `bytes` object kept alive by the + /// Bound — the whole object, or the read-only C-contiguous `memoryview` of it that + /// `SerializationWrapper.unwrap` produces, proven by `borrowable_offset`. Zero-copy. + Borrowed(Bound<'py, PyBytes>, usize, usize), /// Mutable, non-`bytes`-backed, strided, or empty exporter: the only safe answer is a copy. Owned(Vec), } @@ -60,8 +61,7 @@ enum BytesView<'py> { impl BytesView<'_> { fn as_slice(&self) -> &[u8] { match self { - BytesView::Bytes(b) => b.as_bytes(), - BytesView::Window(base, off, len) => &base.as_bytes()[*off..*off + *len], + BytesView::Borrowed(base, off, len) => &base.as_bytes()[*off..*off + *len], BytesView::Owned(v) => v, } } @@ -76,7 +76,7 @@ impl BytesView<'_> { /// `bytes`: bounds-checked by Rust, no `unsafe`, nothing for a stale comment to misstate. fn bytes_view<'py>(py: Python<'py>, obj: &Bound<'py, PyAny>) -> PyResult> { if let Ok(b) = obj.cast::() { - return Ok(BytesView::Bytes(b.clone())); + return Ok(BytesView::Borrowed(b.clone(), 0, b.len()?)); } let buf = PyBuffer::::get(obj)?; let base = obj @@ -85,91 +85,12 @@ fn bytes_view<'py>(py: Python<'py>, obj: &Bound<'py, PyAny>) -> PyResult().ok()); if let Some(base) = base { if let Some(off) = borrowable_offset(&buf, &base) { - return Ok(BytesView::Window(base, off, buf.item_count())); + return Ok(BytesView::Borrowed(base, off, buf.item_count())); } } Ok(BytesView::Owned(buf.to_vec(py)?)) } -/// Structural bound for one untrusted MessagePack document (LAB-2503; protocol -/// spec/interop-mode.md → Decode bounds). Header-only: str/bin/ext payloads are skipped -/// by offset, never read, and nothing is allocated beyond one `u64` per open collection. -/// -/// Rejects, before any decoder pre-allocates a container: -/// - nesting deeper than `max_depth`; -/// - a header declaring more payload bytes than the input holds; -/// - more pending elements (across every open collection) than remaining bytes can back — -/// every element costs >= 1 byte, so a decoder's total container pre-allocation is then -/// bounded by the input length instead of by `depth × declared_len`; -/// - the reserved marker 0xc1 and input that ends mid-document. -/// -/// Trailing bytes after the root element are left to the decoder (`ExtraData`). -pub fn check_msgpack_structure(bytes: &[u8], max_depth: usize) -> Result<(), String> { - fn be(bytes: &[u8], pos: usize, width: usize) -> Result { - let end = pos - .checked_add(width) - .filter(|e| *e <= bytes.len()) - .ok_or_else(|| "ends inside a length prefix".to_owned())?; - Ok(bytes[pos..end] - .iter() - .fold(0u64, |acc, b| (acc << 8) | u64::from(*b))) - } - - let mut pos = 0usize; - let mut pending: u64 = 1; // elements owed across all open collections (the root is one) - let mut open: Vec = Vec::new(); // elements still owed per open collection = depth - while pending > 0 { - while open.last() == Some(&0) { - open.pop(); - } - let marker = *bytes - .get(pos) - .ok_or_else(|| "ends before the document is complete".to_owned())?; - pos += 1; - pending -= 1; - if let Some(innermost) = open.last_mut() { - *innermost -= 1; - } - // (length-prefix bytes, payload bytes after the prefix, child elements) - let (prefix, payload, children): (usize, u64, u64) = match marker { - 0x00..=0x7f | 0xc0 | 0xc2 | 0xc3 | 0xe0..=0xff => (0, 0, 0), - 0x80..=0x8f => (0, 0, 2 * u64::from(marker & 0x0f)), - 0x90..=0x9f => (0, 0, u64::from(marker & 0x0f)), - 0xa0..=0xbf => (0, u64::from(marker & 0x1f), 0), - 0xc1 => return Err("contains the reserved marker 0xc1".to_owned()), - 0xc4 | 0xd9 => (1, be(bytes, pos, 1)?, 0), - 0xc5 | 0xda => (2, be(bytes, pos, 2)?, 0), - 0xc6 | 0xdb => (4, be(bytes, pos, 4)?, 0), - 0xc7 => (1, be(bytes, pos, 1)? + 1, 0), // ext: length prefix, then type byte + data - 0xc8 => (2, be(bytes, pos, 2)? + 1, 0), - 0xc9 => (4, be(bytes, pos, 4)? + 1, 0), - 0xca..=0xd3 => (0, 1u64 << (marker & 0x03), 0), // f32/f64/u8..u64/i8..i64: 4,8,1,2,4,8,1,2,4,8 - 0xd4..=0xd8 => (0, 1 + (1u64 << (marker - 0xd4)), 0), // fixext: type byte + 1/2/4/8/16 - 0xdc => (2, 0, be(bytes, pos, 2)?), - 0xdd => (4, 0, be(bytes, pos, 4)?), - 0xde => (2, 0, 2 * be(bytes, pos, 2)?), - 0xdf => (4, 0, 2 * be(bytes, pos, 4)?), - }; - pos += prefix; - let remaining = (bytes.len() - pos) as u64; - if payload > remaining { - return Err("declares more bytes than the input holds".to_owned()); - } - pos += payload as usize; // <= remaining, so it fits usize - if children > 0 { - if open.len() >= max_depth { - return Err(format!("nests deeper than {max_depth} levels")); - } - open.push(children); - } - pending += children; - if pending > remaining - payload { - return Err("declares more elements than the input can back".to_owned()); - } - } - Ok(()) -} - /// Reject a MessagePack document whose headers would make decoding it allocate out of /// proportion to its size — see `check_msgpack_structure`. Zero-copy for `bytes` and for /// read-only `memoryview`s of `bytes`; raises ValueError naming the violated bound. diff --git a/src/cachekit/serializers/auto_serializer.py b/src/cachekit/serializers/auto_serializer.py index 136e93b8..316cafbe 100644 --- a/src/cachekit/serializers/auto_serializer.py +++ b/src/cachekit/serializers/auto_serializer.py @@ -60,18 +60,10 @@ from cachekit._rust_serializer import ByteStorage -from .base import SerializationError, SerializationFormat, SerializationMetadata, unpackb_bounded +from .base import PAYLOAD_DECODE_ERRORS, SerializationError, SerializationFormat, SerializationMetadata, unpackb_bounded logger = logging.getLogger(__name__) -# What a corrupted or forged payload can make the decode helpers raise: msgpack's own -# errors (ValueError subclasses, plus the UnpackException family), the object-hook -# restorers on a malformed marker (ValueError/TypeError/AttributeError), and the -# DataFrame/Series reconstructors indexing a dict that is not the shape they wrote -# (KeyError/TypeError/ValueError). Anything else — above all RuntimeError for a missing -# optional dependency — is an environment fault, not a bad cache entry, and must bubble. -_PAYLOAD_DECODE_ERRORS = (msgpack.exceptions.UnpackException, ValueError, TypeError, KeyError, AttributeError) - # Error message constants for unsupported types PYDANTIC_ERROR_MESSAGE = ( "AutoSerializer does not support Pydantic models. Use .model_dump() to convert to dict: result = model.model_dump()" @@ -611,7 +603,7 @@ def deserialize(self, data: bytes | memoryview, metadata: Optional[Serialization return unpackb_bounded(original_data, **self._msgpack_unpack_opts) except SerializationError: raise - except _PAYLOAD_DECODE_ERRORS as e: + except PAYLOAD_DECODE_ERRORS as e: raise SerializationError( f"Cache entry payload failed to decode inside a verified envelope (format={detected_format!r}): {e}" ) from e @@ -641,13 +633,13 @@ def deserialize(self, data: bytes | memoryview, metadata: Optional[Serialization except SerializationError: # Re-raise SerializationError (corruption detection) without swallowing raise - except _PAYLOAD_DECODE_ERRORS as msgpack_error: + except PAYLOAD_DECODE_ERRORS as msgpack_error: # If msgpack fails for other reasons, try NumPy-specific deserialization — and if # that fails too, report every reason: the msgpack one is the decode-bound # rejection for a forged entry and must not vanish behind the NumPy header error. try: return self._deserialize_numpy(data) - except (SerializationError, *_PAYLOAD_DECODE_ERRORS) as numpy_error: + except (SerializationError, *PAYLOAD_DECODE_ERRORS) as numpy_error: raise SerializationError( "Cache entry is not a decodable MessagePack or NumPy payload" f"{f' (envelope: {envelope_error})' if envelope_error else ''}" @@ -748,7 +740,8 @@ def _deserialize_numpy(self, data: bytes) -> np.ndarray: raw_bytes = data[offset:] arr = np.frombuffer(raw_bytes, dtype=dtype_str).copy() return arr.reshape(shape) - except (ValueError, IndexError, UnicodeDecodeError) as e: + except (ValueError, TypeError, IndexError) as e: + # TypeError: np.frombuffer on a forged dtype string; UnicodeDecodeError is a ValueError. raise SerializationError(f"Failed to deserialize NumPy array: {e}") from e def _serialize_dataframe(self, df: pd.DataFrame) -> bytes: @@ -935,7 +928,7 @@ def validate_data(self, data: bytes) -> bool: try: unpackb_bounded(data, **self._msgpack_unpack_opts) return True - except _PAYLOAD_DECODE_ERRORS: + except PAYLOAD_DECODE_ERRORS: return False diff --git a/src/cachekit/serializers/base.py b/src/cachekit/serializers/base.py index b8246edc..e77206b7 100644 --- a/src/cachekit/serializers/base.py +++ b/src/cachekit/serializers/base.py @@ -330,13 +330,23 @@ class SuspiciousCacheEntryError(SerializationError): # Owned untrusted-decode bounds (LAB-2503; protocol spec/interop-mode.md → Decode bounds) # --------------------------------------------------------------------------- -#: Nesting depth msgpack-python's C unpacker accepts before raising ``StackError``. -#: Not configurable through its API — pinned here and regression-tested -#: (tests/unit/protocol/test_decode_bounds.py) so a dependency bump that moves it -#: fails a test instead of silently changing the decode ceiling. The protocol -#: requires every SDK's bound to sit in 32..=1024. +#: cachekit's own nesting ceiling, enforced by the Rust ``check_msgpack_structure`` +#: walk before msgpack-python ever sees the document. Two constraints pin it: the +#: protocol requires every SDK's bound to sit in 32..=1024, and it must not exceed +#: msgpack-python's C unpacker stack (a document at the ceiling has to decode after +#: passing the walk; tests/unit/protocol/test_decode_bounds.py checks exactly that). MSGPACK_MAX_NESTING = 1024 +#: Everything a corrupted or forged payload can make a decode raise, for serializers +#: to turn into ``SerializationError``. msgpack's own errors are ``ValueError`` +#: subclasses; the AutoSerializer object hook and the NumPy/DataFrame/Series +#: reconstructors add ``TypeError`` / ``OverflowError`` (``np.frombuffer`` on a forged +#: dtype or itemsize), ``KeyError`` / ``AttributeError`` (indexing a dict that is not +#: the shape they wrote); ``BufferError`` is a non-u8 buffer exporter rejected at the +#: PyO3 boundary (LAB-770). Anything else — above all ``RuntimeError`` for a missing +#: optional dependency — is an environment fault, not a bad cache entry, and must bubble. +PAYLOAD_DECODE_ERRORS = (ValueError, TypeError, KeyError, AttributeError, OverflowError, BufferError) + def unpackb_bounded(data: bytes | bytearray | memoryview, **unpack_opts: Any) -> Any: """Decode one untrusted MessagePack document under cachekit-owned bounds. @@ -348,18 +358,15 @@ def unpackb_bounded(data: bytes | bytearray | memoryview, **unpack_opts: Any) -> (``max_*_len = len(data)``) still permits ~8 x 1024 x len(data) bytes of transient heap — measured 10 KB -> 67 MB. - The bound is a header-only structural walk in the Rust extension - (``check_msgpack_structure``) run before the decode. It reads the input in - place — zero-copy for ``bytes`` and for the read-only ``memoryview`` of - ``bytes`` the read path carries — skips str/bin/ext payloads by offset, and - allocates one integer per open collection. It rejects a document that nests - deeper than :data:`MSGPACK_MAX_NESTING` or whose open headers declare more - elements or bytes than the remaining input can back. Every element that - survives is backed by >= 1 input byte, so the real decode's total container - pre-allocation is bounded by len(data) rather than by depth x declared length. - The explicit ``max_*_len=len(data)`` caps on ``unpackb`` are unreachable once - the walk passes; they are defence in depth against a walk regression, not an - independent bound. + The bound is the Rust extension's zero-copy, header-only walk + (``check_msgpack_structure``, documented there) run before the decode. It + rejects a document that nests deeper than :data:`MSGPACK_MAX_NESTING` or whose + open headers declare more elements or bytes than the remaining input can back. + Every element that survives is backed by >= 1 input byte, so the real decode's + total container pre-allocation is bounded by len(data) rather than by + depth x declared length. The explicit ``max_*_len=len(data)`` caps on + ``unpackb`` are unreachable once the walk passes; they are defence in depth + against a walk regression, not an independent bound. Every rejection is a ``ValueError`` (``FormatError``, ``ExtraData``, or the walk's own ``ValueError`` naming the violated bound), which the read paths diff --git a/src/cachekit/serializers/standard_serializer.py b/src/cachekit/serializers/standard_serializer.py index e69f88fa..fce1e141 100644 --- a/src/cachekit/serializers/standard_serializer.py +++ b/src/cachekit/serializers/standard_serializer.py @@ -27,7 +27,7 @@ from cachekit._rust_serializer import ByteStorage -from .base import SerializationError, SerializationFormat, SerializationMetadata, unpackb_bounded +from .base import PAYLOAD_DECODE_ERRORS, SerializationError, SerializationFormat, SerializationMetadata, unpackb_bounded # Error message constants for unsupported types (Task 2) NUMPY_ERROR_MESSAGE = ( @@ -343,10 +343,7 @@ def deserialize(self, data: bytes | memoryview, metadata: SerializationMetadata except SerializationError: # Re-raise SerializationError (integrity check failure) without swallowing raise - except (msgpack.exceptions.UnpackException, ValueError, TypeError, BufferError) as e: - # BufferError: a non-u8 buffer exporter (e.g. numpy float array) rejected at the - # PyO3 boundary. Pre-LAB-770 bytes() coerced these to raw bytes and envelope - # validation rejected the garbage as ValueError; same contract, new cause. + except PAYLOAD_DECODE_ERRORS as e: raise SerializationError(f"Failed to deserialize MessagePack data: {e}") from e diff --git a/tests/unit/protocol/test_decode_bounds.py b/tests/unit/protocol/test_decode_bounds.py index 728641ee..15aa8afd 100644 --- a/tests/unit/protocol/test_decode_bounds.py +++ b/tests/unit/protocol/test_decode_bounds.py @@ -39,8 +39,8 @@ VECTORS = json.loads(FIXTURE_PATH.read_text(encoding="utf-8")) EXPECTED_COUNTS = {"reject_vectors": 10, "accept_vectors": 2} -# Peak transient heap a rejected decode may cost: a small constant (Unpacker buffer) -# plus a few multiples of the input. Unguarded, the nested_array32_input_len vector +# Peak transient heap a rejected decode may cost: a small constant (tracemalloc + unpackb +# overhead) plus a few multiples of the input. Unguarded, the nested_array32_input_len vector # peaks at ~8000x its input, so this discriminates by three orders of magnitude. PEAK_BUDGET = 2 * 1024 * 1024 PEAK_PER_INPUT_BYTE = 4 @@ -124,10 +124,8 @@ class TestOwnedBounds: """SDK-local guards that go beyond the shared vectors.""" def test_nesting_ceiling_is_exactly_the_pinned_constant(self) -> None: - # msgpack-python exposes no depth option; the C unpacker's fixed stack is the - # bound. If a release moves it, this fails and the constant + protocol note - # (spec/interop-mode.md → Decode bounds, 32 <= bound <= 1024) must be revisited. - assert MSGPACK_MAX_NESTING == 1024 + # The walk rejects one level past MSGPACK_MAX_NESTING; a document AT the ceiling + # must still decode, so the constant may not exceed msgpack-python's C stack. at_bound = b"\x91" * MSGPACK_MAX_NESTING + b"\xc0" assert unpackb_bounded(at_bound) == json.loads("[" * MSGPACK_MAX_NESTING + "null" + "]" * MSGPACK_MAX_NESTING) with pytest.raises(ValueError, match=f"nests deeper than {MSGPACK_MAX_NESTING} levels"): diff --git a/tests/unit/test_auto_serializer_new_types.py b/tests/unit/test_auto_serializer_new_types.py index b05187d7..7975b7b9 100644 --- a/tests/unit/test_auto_serializer_new_types.py +++ b/tests/unit/test_auto_serializer_new_types.py @@ -12,13 +12,35 @@ from uuid import UUID +import msgpack import pytest from hypothesis import given from hypothesis import strategies as st +from cachekit._rust_serializer import ByteStorage from cachekit.serializers.auto_serializer import AutoSerializer from cachekit.serializers.base import SerializationError +# A well-formed msgpack document whose ndarray marker makes np.frombuffer raise +# OverflowError (itemsize past C long) — a forged entry that is neither a decode-bound +# rejection nor a ValueError, so it pins the serializer's exception contract. +FORGED_NDARRAY = msgpack.packb( + {"__ndarray__": True, "dtype": {"names": ["a"], "formats": ["f8"], "itemsize": 2**63}, "shape": [1], "data": b"x" * 8} +) + + +@pytest.mark.parametrize( + "serializer, entry", + [ + (AutoSerializer(enable_integrity_checking=False), FORGED_NDARRAY), + (AutoSerializer(), bytes(ByteStorage("msgpack").store(FORGED_NDARRAY, "msgpack"))), + ], + ids=["plain", "verified-envelope"], +) +def test_forged_payload_failure_is_a_serialization_error(serializer: AutoSerializer, entry: bytes) -> None: + with pytest.raises(SerializationError): + serializer.deserialize(entry) + class TestAutoSerializerUUID: """Test UUID serialization support.""" From 72e8ef5b01e2e12a5382c8884a2c5d7edfd9d458 Mon Sep 17 00:00:00 2001 From: Mark S Date: Mon, 7 Sep 2026 10:12:06 +1000 Subject: [PATCH 5/9] fix(serializers): apply LAB-2503 round-3 review to the decode bound - rust/msgpack_bounds.rs: clippy pedantic (Security Lints, rust 1.97) - doc backticks, `# Errors` section, checked `usize::try_from(payload)` in place of `payload as usize` (line-for-line with cachekit-rs check_structure; walk semantics unchanged, all 13 protocol vectors rejected by the walk alone). - Vendor test-vectors/decode-bounds.json from protocol#59 @2d56cce (13 reject / 2 accept; sha256 + count pins bumped). - Fail closed on forged dtypes: SyntaxError (numpy's comma-string dtype parser runs ast.literal_eval on a forged shape prefix such as "(1,f8") joins PAYLOAD_DECODE_ERRORS and _deserialize_numpy's clause; M8[0ns] (zero datetime unit multiplier) is refused before any array is built - it passes np.frombuffer and then kills the process with SIGFPE inside pandas; the columnar routes refuse any dtype the writer never emits. - _decode_columnar normalises the DataFrame/Series metadata routes, so a bomb behind a forged original_type frame reaches the handler as SerializationError (evict + tamper hook) instead of a bare ValueError that cache_handler logs as a backend fault. - Delete two dead `except SerializationError: raise` clauses left behind by the except-narrowing (SerializationError is outside PAYLOAD_DECODE_ERRORS). - Tests: forged NUMPY_RAW / __ndarray__ / columnar-dtype vectors, every DataFrame/Series read route, validate_data within the peak budget, a bomb behind a dataframe/series frame; codecov/patch 71% -> 88% measured locally. Kody r3919879623 / r3919879856 asked for OverflowError in the numpy clause: not reachable from a dtype string on numpy 1.26.4 / 2.0.2 / 2.2.6 / 2.3.4 (measured), so rejected; the SyntaxError escape the panel found is the real gap on that path. --- rust/src/msgpack_bounds.rs | 15 +++-- src/cachekit/serializers/auto_serializer.py | 63 +++++++++++++------ src/cachekit/serializers/base.py | 8 ++- .../unit/protocol/fixtures/decode-bounds.json | 58 +++++++++++++++-- tests/unit/protocol/test_decode_bounds.py | 35 +++++++++-- ...auto_serializer_mutation_and_corruption.py | 63 ++++++++++++++++++- tests/unit/test_auto_serializer_new_types.py | 46 ++++++++++---- .../test_auto_serializer_numpy_integrity.py | 40 ++++++++++++ 8 files changed, 276 insertions(+), 52 deletions(-) diff --git a/rust/src/msgpack_bounds.rs b/rust/src/msgpack_bounds.rs index e791a42f..ef87a77d 100644 --- a/rust/src/msgpack_bounds.rs +++ b/rust/src/msgpack_bounds.rs @@ -3,18 +3,20 @@ //! a core-shared walk usable from py/rs/wasm is the follow-up. Mirrors the opcode table of //! cachekit-rs `check_structure` so the two SDKs reject the same documents. -/// Header-only walk over one MessagePack document: str/bin/ext payloads are skipped by +/// Header-only walk over one `MessagePack` document: str/bin/ext payloads are skipped by /// offset, never read, and nothing is allocated beyond one `u64` per open collection. /// -/// Rejects, before any decoder pre-allocates a container: +/// Trailing bytes after the root element are left to the decoder (`ExtraData`). +/// +/// # Errors +/// +/// Names the violated bound, before any decoder pre-allocates a container, for: /// - nesting deeper than `max_depth`; /// - a header declaring more payload bytes than the input holds; /// - more pending elements (across every open collection) than remaining bytes can back — /// every element costs >= 1 byte, so a decoder's total container pre-allocation is then /// bounded by the input length instead of by `depth × declared_len`; /// - the reserved marker 0xc1 and input that ends mid-document. -/// -/// Trailing bytes after the root element are left to the decoder (`ExtraData`). pub fn check_msgpack_structure(bytes: &[u8], max_depth: usize) -> Result<(), String> { fn be(bytes: &[u8], pos: usize, width: usize) -> Result { let end = pos @@ -66,7 +68,10 @@ pub fn check_msgpack_structure(bytes: &[u8], max_depth: usize) -> Result<(), Str if payload > remaining { return Err("declares more bytes than the input holds".to_owned()); } - pos += payload as usize; // <= remaining, so it fits usize + // <= remaining, so this cannot fail; `try_from` rather than `as usize` satisfies + // clippy::cast_possible_truncation, line-for-line with cachekit-rs `check_structure`. + pos += usize::try_from(payload) + .map_err(|_| "declares more bytes than the input holds".to_owned())?; if children > 0 { if open.len() >= max_depth { return Err(format!("nests deeper than {max_depth} levels")); diff --git a/src/cachekit/serializers/auto_serializer.py b/src/cachekit/serializers/auto_serializer.py index 316cafbe..d3aa3573 100644 --- a/src/cachekit/serializers/auto_serializer.py +++ b/src/cachekit/serializers/auto_serializer.py @@ -157,6 +157,22 @@ def _is_plain_numpy_numeric(dtype: Any) -> bool: return HAS_PANDAS and not pd.api.types.is_extension_array_dtype(dtype) and dtype.kind in ("i", "u", "f") +def _dtype_from_untrusted(spec: Any, *, numeric_only: bool = False) -> np.dtype: + """``np.dtype(spec)`` for a dtype the cache entry itself supplies, refusing what the writer never emits. + + A forged ``M8[0ns]`` (zero datetime unit multiplier) passes ``np.frombuffer`` and then kills + the process with SIGFPE inside pandas — a signal no ``except`` can catch — so it is refused + before any array is built. Columnar (DataFrame/Series) entries only ever carry dtypes that + pass ``_is_plain_numpy_numeric``, the write-side predicate, so ``numeric_only`` mirrors it. + """ + dtype = np.dtype(spec) + if numeric_only and not _is_plain_numpy_numeric(dtype): + raise SerializationError(f"Forged columnar dtype {dtype}: the writer only emits plain NumPy numeric columns") + if dtype.kind in "Mm" and np.datetime_data(dtype)[1] == 0: + raise SerializationError(f"Forged dtype {dtype}: a zero datetime unit multiplier crashes pandas") + return dtype + + def _na_safe_object_list(series: Any) -> list: """``series.tolist()`` with scalar pandas NA sentinels (pd.NA/NaT/NaN) mapped to None. @@ -297,7 +313,7 @@ def _auto_object_hook(obj: Any) -> Any: if "data" not in obj or "shape" not in obj or "dtype" not in obj: raise SerializationError("Invalid ndarray format: missing required fields in cached data") # .copy(): writable result that does not alias the source buffer (the L1-cached bytes on a hit) — #157. - return np.frombuffer(obj["data"], dtype=obj["dtype"]).reshape(obj["shape"]).copy() + return np.frombuffer(obj["data"], dtype=_dtype_from_untrusted(obj["dtype"])).reshape(obj["shape"]).copy() return obj @@ -550,11 +566,9 @@ def deserialize(self, data: bytes | memoryview, metadata: Optional[Serialization original_data, _ = self._byte_storage.retrieve(data) except (ValueError, SerializationError) as e: raise SerializationError(f"DataFrame integrity check failed (corrupted cache entry): {e}") from e - unpacked_data = unpackb_bounded(original_data, **self._msgpack_unpack_opts) - return self._deserialize_dataframe(unpacked_data) + return self._decode_columnar(original_data, detected_format) # Integrity off: data is direct msgpack (no envelope) - unpacked_data = unpackb_bounded(data, **self._msgpack_unpack_opts) - return self._deserialize_dataframe(unpacked_data) + return self._decode_columnar(data, detected_format) elif detected_format == "series": if self.enable_integrity_checking and len(data) > 4: # Same fail-closed contract as the DataFrame branch above (#156). @@ -562,11 +576,9 @@ def deserialize(self, data: bytes | memoryview, metadata: Optional[Serialization original_data, _ = self._byte_storage.retrieve(data) except (ValueError, SerializationError) as e: raise SerializationError(f"Series integrity check failed (corrupted cache entry): {e}") from e - unpacked_data = unpackb_bounded(original_data, **self._msgpack_unpack_opts) - return self._deserialize_series(unpacked_data) + return self._decode_columnar(original_data, detected_format) # Integrity off: data is direct msgpack (no envelope) - unpacked_data = unpackb_bounded(data, **self._msgpack_unpack_opts) - return self._deserialize_series(unpacked_data) + return self._decode_columnar(data, detected_format) # For Rust-envelope formats, use the Rust layer envelope_error: Exception | None = None @@ -601,8 +613,6 @@ def deserialize(self, data: bytes | memoryview, metadata: Optional[Serialization return self._deserialize_dataframe(unpacked_data) return self._deserialize_series(unpacked_data) return unpackb_bounded(original_data, **self._msgpack_unpack_opts) - except SerializationError: - raise except PAYLOAD_DECODE_ERRORS as e: raise SerializationError( f"Cache entry payload failed to decode inside a verified envelope (format={detected_format!r}): {e}" @@ -630,9 +640,6 @@ def deserialize(self, data: bytes | memoryview, metadata: Optional[Serialization # Python-only path (no Rust compression) - direct msgpack deserialization try: return unpackb_bounded(data, **self._msgpack_unpack_opts) - except SerializationError: - # Re-raise SerializationError (corruption detection) without swallowing - raise except PAYLOAD_DECODE_ERRORS as msgpack_error: # If msgpack fails for other reasons, try NumPy-specific deserialization — and if # that fails too, report every reason: the msgpack one is the decode-bound @@ -738,10 +745,12 @@ def _deserialize_numpy(self, data: bytes) -> np.ndarray: # not alias the source bytes (the L1-cached buffer on a hit) — see #157. frombuffer alone # returns a read-only view aliasing the input. raw_bytes = data[offset:] - arr = np.frombuffer(raw_bytes, dtype=dtype_str).copy() + arr = np.frombuffer(raw_bytes, dtype=_dtype_from_untrusted(dtype_str)).copy() return arr.reshape(shape) - except (ValueError, TypeError, IndexError) as e: - # TypeError: np.frombuffer on a forged dtype string; UnicodeDecodeError is a ValueError. + except (ValueError, TypeError, IndexError, SyntaxError) as e: + # TypeError: np.frombuffer on a forged dtype string; SyntaxError: numpy's comma-string + # dtype parser runs ast.literal_eval on a forged shape prefix such as "(1,f8"; + # UnicodeDecodeError is a ValueError. raise SerializationError(f"Failed to deserialize NumPy array: {e}") from e def _serialize_dataframe(self, df: pd.DataFrame) -> bytes: @@ -801,7 +810,7 @@ def _deserialize_dataframe(self, data) -> pd.DataFrame: for col, col_info in serialized["data"].items(): if col_info["type"] == "numeric": # Reconstruct from NumPy bytes; .copy() → writable, non-aliasing column (#157). - arr = np.frombuffer(col_info["data"], dtype=col_info["dtype"]).copy() + arr = np.frombuffer(col_info["data"], dtype=_dtype_from_untrusted(col_info["dtype"], numeric_only=True)).copy() columns_data[col] = arr else: # Use object data directly @@ -865,7 +874,9 @@ def _deserialize_series(self, data) -> pd.Series: if serialized["type"] == "numeric": # .copy() → writable Series values that do not alias the source buffer (#157). - values = np.frombuffer(serialized["data"], dtype=serialized["dtype"]).copy() + values = np.frombuffer( + serialized["data"], dtype=_dtype_from_untrusted(serialized["dtype"], numeric_only=True) + ).copy() else: values = serialized["data"] @@ -877,6 +888,20 @@ def _deserialize_series(self, data) -> pd.Series: return series + def _decode_columnar(self, payload: bytes | bytearray | memoryview, kind: str) -> pd.DataFrame | pd.Series: + """Decode a ``dataframe`` / ``series`` payload, failing closed as ``SerializationError``. + + The metadata routes in ``deserialize`` reach here outside the verified-envelope + normaliser, and the read handler treats only ``SerializationError`` as a read error + (evict + tamper hook) — a bare ``ValueError`` from the decode bound would be logged as + a backend fault and the poisoned entry kept (LAB-2503). + """ + build = self._deserialize_dataframe if kind == "dataframe" else self._deserialize_series + try: + return build(unpackb_bounded(payload, **self._msgpack_unpack_opts)) + except PAYLOAD_DECODE_ERRORS as e: + raise SerializationError(f"Cache entry payload failed to decode as {kind}: {e}") from e + def _serialize_msgpack(self, obj: Any) -> bytes: """Serialize general object with MessagePack.""" # Pre-process tuples into markers (msgpack natively flattens them to lists) diff --git a/src/cachekit/serializers/base.py b/src/cachekit/serializers/base.py index e77206b7..b2f47c68 100644 --- a/src/cachekit/serializers/base.py +++ b/src/cachekit/serializers/base.py @@ -340,12 +340,14 @@ class SuspiciousCacheEntryError(SerializationError): #: Everything a corrupted or forged payload can make a decode raise, for serializers #: to turn into ``SerializationError``. msgpack's own errors are ``ValueError`` #: subclasses; the AutoSerializer object hook and the NumPy/DataFrame/Series -#: reconstructors add ``TypeError`` / ``OverflowError`` (``np.frombuffer`` on a forged -#: dtype or itemsize), ``KeyError`` / ``AttributeError`` (indexing a dict that is not +#: reconstructors add ``TypeError`` (``np.frombuffer`` on a forged dtype string), +#: ``OverflowError`` (a forged dict dtype whose itemsize is past C long), ``SyntaxError`` +#: (numpy's comma-string dtype parser runs ``ast.literal_eval`` on a forged shape prefix +#: such as ``"(1,f8"``), ``KeyError`` / ``AttributeError`` (indexing a dict that is not #: the shape they wrote); ``BufferError`` is a non-u8 buffer exporter rejected at the #: PyO3 boundary (LAB-770). Anything else — above all ``RuntimeError`` for a missing #: optional dependency — is an environment fault, not a bad cache entry, and must bubble. -PAYLOAD_DECODE_ERRORS = (ValueError, TypeError, KeyError, AttributeError, OverflowError, BufferError) +PAYLOAD_DECODE_ERRORS = (ValueError, TypeError, KeyError, AttributeError, OverflowError, BufferError, SyntaxError) def unpackb_bounded(data: bytes | bytearray | memoryview, **unpack_opts: Any) -> Any: diff --git a/tests/unit/protocol/fixtures/decode-bounds.json b/tests/unit/protocol/fixtures/decode-bounds.json index 6a58520d..21b9180d 100644 --- a/tests/unit/protocol/fixtures/decode-bounds.json +++ b/tests/unit/protocol/fixtures/decode-bounds.json @@ -2,16 +2,16 @@ "version": "1.0.0", "spec": "spec/interop-mode.md#decode-bounds", "generator": "tools/decode-bounds-reference.py generate (CPython stdlib)", - "scope": "Any untrusted MessagePack decode in any SDK: interop/v1 values, auto-mode payloads after the ByteStorage envelope is unwrapped, invalidation events. The bytes are plain MessagePack with no envelope.", + "scope": "Any untrusted MessagePack decode in any SDK: interop/v1 values, the ByteStorage envelope bytes before StorageEnvelope is materialised, auto-mode payloads after the envelope is unwrapped, invalidation events. The bytes are plain MessagePack with no envelope.", "rules": { "depth": "Readers MUST bound nesting depth. The bound MUST be >= 32 and MUST be <= 1024; every reject vector tagged 'depth' nests deeper than 1024.", - "overclaim": "Readers MUST NOT pre-allocate for a collection/str/bin header more than the remaining input can back (each element or byte needs >= 1 input byte), and MUST reject a structurally incomplete document. Every reject vector tagged 'overclaim' has declared_slots > input_len - 1 (the root header is the only byte that is not an element).", + "overclaim": "Readers MUST NOT pre-allocate for a collection/str/bin header more than the remaining input can back (each element or byte needs >= 1 input byte), and MUST reject a structurally incomplete document. Every reject vector tagged 'overclaim' has declared_slots > input_len - 1 (the root header is the only byte that is not an element). A map pair counts as two slots (key + value). Every per-header term and the running sum MUST be computed in >= 64 bits or with checked/saturating arithmetic; an overflow is itself a rejection.", "failure_mode": "Rejection MUST surface as a catchable decode error that the SDK read path turns into a cache miss (fail-closed), never an uncaught crash or an OOM abort." }, "field_notes": { "construction": "input = bytes.fromhex(repeat_hex) * count + bytes.fromhex(suffix_hex)", "nesting_depth": "collection headers along the deepest spine (str/bin count as 0)", - "declared_slots": "sum of every header's declared element/byte count (a nested header counts as one element of its parent)", + "declared_slots": "sum of every header's declared element/byte count; a map pair counts as two slots (key + value); a nested header counts as one element of its parent", "reject_reasons": "which rule(s) the vector violates; a maintainer note, not a normative message" }, "reject_vectors": [ @@ -116,7 +116,7 @@ }, { "name": "map32_max_claim_alone", - "description": "A lone 5-byte map32 header claiming 2^32-1 pairs.", + "description": "A lone 5-byte map32 header claiming 2^32-1 pairs (2^33-2 slots: each pair is a key and a value).", "construction": { "repeat_hex": "dfffffffff", "count": 1, @@ -125,7 +125,55 @@ "input_hex": "dfffffffff", "input_len": 5, "nesting_depth": 1, - "declared_slots": 4294967295, + "declared_slots": 8589934590, + "reject_reasons": [ + "overclaim" + ] + }, + { + "name": "array32_sum_wraps_u32", + "description": "array32 claiming 2^32-1 elements whose first element is an array32 claiming 1: the declared slots sum to exactly 2^32, which a 32-bit accumulator wraps to 0 and then passes the slot budget.", + "construction": { + "repeat_hex": "ddffffffff", + "count": 1, + "suffix_hex": "dd00000001" + }, + "input_hex": "ddffffffffdd00000001", + "input_len": 10, + "nesting_depth": 2, + "declared_slots": 4294967296, + "reject_reasons": [ + "overclaim" + ] + }, + { + "name": "map32_half_claim_wraps_u32_mul", + "description": "A lone map32 header claiming 2^31 pairs: the per-header term 2 x pairs is exactly 2^32, which a 32-bit multiply wraps to 0 before it is ever added to the budget.", + "construction": { + "repeat_hex": "df80000000", + "count": 1, + "suffix_hex": "" + }, + "input_hex": "df80000000", + "input_len": 5, + "nesting_depth": 1, + "declared_slots": 4294967296, + "reject_reasons": [ + "overclaim" + ] + }, + { + "name": "fixmap_short_by_one", + "description": "fixmap claiming 1 pair with the key present and the value missing: the map twin of fixarray_short_by_one. Counting one slot per pair (instead of two) accepts it.", + "construction": { + "repeat_hex": "81", + "count": 1, + "suffix_hex": "c0" + }, + "input_hex": "81c0", + "input_len": 2, + "nesting_depth": 1, + "declared_slots": 2, "reject_reasons": [ "overclaim" ] diff --git a/tests/unit/protocol/test_decode_bounds.py b/tests/unit/protocol/test_decode_bounds.py index 15aa8afd..9de0ad7d 100644 --- a/tests/unit/protocol/test_decode_bounds.py +++ b/tests/unit/protocol/test_decode_bounds.py @@ -35,9 +35,9 @@ from cachekit.serializers.wrapper import SerializationWrapper FIXTURE_PATH = Path(__file__).parent / "fixtures" / "decode-bounds.json" -FIXTURE_SHA256 = "fa8bc750a4911fe3663b9ab68f13438a3e924b6bc742e9f6763ca35ad2407476" # pragma: allowlist secret +FIXTURE_SHA256 = "75c1204e6f58f5220581d3e40e75a68f2df605b4e3c817107b0c690cd7da5cd4" # pragma: allowlist secret VECTORS = json.loads(FIXTURE_PATH.read_text(encoding="utf-8")) -EXPECTED_COUNTS = {"reject_vectors": 10, "accept_vectors": 2} +EXPECTED_COUNTS = {"reject_vectors": 13, "accept_vectors": 2} # Peak transient heap a rejected decode may cost: a small constant (tracemalloc + unpackb # overhead) plus a few multiples of the input. Unguarded, the nested_array32_input_len vector @@ -53,10 +53,10 @@ def _envelope(payload: bytes) -> bytes: CACHE_KEY = "ns:decode:bounds" -@functools.lru_cache(maxsize=1) -def _frame_template() -> tuple[dict[str, Any], str]: +@functools.lru_cache(maxsize=2) +def _frame_template(serializer: str = "default") -> tuple[dict[str, Any], str]: _, metadata, serializer_name = SerializationWrapper.unwrap( - CacheSerializationHandler().serialize_data({"t": 1}, cache_key=CACHE_KEY) + CacheSerializationHandler(serializer).serialize_data({"t": 1}, cache_key=CACHE_KEY) ) return metadata, serializer_name @@ -94,6 +94,10 @@ def _vector_ids(group: str) -> list[str]: return [v["name"] for v in VECTORS[group]] +def _reject_vector(name: str) -> bytes: + return bytes.fromhex(next(v["input_hex"] for v in VECTORS["reject_vectors"] if v["name"] == name)) + + class TestFixtureIsTheVendoredProtocolFile: def test_sha256_and_counts(self) -> None: assert hashlib.sha256(FIXTURE_PATH.read_bytes()).hexdigest() == FIXTURE_SHA256 @@ -134,3 +138,24 @@ def test_nesting_ceiling_is_exactly_the_pinned_constant(self) -> None: def test_trailing_bytes_still_rejected(self) -> None: with pytest.raises(msgpack.exceptions.ExtraData): unpackb_bounded(b"\xc0\xc0") + + def test_validate_data_reports_a_bomb_as_invalid_within_the_peak_budget(self) -> None: + # Python-only validate_data is a decode path too: a bomb must read as invalid (not raise), + # and the walk must have stopped it before the decoder pre-allocated ~8000x the input. + serializer = AutoSerializer(enable_integrity_checking=False) + assert serializer.validate_data(msgpack.packb({"t": 1})) is True + bomb = _reject_vector("nested_array32_input_len_depth_1100") + valid, err, peak = _peak_of(serializer.validate_data, bomb) + assert (valid, err) == (False, None) + assert peak < PEAK_BUDGET + PEAK_PER_INPUT_BYTE * len(bomb), f"validate_data peaked at {peak} bytes" + + @pytest.mark.parametrize("original_type", ["dataframe", "series"]) + def test_bomb_behind_a_dataframe_or_series_frame_is_a_controlled_miss(self, original_type: str) -> None: + # AutoSerializer's metadata routes decode outside the verified-envelope normaliser; the bound's + # rejection must still reach the handler as SerializationError (evict + tamper hook), never a + # bare ValueError. The message match keeps a "Serializer mismatch" error from faking a pass. + metadata, serializer_name = _frame_template("auto") + bomb = _reject_vector("nested_array32_input_len_depth_1100") + frame = SerializationWrapper.wrap(_envelope(bomb), {**metadata, "original_type": original_type}, serializer_name) + with pytest.raises(SerializationError, match=f"failed to decode as {original_type}"): + CacheSerializationHandler("auto").deserialize_data(frame, cache_key=CACHE_KEY) diff --git a/tests/unit/test_auto_serializer_mutation_and_corruption.py b/tests/unit/test_auto_serializer_mutation_and_corruption.py index f30b6163..8f32ee2a 100644 --- a/tests/unit/test_auto_serializer_mutation_and_corruption.py +++ b/tests/unit/test_auto_serializer_mutation_and_corruption.py @@ -12,25 +12,63 @@ DataFrames route through ArrowSerializer when pyarrow is installed, so the columnar msgpack path (``_serialize_dataframe`` / the ``"dataframe"`` branch) is exercised by disabling the arrow serializer. Series never use arrow, so they hit the columnar path unconditionally. + +LAB-2503: ``TestDataFrameSeriesReadRoutes`` pins every DataFrame/Series read route (metadata x +integrity, and metadata-less via the envelope's format_id); it lives here because this file +already forces the columnar path. """ from __future__ import annotations +import msgpack import numpy as np import pandas as pd import pytest +from cachekit._rust_serializer import ByteStorage from cachekit.serializers import AutoSerializer from cachekit.serializers.base import SerializationError -def _no_arrow() -> AutoSerializer: +def _no_arrow(**kwargs: bool) -> AutoSerializer: """An AutoSerializer forced onto the columnar msgpack DataFrame path (pyarrow absent).""" - s = AutoSerializer() + s = AutoSerializer(**kwargs) s._arrow_serializer = None return s +def _assert_equal(out: pd.DataFrame | pd.Series, expected: pd.DataFrame | pd.Series) -> None: + if isinstance(expected, pd.DataFrame): + pd.testing.assert_frame_equal(out, expected) + else: + pd.testing.assert_series_equal(out, expected) + + +FRAME = pd.DataFrame({"x": np.arange(5, dtype=np.float64), "n": np.arange(5, dtype=np.int64)}) +SERIES = pd.Series(np.arange(8, dtype=np.float64), name="v") + + +@pytest.mark.unit +class TestDataFrameSeriesReadRoutes: + """Every route a DataFrame/Series read can take must reconstruct the value: with metadata + on both integrity settings, and — the decorator read path may carry none — from the + verified envelope's own format_id (LAB-2503 moved that route under the fail-closed guard). + """ + + @pytest.mark.parametrize("value", [FRAME, SERIES], ids=["dataframe", "series"]) + @pytest.mark.parametrize("integrity", [True, False], ids=["integrity-on", "integrity-off"]) + def test_roundtrip_with_metadata(self, value: pd.DataFrame | pd.Series, integrity: bool) -> None: + s = _no_arrow(enable_integrity_checking=integrity) + data, meta = s.serialize(value) + _assert_equal(s.deserialize(data, meta), value) + + @pytest.mark.parametrize("value", [FRAME, SERIES], ids=["dataframe", "series"]) + def test_roundtrip_without_metadata_via_envelope_format_id(self, value: pd.DataFrame | pd.Series) -> None: + s = _no_arrow() + data, _ = s.serialize(value) + _assert_equal(s.deserialize(data), value) + + @pytest.mark.unit class TestDeserializedArraysAreWritable: """#157: deserialized numeric arrays must be writable and must not alias the cached buffer.""" @@ -91,3 +129,24 @@ def test_dataframe_corruption_raises_serialization_error(self) -> None: corrupted[len(corrupted) // 2] ^= 0xFF with pytest.raises(SerializationError): s.deserialize(bytes(corrupted), meta) + + +@pytest.mark.unit +class TestForgedColumnarDtypeIsRefused: + """A forged numeric-column dtype is refused before any array is built. ``M8[0ns]`` passes + ``np.frombuffer`` and then kills the process with SIGFPE inside pandas — uncatchable — and the + writer only ever emits plain NumPy numeric dtypes, so anything else is a forgery (LAB-2503). + """ + + @pytest.mark.parametrize("dtype", ["M8[0ns]", "m8[0ns]", "U4"]) + @pytest.mark.parametrize("kind", ["dataframe", "series"]) + def test_forged_column_dtype_is_a_serialization_error(self, kind: str, dtype: str) -> None: + column = {"type": "numeric", "data": b"\x00" * 8, "dtype": dtype} + body = ( + {"columns": ["x"], "index": None, "data": {"x": column}} + if kind == "dataframe" + else {"name": None, "index": None, **column} + ) + entry = bytes(ByteStorage("msgpack").store(msgpack.packb(body), kind)) + with pytest.raises(SerializationError, match="Forged columnar dtype"): + AutoSerializer().deserialize(entry) diff --git a/tests/unit/test_auto_serializer_new_types.py b/tests/unit/test_auto_serializer_new_types.py index 7975b7b9..573f99b2 100644 --- a/tests/unit/test_auto_serializer_new_types.py +++ b/tests/unit/test_auto_serializer_new_types.py @@ -6,10 +6,12 @@ - Nested complex objects with new types - Error detection for unsupported types (Pydantic, ORM, custom classes) - Security: _safe_hasattr prevents code execution +- LAB-2503 exception contract: forged payloads and object-hook diagnostics fail closed as SerializationError """ from __future__ import annotations +from collections.abc import Callable from uuid import UUID import msgpack @@ -21,25 +23,43 @@ from cachekit.serializers.auto_serializer import AutoSerializer from cachekit.serializers.base import SerializationError -# A well-formed msgpack document whose ndarray marker makes np.frombuffer raise -# OverflowError (itemsize past C long) — a forged entry that is neither a decode-bound -# rejection nor a ValueError, so it pins the serializer's exception contract. -FORGED_NDARRAY = msgpack.packb( - {"__ndarray__": True, "dtype": {"names": ["a"], "formats": ["f8"], "itemsize": 2**63}, "shape": [1], "data": b"x" * 8} -) - - +# Well-formed msgpack documents whose ndarray marker makes numpy raise something that is neither +# a decode-bound rejection nor a ValueError, pinning the serializer's exception contract: +# OverflowError (dict dtype with itemsize past C long), SyntaxError (numpy's comma-string dtype +# parser runs ast.literal_eval on the forged shape prefix "(1,f8"), and the M8[0ns] dtype that +# numpy accepts and pandas then dies on with SIGFPE — refused before any array is built. +FORGED_NDARRAYS = { + "itemsize-past-c-long": msgpack.packb( + {"__ndarray__": True, "dtype": {"names": ["a"], "formats": ["f8"], "itemsize": 2**63}, "shape": [1], "data": b"x" * 8} + ), + "dtype-shape-prefix-unparseable": msgpack.packb({"__ndarray__": True, "dtype": "(1,f8", "shape": [1], "data": b"x" * 8}), + "datetime-zero-unit-multiplier": msgpack.packb({"__ndarray__": True, "dtype": "M8[0ns]", "shape": [1], "data": b"x" * 8}), +} + + +@pytest.mark.parametrize("payload", FORGED_NDARRAYS.values(), ids=list(FORGED_NDARRAYS)) @pytest.mark.parametrize( - "serializer, entry", + "serializer, wrap", [ - (AutoSerializer(enable_integrity_checking=False), FORGED_NDARRAY), - (AutoSerializer(), bytes(ByteStorage("msgpack").store(FORGED_NDARRAY, "msgpack"))), + (AutoSerializer(enable_integrity_checking=False), lambda b: b), + (AutoSerializer(), lambda b: bytes(ByteStorage("msgpack").store(b, "msgpack"))), ], ids=["plain", "verified-envelope"], ) -def test_forged_payload_failure_is_a_serialization_error(serializer: AutoSerializer, entry: bytes) -> None: +def test_forged_payload_failure_is_a_serialization_error( + serializer: AutoSerializer, wrap: Callable[[bytes], bytes], payload: bytes +) -> None: with pytest.raises(SerializationError): - serializer.deserialize(entry) + serializer.deserialize(wrap(payload)) + + +def test_hook_diagnostic_propagates_unwrapped_from_the_verified_envelope() -> None: + """The object hook's SerializationError sits outside PAYLOAD_DECODE_ERRORS, so it leaves the + verified-envelope decode unwrapped — that catch must never widen back to ``Exception``.""" + entry = bytes(ByteStorage("msgpack").store(msgpack.packb({"__uuid__": True}), "msgpack")) + with pytest.raises(SerializationError, match=r"^Invalid UUID format: missing 'value' field") as excinfo: + AutoSerializer().deserialize(entry) + assert excinfo.value.__cause__ is None class TestAutoSerializerUUID: diff --git a/tests/unit/test_auto_serializer_numpy_integrity.py b/tests/unit/test_auto_serializer_numpy_integrity.py index b9288593..ef3dde0e 100644 --- a/tests/unit/test_auto_serializer_numpy_integrity.py +++ b/tests/unit/test_auto_serializer_numpy_integrity.py @@ -174,3 +174,43 @@ def test_numpy_roundtrip_through_encryption(self) -> None: result = wrapper.deserialize(data, metadata, cache_key) np.testing.assert_array_equal(result, original) + + +def _numpy_raw(dtype: bytes, shape: tuple[int, ...], payload: bytes) -> bytes: + """A NUMPY_RAW entry laid out exactly as ``_serialize_numpy`` writes it, fields attacker-chosen.""" + shape_data = b"".join(dim.to_bytes(4, "little") for dim in shape) + header = len(dtype).to_bytes(2, "little") + dtype + len(shape_data).to_bytes(2, "little") + shape_data + return b"NUMPY_RAW" + header + payload + + +# Each passes the header checks and reaches _deserialize_numpy's own except clause — the dtype +# decode or numpy raising TypeError / ValueError / SyntaxError (measured on numpy 1.26-2.3). +FORGED_NUMPY_RAW = { + "dtype-not-understood": _numpy_raw(b"not-a-dtype", (1,), b"\x00" * 8), + "itemsize-past-c-long": _numpy_raw(b"V9223372036854775808", (1,), b"\x00" * 8), + "dtype-not-utf8": _numpy_raw(b"\xff\xfe", (1,), b"\x00" * 8), + "shape-does-not-fit": _numpy_raw(b" ast.literal_eval -> SyntaxError +} + + +@pytest.mark.unit +class TestAutoSerializerNumpyForgedEntries: + """A NUMPY_RAW entry is routed to ``_deserialize_numpy`` structurally, with no outer + ``PAYLOAD_DECODE_ERRORS`` normalisation, so its own except clause is the whole fail-closed + contract for a forged entry (LAB-2503). The checksum is unkeyed and does not help: whoever + can write the backend can also write a matching xxHash3-64. + """ + + @pytest.mark.parametrize("entry", FORGED_NUMPY_RAW.values(), ids=list(FORGED_NUMPY_RAW)) + @pytest.mark.parametrize("checksummed", [False, True], ids=["raw", "checksummed"]) + def test_forged_entry_fails_closed_as_serialization_error(self, entry: bytes, checksummed: bool) -> None: + if checksummed: + entry = xxhash.xxh3_64_digest(entry) + entry + with pytest.raises(SerializationError, match="Failed to deserialize NumPy array"): + AutoSerializer().deserialize(entry) + + def test_degenerate_datetime_dtype_is_refused_before_any_array_is_built(self) -> None: + # M8[0ns] passes np.frombuffer and then kills the process with SIGFPE inside pandas. + with pytest.raises(SerializationError, match="zero datetime unit multiplier"): + AutoSerializer().deserialize(_numpy_raw(b"M8[0ns]", (1,), b"\x00" * 8)) From d4a226b48d10d604de2b1e899dd8de50f45abd64 Mon Sep 17 00:00:00 2001 From: Mark S Date: Mon, 7 Sep 2026 10:31:03 +1000 Subject: [PATCH 6/9] fix(serializers): gate forged columnar documents by shape before pandas sees them (LAB-2503) The `__ndarray__` object hook turns its marker into an ndarray wherever it sits in a decoded document, so a forged DataFrame/Series entry can put an array where the writer only ever puts a list (columns, index, object-column data) or a dict (the document, each column). pandas then raises AssertionError (a datetime64 with unit multiplier != 1, e.g. M8[2s], on its dtype-equality assert) or plain indexing raises IndexError - both outside PAYLOAD_DECODE_ERRORS, so they left AutoSerializer.deserialize as raw exceptions for direct callers (the decorator path already mapped them to a controlled miss). `_expect(value, kind, what)` refuses any field whose type the writer never emits, mirroring _serialize_dataframe / _serialize_series, and is applied at every such field in both reconstructors. Found by the adversarial pass that substituted for the Helly R hand-off (774 fork-isolated probes, no abort / hang / allocation class remaining); pinned by seven forged-document cases in tests/unit/test_auto_serializer_mutation_and_corruption.py. --- src/cachekit/serializers/auto_serializer.py | 32 ++++++++++---- ...auto_serializer_mutation_and_corruption.py | 42 +++++++++++++++++-- 2 files changed, 62 insertions(+), 12 deletions(-) diff --git a/src/cachekit/serializers/auto_serializer.py b/src/cachekit/serializers/auto_serializer.py index d3aa3573..978ff090 100644 --- a/src/cachekit/serializers/auto_serializer.py +++ b/src/cachekit/serializers/auto_serializer.py @@ -173,6 +173,19 @@ def _dtype_from_untrusted(spec: Any, *, numeric_only: bool = False) -> np.dtype: return dtype +def _expect(value: Any, kind: type, what: str) -> Any: + """Refuse a columnar field whose type the writer never emits. + + The ``__ndarray__`` object hook can substitute an attacker-typed ndarray for any field of a + forged DataFrame/Series document; pandas then asserts (``AssertionError``) or indexing raises + ``IndexError`` — both outside ``PAYLOAD_DECODE_ERRORS``. The writer emits ``list`` for + ``columns`` / ``index`` / object data and ``dict`` for the document and each column. + """ + if not isinstance(value, kind): + raise SerializationError(f"Forged columnar payload: {what} is {type(value).__name__}, expected {kind.__name__}") + return value + + def _na_safe_object_list(series: Any) -> list: """``series.tolist()`` with scalar pandas NA sentinels (pd.NA/NaT/NaN) mapped to None. @@ -805,22 +818,24 @@ def _deserialize_dataframe(self, data) -> pd.DataFrame: # Otherwise unpack msgpack serialized = unpackb_bounded(data, **self._msgpack_unpack_opts) + serialized = _expect(serialized, dict, "document") # Reconstruct DataFrame column by column columns_data = {} - for col, col_info in serialized["data"].items(): - if col_info["type"] == "numeric": + for col, col_info in _expect(serialized["data"], dict, "data").items(): + info = _expect(col_info, dict, f"column {col!r}") + if info["type"] == "numeric": # Reconstruct from NumPy bytes; .copy() → writable, non-aliasing column (#157). - arr = np.frombuffer(col_info["data"], dtype=_dtype_from_untrusted(col_info["dtype"], numeric_only=True)).copy() + arr = np.frombuffer(info["data"], dtype=_dtype_from_untrusted(info["dtype"], numeric_only=True)).copy() columns_data[col] = arr else: # Use object data directly - columns_data[col] = col_info["data"] + columns_data[col] = _expect(info["data"], list, f"column {col!r} data") - df = pd.DataFrame(columns_data, columns=serialized["columns"]) + df = pd.DataFrame(columns_data, columns=_expect(serialized["columns"], list, "columns")) # Restore index if it was serialized if serialized["index"] is not None: - df.index = pd.Index(serialized["index"]) + df.index = pd.Index(_expect(serialized["index"], list, "index")) return df @@ -872,19 +887,20 @@ def _deserialize_series(self, data) -> pd.Series: # Otherwise unpack msgpack serialized = unpackb_bounded(data, **self._msgpack_unpack_opts) + serialized = _expect(serialized, dict, "document") if serialized["type"] == "numeric": # .copy() → writable Series values that do not alias the source buffer (#157). values = np.frombuffer( serialized["data"], dtype=_dtype_from_untrusted(serialized["dtype"], numeric_only=True) ).copy() else: - values = serialized["data"] + values = _expect(serialized["data"], list, "data") series = pd.Series(values, name=serialized["name"]) # Restore index if it was serialized if serialized["index"] is not None: - series.index = pd.Index(serialized["index"]) + series.index = pd.Index(_expect(serialized["index"], list, "index")) return series diff --git a/tests/unit/test_auto_serializer_mutation_and_corruption.py b/tests/unit/test_auto_serializer_mutation_and_corruption.py index 8f32ee2a..024d09a9 100644 --- a/tests/unit/test_auto_serializer_mutation_and_corruption.py +++ b/tests/unit/test_auto_serializer_mutation_and_corruption.py @@ -131,11 +131,19 @@ def test_dataframe_corruption_raises_serialization_error(self) -> None: s.deserialize(bytes(corrupted), meta) +# A well-formed __ndarray__ marker: the object hook turns it into an ndarray wherever it sits, so a +# forged document can put an array where the writer only ever puts a list or a dict. M8[2s] is a +# dtype numpy accepts and pandas then asserts on (AssertionError, outside PAYLOAD_DECODE_ERRORS). +NDARRAY_M8_2S = {"__ndarray__": True, "dtype": "M8[2s]", "shape": [1], "data": b"\x00" * 8} +F8_COLUMN = {"type": "numeric", "data": b"\x00" * 8, "dtype": " None: + entry = bytes(ByteStorage("msgpack").store(msgpack.packb(body), kind)) + with pytest.raises(SerializationError, match="Forged columnar payload"): + AutoSerializer().deserialize(entry) From b120fd9c5d1a57e0731ce21c974c7554ba93a820 Mon Sep 17 00:00:00 2001 From: Mark S Date: Mon, 7 Sep 2026 10:39:31 +1000 Subject: [PATCH 7/9] fix(serializers): snapshot mutable exporters before the decode walk; pin the vector link (LAB-2503) - unpackb_bounded: a bytearray, or a memoryview over one, could change between check_msgpack_structure and msgpack.unpackb, so the bound the walk proved would not hold for the bytes the decoder reads. Snapshot mutable exporters to bytes once and hand that one object to both; bytes and a memoryview of bytes stay zero-copy (the same containment proof the Rust bytes_view uses - a read-only memoryview over a bytearray is still mutable underneath, so the exporter type, not `readonly`, is the gate). - README: the decode-bounds.json link pointed at protocol main, where the file does not exist until protocol#59 merges (404). Pin it to the vendored commit 2d56cce, the exact revision the fixture sha256 pins. - test_decode_bounds: one exact-width document per fixed-width marker family (float/int 8-64, fixext 1-16, ext8/16/32, str/bin 8/16/32) checked three ways (walks clean at exact width, one byte short is a truncation, a trailing byte reaches the decoder as ExtraData), the reserved 0xc1 marker, and the mutable-exporter inputs. A Python test through the extension because CI has no cargo test lane - this is where CodeRabbit's Rust marker-test ask actually executes. --- README.md | 2 +- src/cachekit/serializers/base.py | 5 +++ tests/unit/protocol/test_decode_bounds.py | 54 ++++++++++++++++++++++- 3 files changed, 59 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 297a3454..665d7e36 100644 --- a/README.md +++ b/README.md @@ -235,7 +235,7 @@ def test_cached_function(): - Connection pooling with thread affinity (+28% throughput) - Distributed locking prevents cache stampedes - Pluggable backend abstraction (Redis, CachekitIO, File, Memcached, custom) -- Untrusted-decode bounds: nesting depth and header-declared allocation are capped on every cache read (a forged entry is a bounded cache miss), verified against the protocol's shared [`decode-bounds.json`](https://github.com/cachekit-io/protocol/blob/main/test-vectors/decode-bounds.json) vectors +- Untrusted-decode bounds: nesting depth and header-declared allocation are capped on every cache read (a forged entry is a bounded cache miss), verified against the protocol's shared [`decode-bounds.json`](https://github.com/cachekit-io/protocol/blob/2d56cce231e193141f09df9316f9afac17a1538e/test-vectors/decode-bounds.json) vectors > [!NOTE] > All reliability features are **enabled by default** with `@cache.production`. Use `@cache.minimal` to disable them for maximum throughput. diff --git a/src/cachekit/serializers/base.py b/src/cachekit/serializers/base.py index b2f47c68..87a05052 100644 --- a/src/cachekit/serializers/base.py +++ b/src/cachekit/serializers/base.py @@ -387,6 +387,11 @@ def unpackb_bounded(data: bytes | bytearray | memoryview, **unpack_opts: Any) -> ... ValueError: Unpack failed: MessagePack document nests deeper than 1024 levels """ + if not isinstance(data, bytes) and not (isinstance(data, memoryview) and isinstance(data.obj, bytes)): + # A mutable exporter (bytearray, a memoryview over one) could change between the walk and + # the decode, so both must see one immutable document. bytes and a memoryview of bytes stay + # zero-copy — the same containment proof the Rust side's bytes_view uses. + data = bytes(data) n = len(data) check_msgpack_structure(data, MSGPACK_MAX_NESTING) return msgpack.unpackb(data, max_str_len=n, max_bin_len=n, max_array_len=n, max_map_len=n, max_ext_len=n, **unpack_opts) diff --git a/tests/unit/protocol/test_decode_bounds.py b/tests/unit/protocol/test_decode_bounds.py index 9de0ad7d..4c2655ca 100644 --- a/tests/unit/protocol/test_decode_bounds.py +++ b/tests/unit/protocol/test_decode_bounds.py @@ -26,7 +26,7 @@ import msgpack import pytest -from cachekit._rust_serializer import ByteStorage +from cachekit._rust_serializer import ByteStorage, check_msgpack_structure from cachekit.cache_handler import CacheSerializationHandler from cachekit.interop import decode_interop_value from cachekit.serializers.auto_serializer import AutoSerializer @@ -139,6 +139,58 @@ def test_trailing_bytes_still_rejected(self) -> None: with pytest.raises(msgpack.exceptions.ExtraData): unpackb_bounded(b"\xc0\xc0") + def test_mutable_exporters_are_accepted(self) -> None: + # A bytearray (or a memoryview over one) is snapshotted so the walk and the decode see one + # immutable document; a memoryview of bytes stays zero-copy. All three must decode. + doc = msgpack.packb({"t": 1}) + assert unpackb_bounded(bytearray(doc), raw=False) == {"t": 1} + assert unpackb_bounded(memoryview(bytearray(doc)), raw=False) == {"t": 1} + assert unpackb_bounded(memoryview(doc)[0:], raw=False) == {"t": 1} + + # One exact-width document per fixed-width marker family: float32/64, uint8..64, int8..64, + # fixext 1/2/4/8/16, ext8/16/32 (2-byte payload), str8/16/32 + fixstr, bin8/16/32. + FIXED_WIDTH_DOCS = [ + b"\xca" + b"\x00" * 4, + b"\xcb" + b"\x00" * 8, + b"\xcc\x00", + b"\xcd\x00\x00", + b"\xce" + b"\x00" * 4, + b"\xcf" + b"\x00" * 8, + b"\xd0\x00", + b"\xd1\x00\x00", + b"\xd2" + b"\x00" * 4, + b"\xd3" + b"\x00" * 8, + b"\xd4\x01\x00", + b"\xd5\x01\x00\x00", + b"\xd6\x01" + b"\x00" * 4, + b"\xd7\x01" + b"\x00" * 8, + b"\xd8\x01" + b"\x00" * 16, + b"\xc7\x02\x01\x00\x00", + b"\xc8\x00\x02\x01\x00\x00", + b"\xc9\x00\x00\x00\x02\x01\x00\x00", + b"\xa1x", + b"\xd9\x01x", + b"\xda\x00\x01x", + b"\xdb\x00\x00\x00\x01x", + b"\xc4\x01x", + b"\xc5\x00\x01x", + b"\xc6\x00\x00\x00\x01x", + ] + + @pytest.mark.parametrize("doc", FIXED_WIDTH_DOCS, ids=lambda d: f"0x{d[0]:02x}") + def test_every_marker_is_walked_to_its_exact_width(self, doc: bytes) -> None: + # Exact length passes the walk; one byte short is a truncation; a trailing byte reaches the + # decoder as ExtraData — together they pin that the walk consumed exactly the marker's width. + check_msgpack_structure(doc, MSGPACK_MAX_NESTING) + with pytest.raises(ValueError, match="Unpack failed"): + check_msgpack_structure(doc[:-1], MSGPACK_MAX_NESTING) + with pytest.raises(msgpack.exceptions.ExtraData): + unpackb_bounded(doc + b"\xc0") + + def test_reserved_marker_is_rejected(self) -> None: + with pytest.raises(ValueError, match="reserved marker 0xc1"): + check_msgpack_structure(b"\xc1", MSGPACK_MAX_NESTING) + def test_validate_data_reports_a_bomb_as_invalid_within_the_peak_budget(self) -> None: # Python-only validate_data is a decode path too: a bomb must read as invalid (not raise), # and the walk must have stopped it before the decoder pre-allocated ~8000x the input. From 338c6dc409c5f23fe917bda8bb7e563247216cc2 Mon Sep 17 00:00:00 2001 From: Mark S Date: Mon, 7 Sep 2026 10:44:14 +1000 Subject: [PATCH 8/9] fix(serializers): drop the unreachable NumPy fallback on the plain decode path (LAB-2503) NUMPY_RAW entries are routed structurally at the top of AutoSerializer.deserialize, so the fallback that retried a failed plain msgpack decode as NumPy could never succeed - it only ever contributed the constant "expected NUMPY_RAW header" to the miss message. Without the [data] extra (the free-threaded CI lane added on main) it did worse: _deserialize_numpy raises RuntimeError for a missing numpy, which round 2's except-narrowing no longer swallowed, so every forged plain entry surfaced as RuntimeError instead of SerializationError - 13 protocol reject vectors red on that lane. Delete the fallback; the miss now reads "Cache entry is not a decodable MessagePack payload (envelope: ...) (msgpack: ...)". Pinned by test_plain_path_miss_does_not_depend_on_numpy (HAS_NUMPY monkeypatched off), which runs in every lane. --- src/cachekit/serializers/auto_serializer.py | 20 +++++++++----------- tests/unit/protocol/test_decode_bounds.py | 7 +++++++ 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/src/cachekit/serializers/auto_serializer.py b/src/cachekit/serializers/auto_serializer.py index 978ff090..f8d06f9e 100644 --- a/src/cachekit/serializers/auto_serializer.py +++ b/src/cachekit/serializers/auto_serializer.py @@ -654,17 +654,15 @@ def deserialize(self, data: bytes | memoryview, metadata: Optional[Serialization try: return unpackb_bounded(data, **self._msgpack_unpack_opts) except PAYLOAD_DECODE_ERRORS as msgpack_error: - # If msgpack fails for other reasons, try NumPy-specific deserialization — and if - # that fails too, report every reason: the msgpack one is the decode-bound - # rejection for a forged entry and must not vanish behind the NumPy header error. - try: - return self._deserialize_numpy(data) - except (SerializationError, *PAYLOAD_DECODE_ERRORS) as numpy_error: - raise SerializationError( - "Cache entry is not a decodable MessagePack or NumPy payload" - f"{f' (envelope: {envelope_error})' if envelope_error else ''}" - f" (msgpack: {msgpack_error}) (numpy: {numpy_error})" - ) from msgpack_error + # NUMPY_RAW entries were routed structurally at the top, so nothing reaching here can be + # a NumPy payload (and a NumPy attempt would raise RuntimeError without the [data] + # extra). Report every reason for the miss: the msgpack one is the decode-bound + # rejection for a forged entry and must not vanish behind the envelope error. + raise SerializationError( + "Cache entry is not a decodable MessagePack payload" + f"{f' (envelope: {envelope_error})' if envelope_error else ''}" + f" (msgpack: {msgpack_error})" + ) from msgpack_error def _serialize_numpy(self, arr: np.ndarray) -> bytes: # type: ignore[name-defined] """Serialize a NumPy array into the ``NUMPY_RAW`` binary format. diff --git a/tests/unit/protocol/test_decode_bounds.py b/tests/unit/protocol/test_decode_bounds.py index 4c2655ca..4ee70138 100644 --- a/tests/unit/protocol/test_decode_bounds.py +++ b/tests/unit/protocol/test_decode_bounds.py @@ -191,6 +191,13 @@ def test_reserved_marker_is_rejected(self) -> None: with pytest.raises(ValueError, match="reserved marker 0xc1"): check_msgpack_structure(b"\xc1", MSGPACK_MAX_NESTING) + def test_plain_path_miss_does_not_depend_on_numpy(self, monkeypatch: pytest.MonkeyPatch) -> None: + # Without the [data] extra (the free-threaded CI lane) a forged plain entry must still be a + # SerializationError — not the RuntimeError a NumPy fallback raises for a missing numpy. + monkeypatch.setattr("cachekit.serializers.auto_serializer.HAS_NUMPY", False) + with pytest.raises(SerializationError, match="not a decodable MessagePack payload"): + AutoSerializer(enable_integrity_checking=False).deserialize(_reject_vector("bin32_overclaim")) + def test_validate_data_reports_a_bomb_as_invalid_within_the_peak_budget(self) -> None: # Python-only validate_data is a decode path too: a bomb must read as invalid (not raise), # and the walk must have stopped it before the decoder pre-allocated ~8000x the input. From d0b5980df8c05b28641d8120006349c46d362338 Mon Sep 17 00:00:00 2001 From: Mark S Date: Tue, 8 Sep 2026 09:39:25 +1000 Subject: [PATCH 9/9] fix(serializers): refuse unknown column type markers on the columnar decode routes (LAB-2503) The DataFrame/Series decoders read every column whose "type" was not "numeric" as object data, so a forged marker such as "forged" reconstructed as a valid frame instead of failing closed (CodeRabbit on #276). Both switches are one allow-list now - _column_values - which refuses anything but "numeric" / "object" as SerializationError and retires the duplicated frombuffer/dtype-gate branch. Panel: the first cut echoed the marker with repr(), which walks an attacker-chosen structure - a list nested ~1000 deep (admitted by the 1024-level walk) raises RecursionError on 3.10/3.11, outside PAYLOAD_DECODE_ERRORS. Only a str is echoed (capped at 40 chars), otherwise the type name; the column name in the same message is capped the same way (an 8 KB envelope carrying a 1 MiB column name produced a 4 MB error line). Pinned on both kinds with "forged" and a 1000-deep list; verified on 3.11. --- src/cachekit/serializers/auto_serializer.py | 40 +++++++++--------- ...auto_serializer_mutation_and_corruption.py | 42 ++++++++++++++----- 2 files changed, 52 insertions(+), 30 deletions(-) diff --git a/src/cachekit/serializers/auto_serializer.py b/src/cachekit/serializers/auto_serializer.py index f8d06f9e..c1fa8c29 100644 --- a/src/cachekit/serializers/auto_serializer.py +++ b/src/cachekit/serializers/auto_serializer.py @@ -186,6 +186,22 @@ def _expect(value: Any, kind: type, what: str) -> Any: return value +def _column_values(info: dict[str, Any], what: str) -> Any: + """Rebuild one column's values from the ``{type, data[, dtype]}`` the writer emits (``dtype`` only for ``"numeric"``). + + ``type`` is an allow-list, not a numeric/else switch: an unknown marker must not be read as object data. + """ + marker = info["type"] + if marker == "numeric": + # .copy() → writable values that do not alias the source buffer (#157). + return np.frombuffer(info["data"], dtype=_dtype_from_untrusted(info["dtype"], numeric_only=True)).copy() + if marker == "object": + return _expect(info["data"], list, f"{what} data") + # Attacker-chosen: echo a str capped at 40 chars; never repr() a structure (RecursionError on 3.10/3.11 at depth ~1000). + shown = marker if isinstance(marker, str) else type(marker).__name__ + raise SerializationError(f"Forged columnar payload: {what} type is {shown!r:.40}, expected 'numeric' or 'object'") + + def _na_safe_object_list(series: Any) -> list: """``series.tolist()`` with scalar pandas NA sentinels (pd.NA/NaT/NaN) mapped to None. @@ -805,6 +821,7 @@ def _deserialize_dataframe(self, data) -> pd.DataFrame: Raises: RuntimeError: If pandas not installed + SerializationError: forged document shape — see ``_expect`` / ``_column_values`` """ if not HAS_PANDAS: raise RuntimeError("Pandas not installed. Install with: pip install cachekit[data]") @@ -817,18 +834,10 @@ def _deserialize_dataframe(self, data) -> pd.DataFrame: serialized = unpackb_bounded(data, **self._msgpack_unpack_opts) serialized = _expect(serialized, dict, "document") - # Reconstruct DataFrame column by column columns_data = {} for col, col_info in _expect(serialized["data"], dict, "data").items(): - info = _expect(col_info, dict, f"column {col!r}") - if info["type"] == "numeric": - # Reconstruct from NumPy bytes; .copy() → writable, non-aliasing column (#157). - arr = np.frombuffer(info["data"], dtype=_dtype_from_untrusted(info["dtype"], numeric_only=True)).copy() - columns_data[col] = arr - else: - # Use object data directly - columns_data[col] = _expect(info["data"], list, f"column {col!r} data") - + what = f"column {col!r:.40}" # col is attacker-chosen: cap the echo + columns_data[col] = _column_values(_expect(col_info, dict, what), what) df = pd.DataFrame(columns_data, columns=_expect(serialized["columns"], list, "columns")) # Restore index if it was serialized @@ -874,6 +883,7 @@ def _deserialize_series(self, data) -> pd.Series: Raises: RuntimeError: If pandas not installed + SerializationError: forged document shape — see ``_expect`` / ``_column_values`` """ if not HAS_PANDAS: raise RuntimeError("Pandas not installed. Install with: pip install cachekit[data]") @@ -886,15 +896,7 @@ def _deserialize_series(self, data) -> pd.Series: serialized = unpackb_bounded(data, **self._msgpack_unpack_opts) serialized = _expect(serialized, dict, "document") - if serialized["type"] == "numeric": - # .copy() → writable Series values that do not alias the source buffer (#157). - values = np.frombuffer( - serialized["data"], dtype=_dtype_from_untrusted(serialized["dtype"], numeric_only=True) - ).copy() - else: - values = _expect(serialized["data"], list, "data") - - series = pd.Series(values, name=serialized["name"]) + series = pd.Series(_column_values(serialized, "series"), name=serialized["name"]) # Restore index if it was serialized if serialized["index"] is not None: diff --git a/tests/unit/test_auto_serializer_mutation_and_corruption.py b/tests/unit/test_auto_serializer_mutation_and_corruption.py index 52fc31c5..7dedb543 100644 --- a/tests/unit/test_auto_serializer_mutation_and_corruption.py +++ b/tests/unit/test_auto_serializer_mutation_and_corruption.py @@ -20,6 +20,8 @@ from __future__ import annotations +import functools + import msgpack import pytest @@ -139,29 +141,48 @@ def test_dataframe_corruption_raises_serialization_error(self) -> None: # dtype numpy accepts and pandas then asserts on (AssertionError, outside PAYLOAD_DECODE_ERRORS). NDARRAY_M8_2S = {"__ndarray__": True, "dtype": "M8[2s]", "shape": [1], "data": b"\x00" * 8} F8_COLUMN = {"type": "numeric", "data": b"\x00" * 8, "dtype": " bytes: + """A checksummed ``dataframe`` / ``series`` entry carrying ``body``.""" + return bytes(ByteStorage("msgpack").store(msgpack.packb(body), kind)) + + +def _columnar_entry(kind: str, column: dict) -> bytes: + """A checksummed ``dataframe`` / ``series`` entry whose single column is ``column``.""" + body = ( + {"columns": ["x"], "index": None, "data": {"x": column}} + if kind == "dataframe" + else {"name": None, "index": None, **column} + ) + return _entry(kind, body) @pytest.mark.unit class TestForgedColumnarPayloadIsRefused: """Forged DataFrame/Series documents are refused before pandas sees them (LAB-2503): a numeric column dtype the writer never emits (``M8[0ns]`` passes ``np.frombuffer`` and then kills the - process with SIGFPE inside pandas — uncatchable), and an ndarray smuggled via the ``__ndarray__`` - hook into a field the writer only ever fills with a list or a dict. + process with SIGFPE inside pandas — uncatchable), a column type marker other than the two the + writer emits, and an ndarray smuggled via the ``__ndarray__`` hook into a field the writer only + ever fills with a list or a dict. """ @pytest.mark.parametrize("dtype", ["M8[0ns]", "m8[0ns]", "U4"]) @pytest.mark.parametrize("kind", ["dataframe", "series"]) def test_forged_column_dtype_is_a_serialization_error(self, kind: str, dtype: str) -> None: - column = {"type": "numeric", "data": b"\x00" * 8, "dtype": dtype} - body = ( - {"columns": ["x"], "index": None, "data": {"x": column}} - if kind == "dataframe" - else {"name": None, "index": None, **column} - ) - entry = bytes(ByteStorage("msgpack").store(msgpack.packb(body), kind)) + entry = _columnar_entry(kind, {**F8_COLUMN, "dtype": dtype}) with pytest.raises(SerializationError, match="Forged columnar dtype"): AutoSerializer().deserialize(entry) + @pytest.mark.parametrize("marker", ["forged", DEEP_LIST], ids=["unknown-string", "list-nested-1000-deep"]) + @pytest.mark.parametrize("kind", ["dataframe", "series"]) + def test_unknown_column_type_marker_is_refused(self, kind: str, marker: object) -> None: + entry = _columnar_entry(kind, {"type": marker, "data": [1, 2]}) + with pytest.raises(SerializationError, match="Forged columnar payload: .* type is"): + AutoSerializer().deserialize(entry) + @pytest.mark.parametrize( "kind, body", [ @@ -184,6 +205,5 @@ def test_forged_column_dtype_is_a_serialization_error(self, kind: str, dtype: st ], ) def test_ndarray_where_the_writer_emits_a_list_or_dict_is_refused(self, kind: str, body: dict) -> None: - entry = bytes(ByteStorage("msgpack").store(msgpack.packb(body), kind)) with pytest.raises(SerializationError, match="Forged columnar payload"): - AutoSerializer().deserialize(entry) + AutoSerializer().deserialize(_entry(kind, body))