diff --git a/docs/concepts/caching.rst b/docs/concepts/caching.rst index 33aa6b645..2a2d4c6b2 100644 --- a/docs/concepts/caching.rst +++ b/docs/concepts/caching.rst @@ -357,6 +357,21 @@ Data version Caching requires the ability to uniquely identify data (e.g., create a hash). By default, all Python primitive types (``int``, ``str``, ``dict``, etc.) are supported and more types can be added via extensions (e.g., ``pandas``). For types not explicitly supported, caching can still function by versioning the object's internal ``__dict__`` instead. However, this could be expensive to compute or less reliable than alternatives. +Hashing backend +~~~~~~~~~~~~~~~ + +By default, hashing uses the standard library's ``hashlib.md5`` (chosen for speed, not for cryptographic security). Installing the optional `xxhash `_ dependency switches the backend to ``xxhash.xxh3_128``, which is substantially faster on buffer-bound paths (e.g. numpy arrays, polars DataFrames): + +.. code-block:: console + + pip install "apache-hamilton[performance]" + +Both backends produce a 16-byte digest, so cache keys and ``data_version`` strings are unaffected in shape. However, the two algorithms produce different digests for the same input, so **installing or removing xxhash changes every ``data_version`` in your dataflow**, invalidating previously persisted caches. Cached results are just recomputed on the next run, at the cost of losing the benefit of the existing cache. + +.. note:: + + Because the resolved backend is process-wide, keep it consistent across the environments that share a cache (e.g. all workers writing to the same cache store) to avoid needless cache misses. + Recursion depth ~~~~~~~~~~~~~~~ diff --git a/hamilton/caching/fingerprinting.py b/hamilton/caching/fingerprinting.py index df488b281..8e30437ab 100644 --- a/hamilton/caching/fingerprinting.py +++ b/hamilton/caching/fingerprinting.py @@ -33,13 +33,15 @@ implementation should pass the `depth` parameter to prevent `RecursionError`. """ +from __future__ import annotations + import base64 import datetime import functools -import hashlib import logging import sys from collections.abc import Mapping, Sequence, Set +from typing import TYPE_CHECKING from hamilton.experimental import h_databackends @@ -49,6 +51,31 @@ except ImportError: NoneType = type(None) +if TYPE_CHECKING: + from collections.abc import Buffer, Callable + from typing import Protocol + + class Hash(Protocol): + def digest(self) -> bytes: ... + + +hash_func: Callable[[str | Buffer], Hash] +try: + # xxh3_128 produces a 16-byte digest (24 base64url chars, the same width as the + # md5 it replaces) while running substantially faster on buffer-bound paths. + # TODO(2.0): revisit once/if a dependency-free `apache-hamilton-core` package + # exists, so this optional performance dependency lands in the right tier. + import xxhash + + hash_func = xxhash.xxh3_128 +except (ModuleNotFoundError, AttributeError): + # ModuleNotFoundError covers xxhash not being installed; AttributeError + # covers an xxhash older than 0.8.0 (which added xxh3_128). + # usedforsecurity=False avoids a ValueError on FIPS-mode Python; it + # doesn't change the digest. + import hashlib + + hash_func = functools.partial(hashlib.md5, usedforsecurity=False) logger = logging.getLogger("hamilton.caching") @@ -81,8 +108,16 @@ def _hash_bytes(data: bytes) -> str: All hashing in this module routes through this single helper so the underlying hashing algorithm can be changed in exactly one place. + + Uses the non-cryptographic xxh3_128 algorithm when the optional + ``xxhash`` package is installed (see the ``performance`` extra), + falling back to hashlib's md5 otherwise. Both produce a 16-byte digest + (24 base64url chars), so digest width is unaffected either way, but the + two algorithms don't produce the same bytes for the same input: + fingerprints are stable within an environment, not across installs with + a different backend. """ - return _compact_hash(hashlib.md5(data).digest()) + return _compact_hash(hash_func(data).digest()) @functools.singledispatch diff --git a/pyproject.toml b/pyproject.toml index 74ad5d586..df3c465f7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -70,6 +70,10 @@ experiments = [ lsp = ["apache-hamilton-lsp"] openlineage = ["openlineage-python"] pandera = ["pandera"] +performance = [ + # Internal performance boosters go here + "xxhash>=0.8.0", +] pydantic = ["pydantic>=2.0"] pyspark = [ # we have to run these dependencies because Spark does not check to ensure the right target was called @@ -134,6 +138,7 @@ test = [ "xgboost; python_version < '3.14'", "xlsx2csv", # for excel data loader "xlsxwriter", # Excel export requires 'xlsxwriter' + "xxhash>=0.8.0", # exercises the high-performance fingerprinting path in tests ] docs = [ {include-group = "dev"}, diff --git a/tests/caching/test_fingerprinting.py b/tests/caching/test_fingerprinting.py index 9c00e072e..c1593035b 100644 --- a/tests/caching/test_fingerprinting.py +++ b/tests/caching/test_fingerprinting.py @@ -20,6 +20,12 @@ the original `hash_value()` and the `hash_primitive()` functions. """ +import functools +import hashlib +import importlib +import sys +import types + import numpy as np import pandas as pd import pytest @@ -27,6 +33,15 @@ from hamilton.caching import fingerprinting +@pytest.fixture +def force_md5_backend(monkeypatch): + """Force the hashlib.md5 fallback backend, regardless of whether xxhash + is installed in the current environment.""" + monkeypatch.setattr( + fingerprinting, "hash_func", functools.partial(hashlib.md5, usedforsecurity=False) + ) + + def test_hash_none(): fingerprint = fingerprinting.hash_value(None) assert fingerprint == "" @@ -146,11 +161,11 @@ def __init__(self, obj): @pytest.mark.parametrize( ("obj", "expected_hash"), [ - ("hello-world", "L1Q1Kh6_t1atHO_H8RbBeA=="), - (17.31231, "mJPTpPyXDSZgU-u8NuztIQ=="), - (16474, "6MgAp1NbMW0ZZpe_8iKVsg=="), - (True, "J2eGynSuIpd5bwVQzO9VVg=="), - (b"\x951!\x89u=\xe6\xadG\xdf", "d1DufDgRQmqi9Kt4Z2PeUQ=="), + ("hello-world", "EXXR8_e47ElS18aP2lThJA=="), + (17.31231, "tVUSIslYiBcW52c-7w4gvA=="), + (16474, "FAJ-iXM_Hwg9TCRreY8AyA=="), + (True, "qkJEg3-XQKmGWk5sWqmonw=="), + (b"\x951!\x89u=\xe6\xadG\xdf", "pPTyYkSU_x7NLB1Fp_YTyA=="), ], ) def test_hash_primitive(obj, expected_hash): @@ -161,8 +176,8 @@ def test_hash_primitive(obj, expected_hash): @pytest.mark.parametrize( ("obj", "expected_hash"), [ - ([0, True, "hello-world"], "mlOjj4yeCrSDFSn5zgdEIg=="), - ((17.0, False, "world"), "BcRSGfyKeIOdym9B6TmAyQ=="), + ([0, True, "hello-world"], "I98OkNhfxtScJrYNTs4ZfQ=="), + ((17.0, False, "world"), "catgOMSnsbQj1_KELNQscw=="), ], ) def test_hash_sequence(obj, expected_hash): @@ -173,7 +188,7 @@ def test_hash_sequence(obj, expected_hash): def test_hash_equals_for_different_sequence_types(): list_obj = [0, True, "hello-world"] tuple_obj = (0, True, "hello-world") - expected_hash = "mlOjj4yeCrSDFSn5zgdEIg==" + expected_hash = "I98OkNhfxtScJrYNTs4ZfQ==" list_fingerprint = fingerprinting.hash_sequence(list_obj) tuple_fingerprint = fingerprinting.hash_sequence(tuple_obj) @@ -182,7 +197,7 @@ def test_hash_equals_for_different_sequence_types(): def test_hash_ordered_mapping(): obj = {0: True, "key": "value", 17.0: None} - expected_hash = "GyxyI9-pq-EJJvSAIN509g==" + expected_hash = "zX6MzhWGAOvxateHIPxOvA==" fingerprint = fingerprinting.hash_mapping(obj, ignore_order=False) assert fingerprint == expected_hash @@ -197,7 +212,7 @@ def test_hash_mapping_where_order_matters(): def test_hash_unordered_mapping(): obj = {0: True, "key": "value", 17.0: None} - expected_hash = "cDuuL2eA3DaSWlWW3u7o9g==" + expected_hash = "4cnTFA4MEEzmBN4a04k6tA==" fingerprint = fingerprinting.hash_mapping(obj, ignore_order=True) assert fingerprint == expected_hash @@ -212,7 +227,7 @@ def test_hash_mapping_where_order_doesnt_matter(): def test_hash_set(): obj = {0, True, "key", "value", 17.0, None} - expected_hash = "E_f_tjbi6qn7KL3NUCZayg==" + expected_hash = "mswHhNBBYN5mv6i-LcEeVw==" fingerprint = fingerprinting.hash_set(obj) assert fingerprint == expected_hash @@ -220,12 +235,129 @@ def test_hash_set(): def test_hash_numpy(): # dtype is pinned explicitly so the literal digest is reproducible across # platforms (the default integer dtype is platform-dependent). + array = np.array([[0, 1], [2, 3]], dtype=np.int64) + expected_hash = "Y1uek_eQTHejo2YtRvdWPQ==" + fingerprint = fingerprinting.hash_value(array) + assert fingerprint == expected_hash + + +# --------------------------------------------------------------------------- +# hashlib.md5 fallback backend +# +# These mirror the pinned tests above but force `hash_func` to the hashlib.md5 +# fallback (via the `force_md5_backend` fixture) so the fallback path used +# when `xxhash` isn't installed is verified regardless of what's installed in +# the environment running the suite. Expected digests are the pre-xxh3_128 +# values this module used before xxhash became the default backend. +# --------------------------------------------------------------------------- + + +@pytest.mark.usefixtures("force_md5_backend") +@pytest.mark.parametrize( + ("obj", "expected_hash"), + [ + ("hello-world", "L1Q1Kh6_t1atHO_H8RbBeA=="), + (17.31231, "mJPTpPyXDSZgU-u8NuztIQ=="), + (16474, "6MgAp1NbMW0ZZpe_8iKVsg=="), + (True, "J2eGynSuIpd5bwVQzO9VVg=="), + (b"\x951!\x89u=\xe6\xadG\xdf", "d1DufDgRQmqi9Kt4Z2PeUQ=="), + ], +) +def test_hash_primitive_md5_fallback(obj, expected_hash): + fingerprint = fingerprinting.hash_primitive(obj) + assert fingerprint == expected_hash + + +@pytest.mark.usefixtures("force_md5_backend") +@pytest.mark.parametrize( + ("obj", "expected_hash"), + [ + ([0, True, "hello-world"], "mlOjj4yeCrSDFSn5zgdEIg=="), + ((17.0, False, "world"), "BcRSGfyKeIOdym9B6TmAyQ=="), + ], +) +def test_hash_sequence_md5_fallback(obj, expected_hash): + fingerprint = fingerprinting.hash_sequence(obj) + assert fingerprint == expected_hash + + +@pytest.mark.usefixtures("force_md5_backend") +def test_hash_ordered_mapping_md5_fallback(): + obj = {0: True, "key": "value", 17.0: None} + expected_hash = "GyxyI9-pq-EJJvSAIN509g==" + fingerprint = fingerprinting.hash_mapping(obj, ignore_order=False) + assert fingerprint == expected_hash + + +@pytest.mark.usefixtures("force_md5_backend") +def test_hash_unordered_mapping_md5_fallback(): + obj = {0: True, "key": "value", 17.0: None} + expected_hash = "cDuuL2eA3DaSWlWW3u7o9g==" + fingerprint = fingerprinting.hash_mapping(obj, ignore_order=True) + assert fingerprint == expected_hash + + +@pytest.mark.usefixtures("force_md5_backend") +def test_hash_set_md5_fallback(): + obj = {0, True, "key", "value", 17.0, None} + expected_hash = "E_f_tjbi6qn7KL3NUCZayg==" + fingerprint = fingerprinting.hash_set(obj) + assert fingerprint == expected_hash + + +@pytest.mark.usefixtures("force_md5_backend") +def test_hash_numpy_md5_fallback(): array = np.array([[0, 1], [2, 3]], dtype=np.int64) expected_hash = "024zwZIcWy6r4dlX4AMTow==" fingerprint = fingerprinting.hash_value(array) assert fingerprint == expected_hash +def test_hash_bytes_digest_width(): + """Both backends produce a 16-byte digest (24 base64url chars), so swapping + the algorithm doesn't change the shape of cache keys / `data_version` strings. + """ + assert len(fingerprinting._hash_bytes(b"x")) == 24 + + +@pytest.mark.usefixtures("force_md5_backend") +def test_hash_bytes_digest_width_md5_fallback(): + assert len(fingerprinting._hash_bytes(b"x")) == 24 + + +# --------------------------------------------------------------------------- +# Import-time backend resolution +# +# These reload the module to actually exercise the try/except at import time +# (as opposed to `force_md5_backend`, which only overrides the already-resolved +# `hash_func`), then restore the module to its real, ambient-environment state +# so later tests aren't affected. +# --------------------------------------------------------------------------- + + +def test_falls_back_when_xxhash_not_installed(monkeypatch): + """Simulate xxhash being absent: `import xxhash` raises ModuleNotFoundError.""" + monkeypatch.setitem(sys.modules, "xxhash", None) + try: + reloaded = importlib.reload(fingerprinting) + assert reloaded.hash_func.func is hashlib.md5 + assert reloaded.hash_func.keywords == {"usedforsecurity": False} + finally: + importlib.reload(fingerprinting) + + +def test_falls_back_when_xxhash_lacks_xxh3_128(monkeypatch): + """Simulate an xxhash older than 0.8.0, which doesn't define xxh3_128.""" + stub = types.ModuleType("xxhash") + monkeypatch.setitem(sys.modules, "xxhash", stub) + try: + reloaded = importlib.reload(fingerprinting) + assert reloaded.hash_func.func is hashlib.md5 + assert reloaded.hash_func.keywords == {"usedforsecurity": False} + finally: + importlib.reload(fingerprinting) + + def test_hash_numpy_different_shapes_differ(): """Arrays with the same raw bytes but different shapes must hash differently.""" a = np.array([1, 2, 3, 4, 5, 6])