Switch the fingerprint algo to xxh3_128

docs/concepts/caching.rst

@@ -357,6 +357,21 @@ Data version
 
 Caching requires the ability to uniquely identify data (e.g., create a hash). By default, all Python primitive types (``int``, ``str``, ``dict``, etc.) are supported and more types can be added via extensions (e.g., ``pandas``). For types not explicitly supported, caching can still function by versioning the object's internal ``__dict__`` instead. However, this could be expensive to compute or less reliable than alternatives.
 
+Hashing backend
+~~~~~~~~~~~~~~~
+
+By default, hashing uses the standard library's ``hashlib.md5`` (chosen for speed, not for cryptographic security). Installing the optional `xxhash <https://github.com/ifduyue/python-xxhash>`_ dependency switches the backend to ``xxhash.xxh3_128``, which is substantially faster on buffer-bound paths (e.g. numpy arrays, polars DataFrames):
+
+.. code-block:: console
+
+    pip install "apache-hamilton[performance]"
+
+Both backends produce a 16-byte digest, so cache keys and ``data_version`` strings are unaffected in shape. However, the two algorithms produce different digests for the same input, so **installing or removing xxhash changes every ``data_version`` in your dataflow**, invalidating previously persisted caches. Cached results are just recomputed on the next run, at the cost of losing the benefit of the existing cache.
+
+.. note::
+
+    Because the resolved backend is process-wide, keep it consistent across the environments that share a cache (e.g. all workers writing to the same cache store) to avoid needless cache misses.
+
 Recursion depth
 ~~~~~~~~~~~~~~~
 

hamilton/caching/fingerprinting.py

@@ -33,13 +33,15 @@
 implementation should pass the `depth` parameter to prevent `RecursionError`.
 """
 
+from __future__ import annotations
+
 import base64
 import datetime
 import functools
-import hashlib
 import logging
 import sys
 from collections.abc import Mapping, Sequence, Set
+from typing import TYPE_CHECKING
 
 from hamilton.experimental import h_databackends
 
@@ -49,6 +51,31 @@
 except ImportError:
     NoneType = type(None)
 
+if TYPE_CHECKING:
+    from collections.abc import Buffer, Callable
+    from typing import Protocol
+
+    class Hash(Protocol):
+        def digest(self) -> bytes: ...
+
+
+hash_func: Callable[[str | Buffer], Hash]
+try:
+    # xxh3_128 produces a 16-byte digest (24 base64url chars, the same width as the
+    # md5 it replaces) while running substantially faster on buffer-bound paths.
+    # TODO(2.0): revisit once/if a dependency-free `apache-hamilton-core` package
+    # exists, so this optional performance dependency lands in the right tier.
+    import xxhash
+
+    hash_func = xxhash.xxh3_128
+except (ModuleNotFoundError, AttributeError):
+    # ModuleNotFoundError covers xxhash not being installed; AttributeError
+    # covers an xxhash older than 0.8.0 (which added xxh3_128).
+    # usedforsecurity=False avoids a ValueError on FIPS-mode Python; it
+    # doesn't change the digest.
+    import hashlib
+
+    hash_func = functools.partial(hashlib.md5, usedforsecurity=False)
 
 logger = logging.getLogger("hamilton.caching")
 
@@ -81,8 +108,16 @@ def _hash_bytes(data: bytes) -> str:
 
     All hashing in this module routes through this single helper so the
     underlying hashing algorithm can be changed in exactly one place.
+
+    Uses the non-cryptographic xxh3_128 algorithm when the optional
+    ``xxhash`` package is installed (see the ``performance`` extra),
+    falling back to hashlib's md5 otherwise. Both produce a 16-byte digest
+    (24 base64url chars), so digest width is unaffected either way, but the
+    two algorithms don't produce the same bytes for the same input:
+    fingerprints are stable within an environment, not across installs with
+    a different backend.
     """
-    return _compact_hash(hashlib.md5(data).digest())
+    return _compact_hash(hash_func(data).digest())
 
 
 @functools.singledispatch

pyproject.toml

@@ -70,6 +70,10 @@ experiments = [
 lsp = ["apache-hamilton-lsp"]
 openlineage = ["openlineage-python"]
 pandera = ["pandera"]
+performance = [
+    # Internal performance boosters go here
+    "xxhash>=0.8.0",
+]
 pydantic = ["pydantic>=2.0"]
 pyspark = [
   # we have to run these dependencies because Spark does not check to ensure the right target was called
@@ -134,6 +138,7 @@ test = [
   "xgboost; python_version < '3.14'",
   "xlsx2csv", # for excel data loader
   "xlsxwriter",  # Excel export requires 'xlsxwriter'
+  "xxhash>=0.8.0", # exercises the high-performance fingerprinting path in tests
 ]
 docs = [
   {include-group = "dev"},

tests/caching/test_fingerprinting.py

@@ -20,13 +20,28 @@
 the original `hash_value()` and the `hash_primitive()` functions.
 """
 
+import functools
+import hashlib
+import importlib
+import sys
+import types
+
 import numpy as np
 import pandas as pd
 import pytest
 
 from hamilton.caching import fingerprinting
 
 
+@pytest.fixture
+def force_md5_backend(monkeypatch):
+    """Force the hashlib.md5 fallback backend, regardless of whether xxhash
+    is installed in the current environment."""
+    monkeypatch.setattr(
+        fingerprinting, "hash_func", functools.partial(hashlib.md5, usedforsecurity=False)
+    )
+
+
 def test_hash_none():
     fingerprint = fingerprinting.hash_value(None)
     assert fingerprint == "<none>"
@@ -146,11 +161,11 @@ def __init__(self, obj):
 @pytest.mark.parametrize(
     ("obj", "expected_hash"),
     [
-        ("hello-world", "L1Q1Kh6_t1atHO_H8RbBeA=="),
-        (17.31231, "mJPTpPyXDSZgU-u8NuztIQ=="),
-        (16474, "6MgAp1NbMW0ZZpe_8iKVsg=="),
-        (True, "J2eGynSuIpd5bwVQzO9VVg=="),
-        (b"\x951!\x89u=\xe6\xadG\xdf", "d1DufDgRQmqi9Kt4Z2PeUQ=="),
+        ("hello-world", "EXXR8_e47ElS18aP2lThJA=="),
+        (17.31231, "tVUSIslYiBcW52c-7w4gvA=="),
+        (16474, "FAJ-iXM_Hwg9TCRreY8AyA=="),
+        (True, "qkJEg3-XQKmGWk5sWqmonw=="),
+        (b"\x951!\x89u=\xe6\xadG\xdf", "pPTyYkSU_x7NLB1Fp_YTyA=="),
     ],
 )
 def test_hash_primitive(obj, expected_hash):
@@ -161,8 +176,8 @@ def test_hash_primitive(obj, expected_hash):
 @pytest.mark.parametrize(
     ("obj", "expected_hash"),
     [
-        ([0, True, "hello-world"], "mlOjj4yeCrSDFSn5zgdEIg=="),
-        ((17.0, False, "world"), "BcRSGfyKeIOdym9B6TmAyQ=="),
+        ([0, True, "hello-world"], "I98OkNhfxtScJrYNTs4ZfQ=="),
+        ((17.0, False, "world"), "catgOMSnsbQj1_KELNQscw=="),
     ],
 )
 def test_hash_sequence(obj, expected_hash):
@@ -173,7 +188,7 @@ def test_hash_sequence(obj, expected_hash):
 def test_hash_equals_for_different_sequence_types():
     list_obj = [0, True, "hello-world"]
     tuple_obj = (0, True, "hello-world")
-    expected_hash = "mlOjj4yeCrSDFSn5zgdEIg=="
+    expected_hash = "I98OkNhfxtScJrYNTs4ZfQ=="
 
     list_fingerprint = fingerprinting.hash_sequence(list_obj)
     tuple_fingerprint = fingerprinting.hash_sequence(tuple_obj)
@@ -182,7 +197,7 @@ def test_hash_equals_for_different_sequence_types():
 
 def test_hash_ordered_mapping():
     obj = {0: True, "key": "value", 17.0: None}
-    expected_hash = "GyxyI9-pq-EJJvSAIN509g=="
+    expected_hash = "zX6MzhWGAOvxateHIPxOvA=="
     fingerprint = fingerprinting.hash_mapping(obj, ignore_order=False)
     assert fingerprint == expected_hash
 
@@ -197,7 +212,7 @@ def test_hash_mapping_where_order_matters():
 
 def test_hash_unordered_mapping():
     obj = {0: True, "key": "value", 17.0: None}
-    expected_hash = "cDuuL2eA3DaSWlWW3u7o9g=="
+    expected_hash = "4cnTFA4MEEzmBN4a04k6tA=="
     fingerprint = fingerprinting.hash_mapping(obj, ignore_order=True)
     assert fingerprint == expected_hash
 
@@ -212,20 +227,137 @@ def test_hash_mapping_where_order_doesnt_matter():
 
 def test_hash_set():
     obj = {0, True, "key", "value", 17.0, None}
-    expected_hash = "E_f_tjbi6qn7KL3NUCZayg=="
+    expected_hash = "mswHhNBBYN5mv6i-LcEeVw=="
     fingerprint = fingerprinting.hash_set(obj)
     assert fingerprint == expected_hash
 
 
 def test_hash_numpy():
     # dtype is pinned explicitly so the literal digest is reproducible across
     # platforms (the default integer dtype is platform-dependent).
+    array = np.array([[0, 1], [2, 3]], dtype=np.int64)
+    expected_hash = "Y1uek_eQTHejo2YtRvdWPQ=="
+    fingerprint = fingerprinting.hash_value(array)
+    assert fingerprint == expected_hash
+
+
+# ---------------------------------------------------------------------------
+# hashlib.md5 fallback backend
+#
+# These mirror the pinned tests above but force `hash_func` to the hashlib.md5
+# fallback (via the `force_md5_backend` fixture) so the fallback path used
+# when `xxhash` isn't installed is verified regardless of what's installed in
+# the environment running the suite. Expected digests are the pre-xxh3_128
+# values this module used before xxhash became the default backend.
+# ---------------------------------------------------------------------------
+
+
+@pytest.mark.usefixtures("force_md5_backend")
+@pytest.mark.parametrize(
+    ("obj", "expected_hash"),
+    [
+        ("hello-world", "L1Q1Kh6_t1atHO_H8RbBeA=="),
+        (17.31231, "mJPTpPyXDSZgU-u8NuztIQ=="),
+        (16474, "6MgAp1NbMW0ZZpe_8iKVsg=="),
+        (True, "J2eGynSuIpd5bwVQzO9VVg=="),
+        (b"\x951!\x89u=\xe6\xadG\xdf", "d1DufDgRQmqi9Kt4Z2PeUQ=="),
+    ],
+)
+def test_hash_primitive_md5_fallback(obj, expected_hash):
+    fingerprint = fingerprinting.hash_primitive(obj)
+    assert fingerprint == expected_hash
+
+
+@pytest.mark.usefixtures("force_md5_backend")
+@pytest.mark.parametrize(
+    ("obj", "expected_hash"),
+    [
+        ([0, True, "hello-world"], "mlOjj4yeCrSDFSn5zgdEIg=="),
+        ((17.0, False, "world"), "BcRSGfyKeIOdym9B6TmAyQ=="),
+    ],
+)
+def test_hash_sequence_md5_fallback(obj, expected_hash):
+    fingerprint = fingerprinting.hash_sequence(obj)
+    assert fingerprint == expected_hash
+
+
+@pytest.mark.usefixtures("force_md5_backend")
+def test_hash_ordered_mapping_md5_fallback():
+    obj = {0: True, "key": "value", 17.0: None}
+    expected_hash = "GyxyI9-pq-EJJvSAIN509g=="
+    fingerprint = fingerprinting.hash_mapping(obj, ignore_order=False)
+    assert fingerprint == expected_hash
+
+
+@pytest.mark.usefixtures("force_md5_backend")
+def test_hash_unordered_mapping_md5_fallback():
+    obj = {0: True, "key": "value", 17.0: None}
+    expected_hash = "cDuuL2eA3DaSWlWW3u7o9g=="
+    fingerprint = fingerprinting.hash_mapping(obj, ignore_order=True)
+    assert fingerprint == expected_hash
+
+
+@pytest.mark.usefixtures("force_md5_backend")
+def test_hash_set_md5_fallback():
+    obj = {0, True, "key", "value", 17.0, None}
+    expected_hash = "E_f_tjbi6qn7KL3NUCZayg=="
+    fingerprint = fingerprinting.hash_set(obj)
+    assert fingerprint == expected_hash
+
+
+@pytest.mark.usefixtures("force_md5_backend")
+def test_hash_numpy_md5_fallback():
     array = np.array([[0, 1], [2, 3]], dtype=np.int64)
     expected_hash = "024zwZIcWy6r4dlX4AMTow=="
     fingerprint = fingerprinting.hash_value(array)
     assert fingerprint == expected_hash
 
 
+def test_hash_bytes_digest_width():
+    """Both backends produce a 16-byte digest (24 base64url chars), so swapping
+    the algorithm doesn't change the shape of cache keys / `data_version` strings.
+    """
+    assert len(fingerprinting._hash_bytes(b"x")) == 24
+
+
+@pytest.mark.usefixtures("force_md5_backend")
+def test_hash_bytes_digest_width_md5_fallback():
+    assert len(fingerprinting._hash_bytes(b"x")) == 24
+
+
+# ---------------------------------------------------------------------------
+# Import-time backend resolution
+#
+# These reload the module to actually exercise the try/except at import time
+# (as opposed to `force_md5_backend`, which only overrides the already-resolved
+# `hash_func`), then restore the module to its real, ambient-environment state
+# so later tests aren't affected.
+# ---------------------------------------------------------------------------
+
+
+def test_falls_back_when_xxhash_not_installed(monkeypatch):
+    """Simulate xxhash being absent: `import xxhash` raises ModuleNotFoundError."""
+    monkeypatch.setitem(sys.modules, "xxhash", None)
+    try:
+        reloaded = importlib.reload(fingerprinting)
+        assert reloaded.hash_func.func is hashlib.md5
+        assert reloaded.hash_func.keywords == {"usedforsecurity": False}
+    finally:
+        importlib.reload(fingerprinting)
+
+
+def test_falls_back_when_xxhash_lacks_xxh3_128(monkeypatch):
+    """Simulate an xxhash older than 0.8.0, which doesn't define xxh3_128."""
+    stub = types.ModuleType("xxhash")
+    monkeypatch.setitem(sys.modules, "xxhash", stub)
+    try:
+        reloaded = importlib.reload(fingerprinting)
+        assert reloaded.hash_func.func is hashlib.md5
+        assert reloaded.hash_func.keywords == {"usedforsecurity": False}
+    finally:
+        importlib.reload(fingerprinting)
+
+
 def test_hash_numpy_different_shapes_differ():
     """Arrays with the same raw bytes but different shapes must hash differently."""
     a = np.array([1, 2, 3, 4, 5, 6])