gh-146287: Fix signed/unsigned mismatch in _hashlib_hmac_digest_size (GH-148407)

* gh-146287: use signed type for HMAC digest size to prevent unsigned wrapping

Change _hashlib_hmac_digest_size() return type from unsigned int to int
so that a hypothetical negative return from EVP_MD_size() is not
silently wrapped to a large positive value. Add an explicit check for
negative digest_size in the legacy OpenSSL path, and use SystemError
(not ValueError) since these conditions indicate internal invariant
violations. Also add debug-build asserts to EVP_get_block_size and
EVP_get_digest_size documenting that the hash context is always
initialized.

Co-authored-by: Bénédikt Tran <10796600+picnixz@users.noreply.github.com>

Author: gpshead

Modules/_hashopenssl.c

@@ -1006,6 +1006,7 @@ _hashlib_HASH_get_blocksize(PyObject *op, void *Py_UNUSED(closure))
 {
     HASHobject *self = HASHobject_CAST(op);
     long block_size = EVP_MD_CTX_block_size(self->ctx);
+    assert(block_size > 0);
     return PyLong_FromLong(block_size);
 }
 
@@ -1014,6 +1015,7 @@ _hashlib_HASH_get_digestsize(PyObject *op, void *Py_UNUSED(closure))
 {
     HASHobject *self = HASHobject_CAST(op);
     long size = EVP_MD_CTX_size(self->ctx);
+    assert(size > 0);
     return PyLong_FromLong(size);
 }
 
@@ -2200,7 +2202,7 @@ _hashlib_hmac_new_impl(PyObject *module, Py_buffer *key, PyObject *msg_obj,
  *
  * On error, set an exception and return BAD_DIGEST_SIZE.
  */
-static unsigned int
+static int
 _hashlib_hmac_digest_size(HMACobject *self)
 {
     assert(EVP_MAX_MD_SIZE < INT_MAX);
@@ -2215,15 +2217,18 @@ _hashlib_hmac_digest_size(HMACobject *self)
     }
     int digest_size = EVP_MD_size(md);
     /* digest_size < 0 iff EVP_MD context is NULL (which is impossible here) */
-    assert(digest_size >= 0);
     assert(digest_size <= (int)EVP_MAX_MD_SIZE);
+    if (digest_size < 0) {
+        raise_ssl_error(PyExc_SystemError, "invalid digest size");
+        return BAD_DIGEST_SIZE;
+    }
 #endif
     /* digest_size == 0 means that the context is not entirely initialized */
     if (digest_size == 0) {
-        raise_ssl_error(PyExc_ValueError, "missing digest size");
+        raise_ssl_error(PyExc_SystemError, "missing digest size");
         return BAD_DIGEST_SIZE;
     }
-    return (unsigned int)digest_size;
+    return (int)digest_size;
 }
 
 static int
@@ -2321,7 +2326,7 @@ _hashlib_HMAC_update_impl(HMACobject *self, PyObject *msg)
 static Py_ssize_t
 _hmac_digest(HMACobject *self, unsigned char *buf)
 {
-    unsigned int digest_size = _hashlib_hmac_digest_size(self);
+    int digest_size = _hashlib_hmac_digest_size(self);
     assert(digest_size <= EVP_MAX_MD_SIZE);
     if (digest_size == BAD_DIGEST_SIZE) {
         assert(PyErr_Occurred());
@@ -2386,7 +2391,7 @@ static PyObject *
 _hashlib_hmac_get_digest_size(PyObject *op, void *Py_UNUSED(closure))
 {
     HMACobject *self = HMACobject_CAST(op);
-    unsigned int size = _hashlib_hmac_digest_size(self);
+    int size = _hashlib_hmac_digest_size(self);
     return size == BAD_DIGEST_SIZE ? NULL : PyLong_FromLong(size);
 }