gh-159786: Convert HACL* update() errors in _hmac after releasing the lock

`_hacl_hmac_state_update_once()` converted the HACL* exit code into an
exception while the thread state was detached and, for HMAC.update(),
while the object's mutex was held. On debug builds, this re-attached the
thread state under the mutex and then called PyErr_Occurred() with the
thread state detached in the chunking loop of `_hacl_hmac_state_update()`.

Return the raw HACL* exit code instead and convert it in `_hmac.new()` and
`_hmac.HMAC.update()` once the thread state is re-attached and the mutex
is released, as `hmac_digest_compute_locked()` already does. Release
builds keep ignoring the update() exit code.

Since `_hacl_convert_errno()` is now always called with an attached
thread state, remove the `PyGILState_Ensure()` added by gh-135561 and
assert that a thread state is attached instead.

Author: picnixz

Misc/NEWS.d/next/Library/2026-10-19-21-20-28.gh-issue-159786.fThehQ.rst

@@ -0,0 +1,4 @@
+Fix a crash on DEBUG builds when the HACL* routine used by
+:meth:`hmac.HMAC.update` fails. This only happens for messages larger
+than 4 GiB once the maximum input length of the underlying hash function
+is exceeded. Patch by Bénédikt Tran.

Modules/hmacmodule.c

@@ -379,16 +379,17 @@ narrow_hmac_hash_kind(hmacmodule_state *state, HMAC_Hash_Kind kind)
  *
  * If 'code' represents a successful operation, this returns 0.
  * Otherwise, this sets an appropriate exception and returns -1.
+ *
+ * The caller must have an attached thread state.
  */
 static int
 _hacl_convert_errno(hacl_errno_t code)
 {
-    assert(PyGILState_GetThisThreadState() != NULL);
+    assert(PyThreadState_GetUnchecked() != NULL);
     if (code == Hacl_Streaming_Types_Success) {
         return 0;
     }
 
-    PyGILState_STATE gstate = PyGILState_Ensure();
     switch (code) {
         case Hacl_Streaming_Types_InvalidAlgorithm: {
             PyErr_SetString(PyExc_ValueError, "invalid HACL* algorithm");
@@ -413,7 +414,6 @@ _hacl_convert_errno(hacl_errno_t code)
             break;
         }
     }
-    PyGILState_Release(gstate);
     return -1;
 }
 
@@ -451,18 +451,18 @@ _hacl_hmac_state_free(HACL_HMAC_state *state)
  *
  * On DEBUG builds, the update() call is verified.
  *
- * Return 0 on success; otherwise, set an exception and return -1 on failure.
+ * Return the HACL* exit code. No exception is set, so that this
+ * can be called without an attached thread state or with a lock held.
 */
-static int
+static hacl_errno_t
 _hacl_hmac_state_update_once(HACL_HMAC_state *state,
                              uint8_t *buf, uint32_t len)
 {
 #ifndef NDEBUG
-    hacl_errno_t code = Hacl_Streaming_HMAC_update(state, buf, len);
-    return _hacl_convert_errno(code);
+    return Hacl_Streaming_HMAC_update(state, buf, len);
 #else
     (void)Hacl_Streaming_HMAC_update(state, buf, len);
-    return 0;
+    return Hacl_Streaming_Types_Success;
 #endif
 }
 
@@ -471,17 +471,20 @@ _hacl_hmac_state_update_once(HACL_HMAC_state *state,
  *
  * On DEBUG builds, each update() call is verified.
  *
- * Return 0 on success; otherwise, set an exception and return -1 on failure.
+ * Return the HACL* exit code. No exception is set, so that this
+ * can be called without an attached thread state or with a lock held.
+ * Use _hacl_convert_errno() afterwards to set an appropriate exception.
  */
-static int
+static hacl_errno_t
 _hacl_hmac_state_update(HACL_HMAC_state *state, uint8_t *buf, Py_ssize_t len)
 {
     assert(len >= 0);
 #ifdef Py_HMAC_SSIZE_LARGER_THAN_UINT32
     while (len > UINT32_MAX_AS_SSIZE_T) {
-        if (_hacl_hmac_state_update_once(state, buf, UINT32_MAX) < 0) {
-            assert(PyErr_Occurred());
-            return -1;
+        hacl_errno_t code = _hacl_hmac_state_update_once(state, buf,
+                                                         UINT32_MAX);
+        if (code != Hacl_Streaming_Types_Success) {
+            return code;
         }
         buf += UINT32_MAX;
         len -= UINT32_MAX;
@@ -786,17 +789,19 @@ _hmac_new_impl(PyObject *module, PyObject *keyobj, PyObject *msgobj,
         GET_BUFFER_VIEW_OR_ERROR(msgobj, &msg, goto error);
         /* Do not use self->mutex here as this is the constructor
          * where it is not yet possible to have concurrent access. */
+        hacl_errno_t code;
         HASHLIB_EXTERNAL_INSTRUCTIONS_UNLOCKED(
             msg.len,
-            rc = _hacl_hmac_state_update(self->state, msg.buf, msg.len)
+            code = _hacl_hmac_state_update(self->state, msg.buf, msg.len)
         );
         PyBuffer_Release(&msg);
 #ifndef NDEBUG
+        rc = _hacl_convert_errno(code);
         if (rc < 0) {
             goto error;
         }
 #else
-        (void)rc;
+        (void)code;
 #endif
     }
     assert(rc == 0);
@@ -889,15 +894,15 @@ static PyObject *
 _hmac_HMAC_update_impl(HMACObject *self, PyObject *msgobj)
 /*[clinic end generated code: output=962134ada5e55985 input=7c0ea830efb03367]*/
 {
-    int rc = 0;
+    hacl_errno_t code;
     Py_buffer msg;
     GET_BUFFER_VIEW_OR_ERROUT(msgobj, &msg);
     HASHLIB_EXTERNAL_INSTRUCTIONS_LOCKED(
         self, msg.len,
-        rc = _hacl_hmac_state_update(self->state, msg.buf, msg.len)
+        code = _hacl_hmac_state_update(self->state, msg.buf, msg.len)
     );
     PyBuffer_Release(&msg);
-    return rc < 0 ? NULL : Py_None;
+    return _hacl_convert_errno(code) < 0 ? NULL : Py_None;
 }
 
 /*