restore GIL_MINSIZE

Author: picnixz

Lib/test/support/hashlib_helper.py

@@ -1,5 +1,6 @@
 import functools
 import hashlib
+import importlib
 import unittest
 from test.support.import_helper import import_module
 
@@ -307,3 +308,22 @@ def sha3_384(self):
     @property
     def sha3_512(self):
         return self._find_constructor_in("_sha3","sha3_512")
+
+
+def find_gil_minsize(modules_names, default=2048):
+    """Get the largest GIL_MINSIZE value for the given cryptographic modules.
+
+    The valid module names are the following:
+
+    - _hashlib
+    - _md5, _sha1, _sha2, _sha3, _blake2
+    - _hmac
+    """
+    sizes = []
+    for module_name in modules_names:
+        try:
+            module = importlib.import_module(module_name)
+        except ImportError:
+            continue
+        sizes.append(getattr(module, '_GIL_MINSIZE', default))
+    return max(sizes, default=default)

Lib/test/test_hashlib.py

@@ -20,6 +20,7 @@
 import unittest
 from test import support
 from test.support import _4G, bigmemtest
+from test.support import hashlib_helper
 from test.support.import_helper import import_fresh_module
 from test.support import requires_resource
 from test.support import threading_helper
@@ -408,7 +409,7 @@ def test_large_update(self):
         aas = b'a' * 128
         bees = b'b' * 127
         cees = b'c' * 126
-        dees = b'd' * 2048
+        dees = b'd' * 2048 #  HASHLIB_GIL_MINSIZE
 
         for cons in self.hash_constructors:
             m1 = cons(usedforsecurity=False)
@@ -989,6 +990,40 @@ def test_case_shake256_vector(self):
         for msg, md in read_vectors('shake_256'):
             self.check('shake_256', msg, md, True)
 
+    def test_gil(self):
+        # Check things work fine with an input larger than the size required
+        # for multithreaded operation. Currently, all cryptographic modules
+        # have the same constant value (2048) but in the future it might not
+        # be the case.
+        mods = ['_md5', '_sha1', '_sha2', '_sha3', '_blake2', '_hashlib']
+        gil_minsize = hashlib_helper.find_gil_minsize(mods)
+        for cons in self.hash_constructors:
+            # constructors belong to one of the above modules
+            m = cons(usedforsecurity=False)
+            m.update(b'1')
+            m.update(b'#' * gil_minsize)
+            m.update(b'1')
+
+            m = cons(b'x' * gil_minsize, usedforsecurity=False)
+            m.update(b'1')
+
+    def test_sha256_gil(self):
+        gil_minsize = hashlib_helper.find_gil_minsize(['_sha2', '_hashlib'])
+        m = hashlib.sha256()
+        m.update(b'1')
+        m.update(b'#' * gil_minsize)
+        m.update(b'1')
+        self.assertEqual(
+            m.hexdigest(),
+            '1cfceca95989f51f658e3f3ffe7f1cd43726c9e088c13ee10b46f57cef135b94'
+        )
+
+        m = hashlib.sha256(b'1' + b'#' * gil_minsize + b'1')
+        self.assertEqual(
+            m.hexdigest(),
+            '1cfceca95989f51f658e3f3ffe7f1cd43726c9e088c13ee10b46f57cef135b94'
+        )
+
     @threading_helper.reap_threads
     @threading_helper.requires_working_threading()
     def test_threaded_hashing(self):

Lib/test/test_hmac.py

@@ -1133,6 +1133,11 @@ def HMAC(self, key, msg=None):
         """Create a HMAC object."""
         raise NotImplementedError
 
+    @property
+    def gil_minsize(self):
+        """Get the maximal input length for the GIL to be held."""
+        raise NotImplementedError
+
     def check_update(self, key, chunks):
         chunks = list(chunks)
         msg = b''.join(chunks)
@@ -1150,6 +1155,13 @@ def test_update(self):
         with self.subTest(key=key, msg=msg):
             self.check_update(key, [msg])
 
+    def test_update_large(self):
+        gil_minsize = self.gil_minsize
+        key = random.randbytes(16)
+        top = random.randbytes(gil_minsize + 1)
+        bot = random.randbytes(gil_minsize + 1)
+        self.check_update(key, [top, bot])
+
     def test_update_exceptions(self):
         h = self.HMAC(b"key")
         for msg in ['invalid msg', 123, (), []]:
@@ -1163,13 +1175,21 @@ class PyUpdateTestCase(PyModuleMixin, UpdateTestCaseMixin, unittest.TestCase):
     def HMAC(self, key, msg=None):
         return self.hmac.HMAC(key, msg, digestmod='sha256')
 
+    @property
+    def gil_minsize(self):
+        return sha2._GIL_MINSIZE
+
 
 @hashlib_helper.requires_openssl_hashdigest('sha256')
 class OpenSSLUpdateTestCase(UpdateTestCaseMixin, unittest.TestCase):
 
     def HMAC(self, key, msg=None):
         return _hashlib.hmac_new(key, msg, digestmod='sha256')
 
+    @property
+    def gil_minsize(self):
+        return _hashlib._GIL_MINSIZE
+
 
 class BuiltinUpdateTestCase(BuiltinModuleMixin,
                             UpdateTestCaseMixin, unittest.TestCase):
@@ -1179,6 +1199,10 @@ def HMAC(self, key, msg=None):
         # are still built, making it possible to use SHA-2 hashes.
         return self.hmac.new(key, msg, digestmod='sha256')
 
+    @property
+    def gil_minsize(self):
+        return self.hmac._GIL_MINSIZE
+
 
 class CopyBaseTestCase:
 

Modules/_hashopenssl.c

@@ -2458,6 +2458,17 @@ hashlib_exception(PyObject *module)
     return 0;
 }
 
+static int
+hashlib_constants(PyObject *module)
+{
+    if (PyModule_AddIntConstant(module, "_GIL_MINSIZE",
+                                HASHLIB_GIL_MINSIZE) < 0)
+    {
+        return -1;
+    }
+    return 0;
+}
+
 static PyModuleDef_Slot hashlib_slots[] = {
     {Py_mod_exec, hashlib_init_hashtable},
     {Py_mod_exec, hashlib_init_HASH_type},
@@ -2466,6 +2477,7 @@ static PyModuleDef_Slot hashlib_slots[] = {
     {Py_mod_exec, hashlib_md_meth_names},
     {Py_mod_exec, hashlib_init_constructors},
     {Py_mod_exec, hashlib_exception},
+    {Py_mod_exec, hashlib_constants},
     {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED},
     {Py_mod_gil, Py_MOD_GIL_NOT_USED},
     {0, NULL}

Modules/blake2module.c

@@ -227,6 +227,8 @@ blake2_exec(PyObject *m)
         }                                                   \
     } while (0)
 
+    ADD_INT_CONST("_GIL_MINSIZE", HASHLIB_GIL_MINSIZE);
+
     st->blake2b_type = (PyTypeObject *)PyType_FromModuleAndSpec(
         m, &blake2b_type_spec, NULL);
 

Modules/hmacmodule.c

@@ -1534,6 +1534,20 @@ hmacmodule_init_strings(hmacmodule_state *state)
     return 0;
 }
 
+static int
+hmacmodule_init_globals(PyObject *module, hmacmodule_state *state)
+{
+#define ADD_INT_CONST(NAME, VALUE)                                  \
+    do {                                                            \
+        if (PyModule_AddIntConstant(module, (NAME), (VALUE)) < 0) { \
+            return -1;                                              \
+        }                                                           \
+    } while (0)
+    ADD_INT_CONST("_GIL_MINSIZE", HASHLIB_GIL_MINSIZE);
+#undef ADD_INT_CONST
+    return 0;
+}
+
 static void
 hmacmodule_init_cpu_features(hmacmodule_state *state)
 {
@@ -1624,6 +1638,9 @@ hmacmodule_exec(PyObject *module)
     if (hmacmodule_init_strings(state) < 0) {
         return -1;
     }
+    if (hmacmodule_init_globals(module, state) < 0) {
+        return -1;
+    }
     hmacmodule_init_cpu_features(state);
     return 0;
 }

Modules/md5module.c

@@ -352,6 +352,9 @@ md5_exec(PyObject *m)
     if (PyModule_AddObjectRef(m, "MD5Type", (PyObject *)st->md5_type) < 0) {
         return -1;
     }
+    if (PyModule_AddIntConstant(m, "_GIL_MINSIZE", HASHLIB_GIL_MINSIZE) < 0) {
+        return -1;
+    }
 
     return 0;
 }

Modules/sha1module.c

@@ -355,6 +355,12 @@ _sha1_exec(PyObject *module)
     {
         return -1;
     }
+    if (PyModule_AddIntConstant(module,
+                                "_GIL_MINSIZE",
+                                HASHLIB_GIL_MINSIZE) < 0)
+    {
+        return -1;
+    }
 
     return 0;
 }

Modules/sha2module.c

@@ -877,6 +877,13 @@ static int sha2_exec(PyObject *module)
         return -1;
     }
 
+    if (PyModule_AddIntConstant(module,
+                                "_GIL_MINSIZE",
+                                HASHLIB_GIL_MINSIZE) < 0)
+    {
+        return -1;
+    }
+
     return 0;
 }
 

Modules/sha3module.c

@@ -630,6 +630,9 @@ _sha3_exec(PyObject *m)
     if (PyModule_AddStringConstant(m, "implementation", "HACL") < 0) {
         return -1;
     }
+    if (PyModule_AddIntConstant(m, "_GIL_MINSIZE", HASHLIB_GIL_MINSIZE) < 0) {
+        return -1;
+    }
 
     return 0;
 }