hashlib: Refactor, split, and optimise.

This splits out each algorithm into its own extension package, so that only
the necessary algorithms can be installed.

This allows for a significant reduction in RAM and flash. i.e. previously
installing hashlib meant that all algorithms were imported.

Additionally ensures that any built-in hash algorithms (from uhashlib) are
still exposed (e.g. `md5`), and retains the existing behavior to use the
built-in preferentially.

Also includes a refactoring of the algorithms to reduce code size and
reduce the number of allocations they do as well as using bytearrays in
place of list-of-int where possible.

Add more comprehensive tests (using unittest).

This work was funded through GitHub Sponsors.

Signed-off-by: Jim Mussared <[email protected]>

Author: jimmo

python-stdlib/hashlib-core/hashlib/__init__.py

@@ -0,0 +1,29 @@
+# Use built-in algorithms preferentially (on many ports this is just sha256).
+try:
+    from uhashlib import *
+except ImportError:
+    pass
+
+
+# Add missing algorithms based on installed extensions.
+def _init():
+    for algo in ("sha224", "sha256", "sha384", "sha512"):
+        if algo not in globals():
+            try:
+                # from ._{algo} import {algo}
+                c = __import__("_" + algo, None, None, (), 1)
+                globals()[algo] = getattr(c, algo)
+            except ImportError:
+                pass
+
+
+_init()
+del _init
+
+
+def new(algo, data=b""):
+    try:
+        c = globals()[algo]
+        return c(data)
+    except KeyError:
+        raise ValueError(algo)

python-stdlib/hashlib-core/hashlib/_sha.py

@@ -0,0 +1,42 @@
+# MIT license; Copyright (c) 2023 Jim Mussared
+# Originally ported from CPython by Paul Sokolovsky
+
+
+# Base class for SHA implementations, which must provide:
+#   .digestsize & .digest_size
+#   .block_size
+#   ._iv
+#   ._update
+#   ._final
+class sha:
+    def __init__(self, s=None):
+        self._digest = self._iv[:]
+        self._count_lo = 0
+        self._count_hi = 0
+        self._data = bytearray(self.block_size)
+        self._local = 0
+        self._digestsize = self.digest_size
+        if s:
+            self.update(s)
+
+    def update(self, s):
+        if isinstance(s, str):
+            s = s.encode("ascii")
+        else:
+            s = bytes(s)
+        self._update(s)
+
+    def digest(self):
+        return self.copy()._final()[: self._digestsize]
+
+    def hexdigest(self):
+        return "".join(["%.2x" % i for i in self.digest()])
+
+    def copy(self):
+        new = type(self)()
+        new._digest = self._digest[:]
+        new._count_lo = self._count_lo
+        new._count_hi = self._count_hi
+        new._data = self._data[:]
+        new._local = self._local
+        return new

python-stdlib/hashlib-core/manifest.py

@@ -0,0 +1,3 @@
+metadata(version="1.0")
+
+package("hashlib")

python-stdlib/hashlib-sha224/hashlib/_sha224.py

@@ -0,0 +1,18 @@
+# MIT license; Copyright (c) 2023 Jim Mussared
+# Originally ported from CPython by Paul Sokolovsky
+
+from ._sha256 import sha256
+
+
+class sha224(sha256):
+    digest_size = digestsize = 28
+    _iv = [
+        0xC1059ED8,
+        0x367CD507,
+        0x3070DD17,
+        0xF70E5939,
+        0xFFC00B31,
+        0x68581511,
+        0x64F98FA7,
+        0xBEFA4FA4,
+    ]

python-stdlib/hashlib-sha224/manifest.py

@@ -0,0 +1,4 @@
+metadata(version="1.0", description="Adds the SHA224 hash algorithm to hashlib.")
+
+require("hashlib-sha256")
+package("hashlib")

python-stdlib/hashlib-sha256/hashlib/_sha256.py

@@ -0,0 +1,194 @@
+# MIT license; Copyright (c) 2023 Jim Mussared
+# Originally ported from CPython by Paul Sokolovsky
+
+from ._sha import sha
+
+_SHA_BLOCKSIZE = const(64)
+
+
+ROR = lambda x, y: (((x & 0xFFFFFFFF) >> (y & 31)) | (x << (32 - (y & 31)))) & 0xFFFFFFFF
+Ch = lambda x, y, z: (z ^ (x & (y ^ z)))
+Maj = lambda x, y, z: (((x | y) & z) | (x & y))
+S = lambda x, n: ROR(x, n)
+R = lambda x, n: (x & 0xFFFFFFFF) >> n
+Sigma0 = lambda x: (S(x, 2) ^ S(x, 13) ^ S(x, 22))
+Sigma1 = lambda x: (S(x, 6) ^ S(x, 11) ^ S(x, 25))
+Gamma0 = lambda x: (S(x, 7) ^ S(x, 18) ^ R(x, 3))
+Gamma1 = lambda x: (S(x, 17) ^ S(x, 19) ^ R(x, 10))
+
+
+class sha256(sha):
+    digest_size = digestsize = 32
+    block_size = _SHA_BLOCKSIZE
+    _iv = [
+        0x6A09E667,
+        0xBB67AE85,
+        0x3C6EF372,
+        0xA54FF53A,
+        0x510E527F,
+        0x9B05688C,
+        0x1F83D9AB,
+        0x5BE0CD19,
+    ]
+
+    def _transform(self):
+        W = []
+
+        d = self._data
+        for i in range(0, 16):
+            W.append((d[4 * i] << 24) + (d[4 * i + 1] << 16) + (d[4 * i + 2] << 8) + d[4 * i + 3])
+
+        for i in range(16, 64):
+            W.append((Gamma1(W[i - 2]) + W[i - 7] + Gamma0(W[i - 15]) + W[i - 16]) & 0xFFFFFFFF)
+
+        ss = self._digest[:]
+
+        def RND(a, b, c, d, e, f, g, h, i, ki):
+            t0 = h + Sigma1(e) + Ch(e, f, g) + ki + W[i]
+            t1 = Sigma0(a) + Maj(a, b, c)
+            d += t0
+            h = t0 + t1
+            return d & 0xFFFFFFFF, h & 0xFFFFFFFF
+
+        ss[3], ss[7] = RND(ss[0], ss[1], ss[2], ss[3], ss[4], ss[5], ss[6], ss[7], 0, 0x428A2F98)
+        ss[2], ss[6] = RND(ss[7], ss[0], ss[1], ss[2], ss[3], ss[4], ss[5], ss[6], 1, 0x71374491)
+        ss[1], ss[5] = RND(ss[6], ss[7], ss[0], ss[1], ss[2], ss[3], ss[4], ss[5], 2, 0xB5C0FBCF)
+        ss[0], ss[4] = RND(ss[5], ss[6], ss[7], ss[0], ss[1], ss[2], ss[3], ss[4], 3, 0xE9B5DBA5)
+        ss[7], ss[3] = RND(ss[4], ss[5], ss[6], ss[7], ss[0], ss[1], ss[2], ss[3], 4, 0x3956C25B)
+        ss[6], ss[2] = RND(ss[3], ss[4], ss[5], ss[6], ss[7], ss[0], ss[1], ss[2], 5, 0x59F111F1)
+        ss[5], ss[1] = RND(ss[2], ss[3], ss[4], ss[5], ss[6], ss[7], ss[0], ss[1], 6, 0x923F82A4)
+        ss[4], ss[0] = RND(ss[1], ss[2], ss[3], ss[4], ss[5], ss[6], ss[7], ss[0], 7, 0xAB1C5ED5)
+        ss[3], ss[7] = RND(ss[0], ss[1], ss[2], ss[3], ss[4], ss[5], ss[6], ss[7], 8, 0xD807AA98)
+        ss[2], ss[6] = RND(ss[7], ss[0], ss[1], ss[2], ss[3], ss[4], ss[5], ss[6], 9, 0x12835B01)
+        ss[1], ss[5] = RND(ss[6], ss[7], ss[0], ss[1], ss[2], ss[3], ss[4], ss[5], 10, 0x243185BE)
+        ss[0], ss[4] = RND(ss[5], ss[6], ss[7], ss[0], ss[1], ss[2], ss[3], ss[4], 11, 0x550C7DC3)
+        ss[7], ss[3] = RND(ss[4], ss[5], ss[6], ss[7], ss[0], ss[1], ss[2], ss[3], 12, 0x72BE5D74)
+        ss[6], ss[2] = RND(ss[3], ss[4], ss[5], ss[6], ss[7], ss[0], ss[1], ss[2], 13, 0x80DEB1FE)
+        ss[5], ss[1] = RND(ss[2], ss[3], ss[4], ss[5], ss[6], ss[7], ss[0], ss[1], 14, 0x9BDC06A7)
+        ss[4], ss[0] = RND(ss[1], ss[2], ss[3], ss[4], ss[5], ss[6], ss[7], ss[0], 15, 0xC19BF174)
+        ss[3], ss[7] = RND(ss[0], ss[1], ss[2], ss[3], ss[4], ss[5], ss[6], ss[7], 16, 0xE49B69C1)
+        ss[2], ss[6] = RND(ss[7], ss[0], ss[1], ss[2], ss[3], ss[4], ss[5], ss[6], 17, 0xEFBE4786)
+        ss[1], ss[5] = RND(ss[6], ss[7], ss[0], ss[1], ss[2], ss[3], ss[4], ss[5], 18, 0x0FC19DC6)
+        ss[0], ss[4] = RND(ss[5], ss[6], ss[7], ss[0], ss[1], ss[2], ss[3], ss[4], 19, 0x240CA1CC)
+        ss[7], ss[3] = RND(ss[4], ss[5], ss[6], ss[7], ss[0], ss[1], ss[2], ss[3], 20, 0x2DE92C6F)
+        ss[6], ss[2] = RND(ss[3], ss[4], ss[5], ss[6], ss[7], ss[0], ss[1], ss[2], 21, 0x4A7484AA)
+        ss[5], ss[1] = RND(ss[2], ss[3], ss[4], ss[5], ss[6], ss[7], ss[0], ss[1], 22, 0x5CB0A9DC)
+        ss[4], ss[0] = RND(ss[1], ss[2], ss[3], ss[4], ss[5], ss[6], ss[7], ss[0], 23, 0x76F988DA)
+        ss[3], ss[7] = RND(ss[0], ss[1], ss[2], ss[3], ss[4], ss[5], ss[6], ss[7], 24, 0x983E5152)
+        ss[2], ss[6] = RND(ss[7], ss[0], ss[1], ss[2], ss[3], ss[4], ss[5], ss[6], 25, 0xA831C66D)
+        ss[1], ss[5] = RND(ss[6], ss[7], ss[0], ss[1], ss[2], ss[3], ss[4], ss[5], 26, 0xB00327C8)
+        ss[0], ss[4] = RND(ss[5], ss[6], ss[7], ss[0], ss[1], ss[2], ss[3], ss[4], 27, 0xBF597FC7)
+        ss[7], ss[3] = RND(ss[4], ss[5], ss[6], ss[7], ss[0], ss[1], ss[2], ss[3], 28, 0xC6E00BF3)
+        ss[6], ss[2] = RND(ss[3], ss[4], ss[5], ss[6], ss[7], ss[0], ss[1], ss[2], 29, 0xD5A79147)
+        ss[5], ss[1] = RND(ss[2], ss[3], ss[4], ss[5], ss[6], ss[7], ss[0], ss[1], 30, 0x06CA6351)
+        ss[4], ss[0] = RND(ss[1], ss[2], ss[3], ss[4], ss[5], ss[6], ss[7], ss[0], 31, 0x14292967)
+        ss[3], ss[7] = RND(ss[0], ss[1], ss[2], ss[3], ss[4], ss[5], ss[6], ss[7], 32, 0x27B70A85)
+        ss[2], ss[6] = RND(ss[7], ss[0], ss[1], ss[2], ss[3], ss[4], ss[5], ss[6], 33, 0x2E1B2138)
+        ss[1], ss[5] = RND(ss[6], ss[7], ss[0], ss[1], ss[2], ss[3], ss[4], ss[5], 34, 0x4D2C6DFC)
+        ss[0], ss[4] = RND(ss[5], ss[6], ss[7], ss[0], ss[1], ss[2], ss[3], ss[4], 35, 0x53380D13)
+        ss[7], ss[3] = RND(ss[4], ss[5], ss[6], ss[7], ss[0], ss[1], ss[2], ss[3], 36, 0x650A7354)
+        ss[6], ss[2] = RND(ss[3], ss[4], ss[5], ss[6], ss[7], ss[0], ss[1], ss[2], 37, 0x766A0ABB)
+        ss[5], ss[1] = RND(ss[2], ss[3], ss[4], ss[5], ss[6], ss[7], ss[0], ss[1], 38, 0x81C2C92E)
+        ss[4], ss[0] = RND(ss[1], ss[2], ss[3], ss[4], ss[5], ss[6], ss[7], ss[0], 39, 0x92722C85)
+        ss[3], ss[7] = RND(ss[0], ss[1], ss[2], ss[3], ss[4], ss[5], ss[6], ss[7], 40, 0xA2BFE8A1)
+        ss[2], ss[6] = RND(ss[7], ss[0], ss[1], ss[2], ss[3], ss[4], ss[5], ss[6], 41, 0xA81A664B)
+        ss[1], ss[5] = RND(ss[6], ss[7], ss[0], ss[1], ss[2], ss[3], ss[4], ss[5], 42, 0xC24B8B70)
+        ss[0], ss[4] = RND(ss[5], ss[6], ss[7], ss[0], ss[1], ss[2], ss[3], ss[4], 43, 0xC76C51A3)
+        ss[7], ss[3] = RND(ss[4], ss[5], ss[6], ss[7], ss[0], ss[1], ss[2], ss[3], 44, 0xD192E819)
+        ss[6], ss[2] = RND(ss[3], ss[4], ss[5], ss[6], ss[7], ss[0], ss[1], ss[2], 45, 0xD6990624)
+        ss[5], ss[1] = RND(ss[2], ss[3], ss[4], ss[5], ss[6], ss[7], ss[0], ss[1], 46, 0xF40E3585)
+        ss[4], ss[0] = RND(ss[1], ss[2], ss[3], ss[4], ss[5], ss[6], ss[7], ss[0], 47, 0x106AA070)
+        ss[3], ss[7] = RND(ss[0], ss[1], ss[2], ss[3], ss[4], ss[5], ss[6], ss[7], 48, 0x19A4C116)
+        ss[2], ss[6] = RND(ss[7], ss[0], ss[1], ss[2], ss[3], ss[4], ss[5], ss[6], 49, 0x1E376C08)
+        ss[1], ss[5] = RND(ss[6], ss[7], ss[0], ss[1], ss[2], ss[3], ss[4], ss[5], 50, 0x2748774C)
+        ss[0], ss[4] = RND(ss[5], ss[6], ss[7], ss[0], ss[1], ss[2], ss[3], ss[4], 51, 0x34B0BCB5)
+        ss[7], ss[3] = RND(ss[4], ss[5], ss[6], ss[7], ss[0], ss[1], ss[2], ss[3], 52, 0x391C0CB3)
+        ss[6], ss[2] = RND(ss[3], ss[4], ss[5], ss[6], ss[7], ss[0], ss[1], ss[2], 53, 0x4ED8AA4A)
+        ss[5], ss[1] = RND(ss[2], ss[3], ss[4], ss[5], ss[6], ss[7], ss[0], ss[1], 54, 0x5B9CCA4F)
+        ss[4], ss[0] = RND(ss[1], ss[2], ss[3], ss[4], ss[5], ss[6], ss[7], ss[0], 55, 0x682E6FF3)
+        ss[3], ss[7] = RND(ss[0], ss[1], ss[2], ss[3], ss[4], ss[5], ss[6], ss[7], 56, 0x748F82EE)
+        ss[2], ss[6] = RND(ss[7], ss[0], ss[1], ss[2], ss[3], ss[4], ss[5], ss[6], 57, 0x78A5636F)
+        ss[1], ss[5] = RND(ss[6], ss[7], ss[0], ss[1], ss[2], ss[3], ss[4], ss[5], 58, 0x84C87814)
+        ss[0], ss[4] = RND(ss[5], ss[6], ss[7], ss[0], ss[1], ss[2], ss[3], ss[4], 59, 0x8CC70208)
+        ss[7], ss[3] = RND(ss[4], ss[5], ss[6], ss[7], ss[0], ss[1], ss[2], ss[3], 60, 0x90BEFFFA)
+        ss[6], ss[2] = RND(ss[3], ss[4], ss[5], ss[6], ss[7], ss[0], ss[1], ss[2], 61, 0xA4506CEB)
+        ss[5], ss[1] = RND(ss[2], ss[3], ss[4], ss[5], ss[6], ss[7], ss[0], ss[1], 62, 0xBEF9A3F7)
+        ss[4], ss[0] = RND(ss[1], ss[2], ss[3], ss[4], ss[5], ss[6], ss[7], ss[0], 63, 0xC67178F2)
+
+        for i in range(len(self._digest)):
+            self._digest[i] = (self._digest[i] + ss[i]) & 0xFFFFFFFF
+
+    def _update(self, buffer):
+        if isinstance(buffer, str):
+            raise TypeError("Unicode strings must be encoded before hashing")
+        count = len(buffer)
+        buffer_idx = 0
+        clo = (self._count_lo + (count << 3)) & 0xFFFFFFFF
+        if clo < self._count_lo:
+            self._count_hi += 1
+        self._count_lo = clo
+
+        self._count_hi += count >> 29
+
+        if self._local:
+            i = _SHA_BLOCKSIZE - self._local
+            if i > count:
+                i = count
+
+            # copy buffer
+            for x in enumerate(buffer[buffer_idx : buffer_idx + i]):
+                self._data[self._local + x[0]] = x[1]
+
+            count -= i
+            buffer_idx += i
+
+            self._local += i
+            if self._local == _SHA_BLOCKSIZE:
+                self._transform()
+                self._local = 0
+            else:
+                return
+
+        while count >= _SHA_BLOCKSIZE:
+            # copy buffer
+            self._data = bytearray(buffer[buffer_idx : buffer_idx + _SHA_BLOCKSIZE])
+            count -= _SHA_BLOCKSIZE
+            buffer_idx += _SHA_BLOCKSIZE
+            self._transform()
+
+        # copy buffer
+        pos = self._local
+        self._data[pos : pos + count] = buffer[buffer_idx : buffer_idx + count]
+        self._local = count
+
+    def _final(self):
+        lo_bit_count = self._count_lo
+        hi_bit_count = self._count_hi
+        count = (lo_bit_count >> 3) & 0x3F
+        self._data[count] = 0x80
+        count += 1
+        if count > _SHA_BLOCKSIZE - 8:
+            # zero the bytes in data after the count
+            self._data = self._data[:count] + bytes(_SHA_BLOCKSIZE - count)
+            self._transform()
+            # zero bytes in data
+            self._data = bytearray(_SHA_BLOCKSIZE)
+        else:
+            self._data = self._data[:count] + bytes(_SHA_BLOCKSIZE - count)
+
+        self._data[56] = (hi_bit_count >> 24) & 0xFF
+        self._data[57] = (hi_bit_count >> 16) & 0xFF
+        self._data[58] = (hi_bit_count >> 8) & 0xFF
+        self._data[59] = (hi_bit_count >> 0) & 0xFF
+        self._data[60] = (lo_bit_count >> 24) & 0xFF
+        self._data[61] = (lo_bit_count >> 16) & 0xFF
+        self._data[62] = (lo_bit_count >> 8) & 0xFF
+        self._data[63] = (lo_bit_count >> 0) & 0xFF
+
+        self._transform()
+
+        dig = bytearray()
+        for i in self._digest:
+            for j in range(4):
+                dig.append((i >> ((3 - j) * 8)) & 0xFF)
+        return dig

python-stdlib/hashlib-sha256/manifest.py

@@ -0,0 +1,4 @@
+metadata(version="1.0", description="Adds the SHA256 hash algorithm to hashlib.")
+
+require("hashlib-core")
+package("hashlib")

python-stdlib/hashlib-sha384/hashlib/_sha384.py

@@ -0,0 +1,18 @@
+# MIT license; Copyright (c) 2023 Jim Mussared
+# Originally ported from CPython by Paul Sokolovsky
+
+from ._sha512 import sha512
+
+
+class sha384(sha512):
+    digest_size = digestsize = 48
+    _iv = [
+        0xCBBB9D5DC1059ED8,
+        0x629A292A367CD507,
+        0x9159015A3070DD17,
+        0x152FECD8F70E5939,
+        0x67332667FFC00B31,
+        0x8EB44A8768581511,
+        0xDB0C2E0D64F98FA7,
+        0x47B5481DBEFA4FA4,
+    ]

python-stdlib/hashlib-sha384/manifest.py

@@ -0,0 +1,4 @@
+metadata(version="1.0", description="Adds the SHA384 hash algorithm to hashlib.")
+
+require("hashlib-sha512")
+package("hashlib")