switched to GMPY2; all tests passing and runs much faster

.github/workflows/tests.yml

@@ -18,6 +18,9 @@ jobs:
         python-version: ${{ matrix.python-version }}
     - name: Install dependencies
       run: |
+        sudo apt-get install libgmp-dev
+        sudo apt-get install libmpfr-dev
+        sudo apt-get install libmpc-dev
         python -m pip install --upgrade pip
         pip install tox tox-gh-actions
     - name: Test with tox

Pipfile

@@ -7,6 +7,7 @@ verify_ssl = true
 
 [packages]
 hypothesis = "==5.6.0"
+gmpy2 = "==2.1.0b4"
 
 [requires]
 python_version = "3.8"

Pipfile.lock

@@ -1,7 +1,7 @@
 {
     "_meta": {
         "hash": {
-            "sha256": "65576cc3d2517493ef8d8276436e729527f0e69ad2fc76af4452f83795a7d93e"
+            "sha256": "3278ce4c3833fb46260e5c3c2ac015e2a9c3f54830509bc1b5867073d2fc5b0c"
         },
         "pipfile-spec": 6,
         "requires": {
@@ -23,6 +23,13 @@
             ],
             "version": "==19.3.0"
         },
+        "gmpy2": {
+            "hashes": [
+                "sha256:9564deb6dcc7045749c0c5d73b23855ef6220c60b4cc6ffa4b1e0b1b1ee95eaf"
+            ],
+            "index": "pypi",
+            "version": "==2.1.0b4"
+        },
         "hypothesis": {
             "hashes": [
                 "sha256:22fb60bd0c6eb7849121a7df263a91da23b4e8506d3ba9e92ac696d2720ac0f5",

mypy.ini

@@ -5,4 +5,4 @@ python_version = 3.8
 warn_return_any = True
 warn_unused_configs = True
 disallow_untyped_defs = True
-mypy_path = "src"
+mypy_path = "src:stubs"

setup.py

@@ -4,7 +4,6 @@ from __future__ import absolute_import
 from __future__ import print_function
 
 import io
-import re
 from glob import glob
 from os.path import basename
 from os.path import dirname
@@ -16,6 +15,7 @@ from setuptools import setup
 
 # Borrowed from here: https://blog.ionelmc.ro/2014/05/25/python-packaging/
 
+
 def read(*names, **kwargs):
     with io.open(
         join(dirname(__file__), *names),
@@ -76,6 +76,7 @@ setup(
         'hypothesis==5.6.0'
     ],
     install_requires=[
+        'gmpy2==2.1.0b4'
         # eg: 'aspectlib==1.1.1', 'six>=1.7',
     ],
     extras_require={

src/electionguard/GMPY_NOTES.md

@@ -0,0 +1,30 @@
+# Note on the use of GMPY2
+
+We're using the GMPY2 multiprecision numeric library to go faster than Python's builtin int type, even 
+though Python natively supports modular exponentiation. GMPY2 is *much* faster.
+
+## Useful hyperlinks:
+- [Top-level documentation](https://gmpy2.readthedocs.io/en/latest/index.html)
+- [Documentation on the `mpz` (multi-precision integer) type](https://gmpy2.readthedocs.io/en/latest/mpz.html)
+
+## General usage
+There's a constructor, `mpz()`, to go from a regular integer or string to an mpz. After
+that, you can use mpz's and regular Python int's interchangeably. Equality and
+everything appears to just work. GMPY2 defines a `powmod` function that looks to be equivalent
+to Python's three-argument `pow` function, so we'll use `powmod` in case it's faster.
+
+## Multithreading
+GMPY2.0.8 is the most recent "stable" version.
+GMPY2.1b4 is the most recent "beta" 
+([release notes](https://gmpy2.readthedocs.io/en/latest/intro.html#enhancements-in-gmpy2-2-1),
+[GitHub release tag](https://github.com/aleaxit/gmpy/releases/tag/gmpy2-2.1.0b4))
+claims to have "thread-safe contexts", implying the absence of this from earlier releases.
+So far, all our tests with seem to pass with 2.1b4, but we can downgrade to 2.0.8 if necessary.
+
+Why go bleeding edge? Multithreading support seems important if/when we want to run EG in parallel,
+which will probably be necessary and useful when computing with large ballot manifests.
+
+## Python type hints
+GMPY2 has no type hints, nor are any present in [Typeshed](https://github.com/python/typeshed).
+This makes `mypy` unhappy with our code in `group.py`. If you look in `stubs/gmpy2.pyi`, you'll see
+just enough stubs to get `group.py` to compile without warnings.

src/electionguard/group.py

@@ -3,6 +3,7 @@
 # made about timing or other side-channels.
 
 from typing import Final, Union, NamedTuple
+from gmpy2 import mpz, powmod
 
 # Constants used by ElectionGuard
 Q: Final[int] = pow(2, 256) - 189
@@ -14,22 +15,22 @@ G_INV: Final[int] = pow(G, -1, P)
 
 class ElementModQ(NamedTuple):
     """An element of the smaller `mod q` space, i.e., in [0, Q), where Q is a 256-bit prime."""
-    elem: int
+    elem: mpz
 
 
-ZERO_MOD_Q: Final[ElementModQ] = ElementModQ(0)
-ONE_MOD_Q: Final[ElementModQ] = ElementModQ(1)
-TWO_MOD_Q: Final[ElementModQ] = ElementModQ(2)
+ZERO_MOD_Q: Final[ElementModQ] = ElementModQ(mpz(0))
+ONE_MOD_Q: Final[ElementModQ] = ElementModQ(mpz(1))
+TWO_MOD_Q: Final[ElementModQ] = ElementModQ(mpz(2))
 
 
 class ElementModP(NamedTuple):
     """An element of the larger `mod p` space, i.e., in [0, P), where P is a 4096-bit prime."""
-    elem: int
+    elem: mpz
 
 
-ZERO_MOD_P: Final[ElementModP] = ElementModP(0)
-ONE_MOD_P: Final[ElementModP] = ElementModP(1)
-TWO_MOD_P: Final[ElementModP] = ElementModP(2)
+ZERO_MOD_P: Final[ElementModP] = ElementModP(mpz(0))
+ONE_MOD_P: Final[ElementModP] = ElementModP(mpz(1))
+TWO_MOD_P: Final[ElementModP] = ElementModP(mpz(2))
 
 ElementModPOrQ = Union[ElementModP, ElementModQ]  # generally useful typedef
 
@@ -40,7 +41,7 @@ def int_to_q(i: int) -> ElementModQ:
     Raises an exception if it's out of bounds.
     """
     if 0 <= i < Q:
-        return ElementModQ(i)
+        return ElementModQ(mpz(i))
     else:
         raise Exception("given element doesn't fit in Q: " + str(i))
 
@@ -52,7 +53,7 @@ def int_to_q_unchecked(i: int) -> ElementModQ:
     element (i.e., outside of [0,Q)). Useful for tests.
     Don't use anywhere else.
     """
-    return ElementModQ(i)
+    return ElementModQ(mpz(i))
 
 
 def int_to_p(i: int) -> ElementModP:
@@ -61,7 +62,7 @@ def int_to_p(i: int) -> ElementModP:
     Raises an exception if it's out of bounds.
     """
     if 0 <= i < P:
-        return ElementModP(i)
+        return ElementModP(mpz(i))
     else:
         raise Exception("given element doesn't fit in P: " + str(i))
 
@@ -73,7 +74,7 @@ def int_to_p_unchecked(i: int) -> ElementModP:
     element (i.e., outside of [0,P)). Useful for tests.
     Don't use anywhere else.
     """
-    return ElementModP(i)
+    return ElementModP(mpz(i))
 
 
 def elem_to_int(a: ElementModPOrQ) -> int:
@@ -87,7 +88,7 @@ def add_q(*elems: ElementModQ) -> ElementModQ:
     """
     Adds together one or more elements in Q, returns the sum mod Q.
     """
-    t = 0
+    t = mpz(0)
     for e in elems:
         t = (t + e.elem) % Q
 
@@ -122,7 +123,7 @@ def mult_inv_p(e: ElementModPOrQ) -> ElementModP:
     """
     if e.elem == 0:
         raise Exception("No multiplicative inverse for zero")
-    return ElementModP(pow(e.elem, -1, P))
+    return ElementModP(powmod(e.elem, -1, P))
 
 
 def pow_p(b: ElementModPOrQ, e: ElementModPOrQ) -> ElementModP:
@@ -131,7 +132,7 @@ def pow_p(b: ElementModPOrQ, e: ElementModPOrQ) -> ElementModP:
     :param b: An element in [0,P).
     :param e: An element in [0,P).
     """
-    return ElementModP(pow(b.elem, e.elem, P))
+    return ElementModP(powmod(b.elem, e.elem, P))
 
 
 def mult_p(*elems: ElementModPOrQ) -> ElementModP:
@@ -139,10 +140,10 @@ def mult_p(*elems: ElementModPOrQ) -> ElementModP:
     Computes the product, mod p, of all elements.
     :param elems: Zero or more elements in [0,P).
     """
-    product = ONE_MOD_P
+    product = mpz(1)
     for x in elems:
-        product = ElementModP((product.elem * x.elem) % P)
-    return product
+        product = (product * x.elem) % P
+    return ElementModP(product)
 
 
 def g_pow_p(e: ElementModPOrQ) -> ElementModP:
@@ -150,7 +151,7 @@ def g_pow_p(e: ElementModPOrQ) -> ElementModP:
     Computes g^e mod p.
     :param e: An element in [0,P).
     """
-    return pow_p(ElementModP(G), e)
+    return pow_p(ElementModP(mpz(G)), e)
 
 
 def in_bounds_p(p: ElementModP) -> bool:
@@ -191,5 +192,5 @@ def valid_residue(x: ElementModP) -> bool:
     Returns true if all is good, false if something's wrong.
     """
     bounds = 0 <= x.elem < P
-    residue = pow_p(x, ElementModQ(Q)) == ONE_MOD_P
+    residue = pow_p(x, ElementModQ(mpz(Q))) == ONE_MOD_P
     return bounds and residue

stubs/gmpy2.pyi

@@ -0,0 +1,83 @@
+# This is just enough stubs for GMPY2 to allow ELectionGuard's use of it to typecheck.
+
+# This file started by running `stubgen -p gmpy2`, and then hacking the output manually
+# until all the warnings went away. As such, several things here are probably very wrong.
+# Still, it's useful to be able to get a clean bill of health from mypy.
+
+from typing import Union, Any, Tuple, Text, Optional
+
+
+class mpz(int):
+    def __new__(self, x: Union[Text, bytes, bytearray, int], base: int = ...) -> 'mpz': ...
+    def bit_clear(self, n: int) -> mpz: ...
+    def bit_flip(self, n: int) -> mpz: ...
+    def bit_length(self, *args: int, **kwargs: Any) -> int: ...
+    def bit_scan0(self, n: int = ...) -> int: ...
+    def bit_scan1(self, n: int = ...) -> int: ...
+    def bit_set(self, n: int) -> mpz: ...
+    def bit_test(self, n: int) -> bool: ...
+    def is_divisible(self, d: int) -> bool: ...
+    def is_even(self) -> bool: ...
+    def is_odd(self) -> bool: ...
+    def is_power(self) -> bool: ...
+    def is_prime(self) -> bool: ...
+    def is_square(self) -> bool: ...
+    def num_digits(self, base: int = ...) -> int: ...
+    def __abs__(self) -> mpz: ...
+    def __add__(self, other: int) -> mpz: ...
+    def __and__(self, other: int) -> mpz: ...
+    def __bool__(self) -> bool: ...
+    def __ceil__(self) -> mpz: ...
+    def __divmod__(self, other: int) -> Tuple[int, int]: ...
+    def __eq__(self, other: object) -> bool: ...
+    def __float__(self) -> mpz: ...   # maybe not mpz?
+    def __floor__(self) -> mpz: ...
+    def __floordiv__(self, other: int) -> mpz: ...
+    def __format__(self, *args: Any, **kwargs: Any) -> str: ...
+    def __ge__(self, other: int) -> bool: ...
+    def __getitem__(self, index: int) -> mpz: ...
+    def __gt__(self, other: int) -> bool: ...
+    def __hash__(self) -> int: ...
+    def __iadd__(self, other: int) -> mpz: ...
+    def __ifloordiv__(self, other: int) -> mpz: ...
+    def __ilshift__(self, other: int) -> mpz: ...
+    def __imod__(self, other: int) -> mpz: ...
+    def __imul__(self, other: int) -> mpz: ...
+    def __index__(self) -> int: ...
+    def __int__(self) -> int: ...
+    def __invert__(self) -> mpz: ...
+    def __ipow__(self, other: int, __modulo: Optional[int] = ...) -> mpz: ...
+    def __irshift__(self, other: int) -> mpz: ...
+    def __isub__(self, other: int) -> mpz: ...
+    def __le__(self, other: int) -> bool: ...
+    def __len__(self) -> int: ...
+    def __lshift__(self, other: int) -> mpz: ...
+    def __lt__(self, other: int) -> bool: ...
+    def __mod__(self, other: int) -> mpz: ...
+    def __mul__(self, other: int) -> mpz: ...
+    def __ne__(self, other: object) -> bool: ...
+    def __neg__(self) -> mpz: ...
+    def __or__(self, other: int) -> mpz: ...
+    def __pos__(self) -> bool: ...
+    def __radd__(self, other: int) -> mpz: ...
+    def __rand__(self, other: int) -> mpz: ...
+    def __rdivmod__(self, other: int) -> Tuple[int, int]: ...
+    def __rfloordiv__(self, other: int) -> mpz: ...
+    def __rlshift__(self, other: int) -> mpz: ...
+    def __rmod__(self, other: int) -> mpz: ...
+    def __rmul__(self, other: int) -> mpz: ...
+    def __ror__(self, other: int) -> mpz: ...
+    def __rpow__(self, other: int, __modulo: Optional[int] = ...) -> mpz: ...
+    def __rrshift__(self, other: int) -> mpz: ...
+    def __rshift__(self, other: int) -> mpz: ...
+    def __rsub__(self, other: int) -> mpz: ...
+    def __rtruediv__(self, other: float) -> float: ...
+    def __rxor__(self, other: int) -> mpz: ...
+    def __sizeof__(self) -> int: ...
+    def __sub__(self, other: int) -> mpz: ...
+    def __truediv__(self, other: float) -> float: ...
+    def __trunc__(self) -> mpz: ...
+    def __xor__(self, other: int) -> mpz: ...
+
+
+def powmod(a: int, e: int, p: int) -> mpz: ...

tests/test_group.py

@@ -75,23 +75,33 @@ class TestModularArithmetic(unittest.TestCase):
     @given(arb_element_mod_q())
     def test_in_bounds_q(self, q: ElementModQ):
         self.assertTrue(in_bounds_q(q))
-        self.assertFalse(in_bounds_q(int_to_q_unchecked(elem_to_int(q) + Q)))
-        self.assertFalse(in_bounds_q(int_to_q_unchecked(elem_to_int(q) - Q)))
+        too_big = elem_to_int(q) + Q
+        too_small = elem_to_int(q) - Q
+        self.assertFalse(in_bounds_q(int_to_q_unchecked(too_big)))
+        self.assertFalse(in_bounds_q(int_to_q_unchecked(too_small)))
+        self.assertRaises(Exception, int_to_q, too_big)
+        self.assertRaises(Exception, int_to_q, too_small)
 
     @given(arb_element_mod_p())
     def test_in_bounds_p(self, p: ElementModP):
         self.assertTrue(in_bounds_p(p))
-        self.assertFalse(in_bounds_p(int_to_p_unchecked(elem_to_int(p) + P)))
-        self.assertFalse(in_bounds_p(int_to_p_unchecked(elem_to_int(p) - P)))
+        too_big = elem_to_int(p) + P
+        too_small = elem_to_int(p) - P
+        self.assertFalse(in_bounds_p(int_to_p_unchecked(too_big)))
+        self.assertFalse(in_bounds_p(int_to_p_unchecked(too_small)))
+        self.assertRaises(Exception, int_to_p, too_big)
+        self.assertRaises(Exception, int_to_p, too_small)
 
     @given(arb_element_mod_q_no_zero())
     def test_in_bounds_q_no_zero(self, q: ElementModQ):
         self.assertTrue(in_bounds_q_no_zero(q))
+        self.assertFalse(in_bounds_q_no_zero(ZERO_MOD_Q))
         self.assertFalse(in_bounds_q_no_zero(int_to_q_unchecked(elem_to_int(q) + Q)))
         self.assertFalse(in_bounds_q_no_zero(int_to_q_unchecked(elem_to_int(q) - Q)))
 
     @given(arb_element_mod_p_no_zero())
     def test_in_bounds_p_no_zero(self, p: ElementModP):
         self.assertTrue(in_bounds_p_no_zero(p))
+        self.assertFalse(in_bounds_p_no_zero(ZERO_MOD_P))
         self.assertFalse(in_bounds_p_no_zero(int_to_p_unchecked(elem_to_int(p) + P)))
         self.assertFalse(in_bounds_p_no_zero(int_to_p_unchecked(elem_to_int(p) - P)))

tox.ini

@@ -31,6 +31,7 @@ usedevelop =
     cover: true
     nocov: false
 deps =
+    gmpy2 >= 2.1.0b4
     hypothesis
     pytest
     cover: pytest-cov
@@ -109,5 +110,6 @@ whitelist_externals =
     /bin/sh
 mypy_paths =
     src
+    stubs
 commands =
     mypy {posargs:{[testenv:mypy]mypy_paths}}