make roots data read-only, shareable between processes, and smaller on 64-bit platforms (#38)

* don't clang-format generated roots

The formatted arrays are significantly less readable.

* eliminate the need for RootsInv data

Since RootsInv is essentially Roots stored in reverse order, we don't
need to store an entirely separate array for the inverse roots.  We can
simply iterate over Roots in reverse order to produce the inverse roots.

* make roots data read-only and shareable between processes

The roots of unity are currently stored as:

static const char *Roots[] = { ... };

which is inefficient for two reasons:

1. `Roots` is a writable array, even though we only ever read from it.
2. `Roots` contain pointers, which require run-time relocations.
   These run-time relocations can be at least as large as the entries
   in the array.

The upshot is that `Roots` requires more space than necessary and cannot
be shared between processes.

To fix both of these problems, let's change the storage format to one
long character array, where individual roots are formatted to all be the
same width.  This format enables efficient access to individual roots
and is pointer-free, so no run-time relocations are required.

* Script works with Python3, add one sanity check

prio/config.c

@@ -24,13 +24,28 @@
 //    p = (2^k)q + 1.
 // The roots are integers such that r^{2^k} = 1 mod p.
 static SECStatus
-initialize_roots(MPArray arr, const char* values[])
+initialize_roots(MPArray arr, const char values[], bool inverted)
 {
   // TODO: Read in only the number of roots of unity we need.
   // Right now we read in all 4096 roots whether or not we use
   // them all.
-  for (int i = 0; i < arr->len; i++) {
-    MP_CHECK(mp_read_radix(&arr->data[i], values[i], 16));
+  MP_CHECK(mp_read_radix(&arr->data[0], &values[0], 16));
+  unsigned int len = arr->len;
+  unsigned int n_chars = len * RootWidth;
+
+  if (n_chars != sizeof(Roots)) {
+    return SECFailure;
+  }
+
+  if (inverted) {
+    for (unsigned int i = n_chars - RootWidth, j = 1; i > 0;
+         i -= RootWidth, j++) {
+      MP_CHECK(mp_read_radix(&arr->data[j], &values[i], 16));
+    }
+  } else {
+    for (unsigned int i = RootWidth, j = 1; i < n_chars; i += RootWidth, j++) {
+      MP_CHECK(mp_read_radix(&arr->data[j], &values[i], 16));
+    }
   }
 
   return SECSuccess;
@@ -74,8 +89,8 @@ PrioConfig_new(int n_fields, PublicKey server_a, PublicKey server_b,
 
   P_CHECKA(cfg->roots = MPArray_new(cfg->n_roots));
   P_CHECKA(cfg->rootsInv = MPArray_new(cfg->n_roots));
-  MP_CHECKC(initialize_roots(cfg->roots, Roots));
-  MP_CHECKC(initialize_roots(cfg->rootsInv, RootsInv));
+  MP_CHECKC(initialize_roots(cfg->roots, Roots, /*inverted=*/false));
+  MP_CHECKC(initialize_roots(cfg->rootsInv, Roots, /*inverted=*/true));
 
 cleanup:
   if (rv != SECSuccess) {

prio/gen_params.py

@@ -1,90 +0,0 @@
-"""
-/*
- * Copyright (c) 2018, Henry Corrigan-Gibbs
- * 
- * This Source Code Form is subject to the terms of the Mozilla Public
- * License, v. 2.0. If a copy of the MPL was not distributed with this
- * file, You can obtain one at http://mozilla.org/MPL/2.0/. 
- */
-"""
-
-
-# Has a subgroup of order 2^19
-modulus = int('0x8000000000000000080001', 16)
-# Generates the subgroup of order 2^19
-gen19 = int('0x2597c14f48d5b65ed8dcca', 16)
-
-# We want a generator of order 2^12, so compute
-#   gen19^(2^7) = gen19^128   (mod p)
-gen12 = gen19
-for i in range(7):
-    gen12 *= gen12
-    gen12 %= modulus
-    #print gen12
-
-# Sanity check
-rootsL = [1] * 2**12
-rootsInvL = [1] * 2**12
-for i in range(1, 2**12):
-    rootsL[i] = (rootsL[i-1] * gen12) % modulus
-
-assert ((rootsL[2**12 - 1] * gen12) % modulus) == 1
-
-gen12inv = rootsL[2**12 - 1]
-for i in range(1, 2**12):
-    rootsInvL[i] = (rootsInvL[i-1] * gen12inv) % modulus
-    
-    assert rootsInvL[i] != 1 
-assert ((rootsInvL[2**12 - 1] * gen12inv) % modulus) == 1
-
-rootsL = map(lambda x: '   "%x"' % x, rootsL)
-rootsInvL = map(lambda x: '   "%x"' % x, rootsInvL)
-
-roots = ",\n".join(rootsL)
-rootsInv = ",\n".join(rootsInvL)
-
-output = """
-/*
- * Copyright (c) 2018, Henry Corrigan-Gibbs
- * 
- * This Source Code Form is subject to the terms of the Mozilla Public
- * License, v. 2.0. If a copy of the MPL was not distributed with this
- * file, You can obtain one at http://mozilla.org/MPL/2.0/. 
- */
-
-/****
- * NOTE: This file was auto-generated from gen_params.py.
- * Do not edit this file. Instead, edit the script.
- */
-
-#ifndef __PARAMS_H__
-#define __PARAMS_H__
-
-// A prime modulus p.
-static const char *Modulus = "%(modulus)x";
-
-// A generator g of a subgroup of Z*_p.
-// static const char *Generator = "%(generator)x";
-
-// The generator g generates a subgroup of
-// order 2^IntGen2Order in Z*_p.
-static const int Generator2Order = %(twoorder)d;
-
-static const char *Roots[] = {
-    %(roots)s
-};
-
-static const char *RootsInv[] = {
-    %(rootsInv)s
-};
-
-#endif /* __PARAMS_H__ */
-""" % {
-    'modulus': modulus,
-    'generator': gen12,
-    'twoorder': 12,
-    'roots': roots, 
-    'rootsInv': rootsInv, 
-}
-
-print output

scripts/gen_params.py

@@ -0,0 +1,126 @@
+"""
+/*
+ * Copyright (c) 2018, Henry Corrigan-Gibbs
+ * 
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. 
+ */
+
+
+This script generates prio/params.h.
+"""
+
+
+# Has a subgroup of order 2^19
+modulus = int('0x8000000000000000080001', 16)
+# Generates the subgroup of order 2^19
+gen19 = int('0x2597c14f48d5b65ed8dcca', 16)
+
+# We want a generator of order 2^12, so compute
+#   gen19^(2^7) = gen19^128   (mod p)
+gen12 = gen19
+for i in range(7):
+    gen12 *= gen12
+    gen12 %= modulus
+    #print gen12
+
+# Sanity check
+rootsL = [1] * 2**12
+rootsInvL = [1] * 2**12
+for i in range(1, 2**12):
+    rootsL[i] = (rootsL[i-1] * gen12) % modulus
+
+assert ((rootsL[2**12 - 1] * gen12) % modulus) == 1
+
+gen12inv = rootsL[2**12 - 1]
+for i in range(1, 2**12):
+    rootsInvL[i] = (rootsInvL[i-1] * gen12inv) % modulus
+    
+    assert rootsInvL[i] != 1 
+assert ((rootsInvL[2**12 - 1] * gen12inv) % modulus) == 1
+
+# We're going to save space by storing the roots once, and using that same
+# data for both the roots and the inverse roots, so make sure we can do that.
+assert rootsL[0] == rootsInvL[0]
+nontrivialRoots = rootsL[1:]
+nontrivialRootsInv = rootsInvL[1:]
+nontrivialRootsInv.reverse()
+assert nontrivialRoots == nontrivialRootsInv
+
+# Instead of generating:
+#
+# static const char* const Roots[] = { "...", ... };
+#
+# we generate one long character array that is the equivalent of:
+#
+# struct roots {
+#   const char r0[SIZE];
+#   const char r1[SIZE];
+#   ...
+# };
+#
+# Because we're no longer storing pointers, just the raw character data,
+# this storage format is smaller and can be shared between processes.
+#
+# We use individual characters, rather than strings, because some compilers
+# reject long concatenated string constants.
+def c_table(strings):
+    def entry(s):
+        chars = ', '.join("'%s'" % x for x in s)
+        return '/* "{root}" */ {chars}, \'\\0\''.format(root=s, chars=chars)
+
+    # Pad all strings to be the same width.
+    width = max(len(s) for s in strings)
+    strings = ["{root:0>{width}s}".format(root=s, width=width) for s in strings]
+    # + 1 for the null terminator in each entry.
+    return width + 1, ',\n    '.join(entry(s) for s in strings)
+
+(width, table) = c_table(['%x' % x for x in rootsL])
+
+output = """
+/*
+ * Copyright (c) 2018, Henry Corrigan-Gibbs
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ */
+
+/****
+ * NOTE: This file was auto-generated from scripts/gen_params.py.
+ * Do not edit this file. Instead, edit the script.
+ */
+
+#ifndef __PARAMS_H__
+#define __PARAMS_H__
+
+// A prime modulus p.
+static const char Modulus[] = "%(modulus)x";
+
+// A generator g of a subgroup of Z*_p.
+// static const char Generator[] = "%(generator)x";
+
+// The generator g generates a subgroup of
+// order 2^Generator2Order in Z*_p.
+static const int Generator2Order = %(twoorder)d;
+
+// Width of entries in Roots.
+static const unsigned int RootWidth = %(width)d;
+
+// clang-format off
+static const char Roots[] = {
+    %(roots)s
+};
+// clang-format on
+
+#endif /* __PARAMS_H__ */
+""" % {
+    'modulus': modulus,
+    'generator': gen12,
+    'twoorder': 12,
+    'width': width,
+    'roots': table, 
+}
+
+print (output,)