Bug 927705 (part 2a) - Increase pldhash's max capacity from 1<<23 to 1<<26, and protect against uint32_t overflow when computing storage size. r=jorendorff.

--HG--
extra : rebase_source : 9293ec75e5ba8ad416d66ad6bf0c0788c0d35b02

xpcom/glue/pldhash.cpp

@@ -169,6 +169,14 @@ PL_DHashGetStubOps(void)
     return &stub_ops;
 }
 
+static bool
+SizeOfEntryStore(uint32_t capacity, uint32_t entrySize, uint32_t *nbytes)
+{
+    uint64_t nbytes64 = uint64_t(capacity) * uint64_t(entrySize);
+    *nbytes = capacity * entrySize;
+    return uint64_t(*nbytes) == nbytes64;   // returns false on overflow
+}
+
 PLDHashTable *
 PL_NewDHashTable(const PLDHashTableOps *ops, void *data, uint32_t entrySize,
                  uint32_t capacity)
@@ -217,13 +225,14 @@ PL_DHashTableInit(PLDHashTable *table, const PLDHashTableOps *ops, void *data,
     PR_CEILING_LOG2(log2, capacity);
 
     capacity = 1u << log2;
-    if (capacity >= PL_DHASH_SIZE_LIMIT)
+    if (capacity > PL_DHASH_MAX_SIZE)
         return false;
     table->hashShift = PL_DHASH_BITS - log2;
     table->entrySize = entrySize;
     table->entryCount = table->removedCount = 0;
     table->generation = 0;
-    nbytes = capacity * entrySize;
+    if (!SizeOfEntryStore(capacity, entrySize, &nbytes))
+        return false;   // overflowed
 
     table->entryStore = (char *) ops->allocTable(table,
                                                  nbytes + ENTRY_STORE_EXTRA);
@@ -473,10 +482,11 @@ ChangeTable(PLDHashTable *table, int deltaLog2)
     newLog2 = oldLog2 + deltaLog2;
     oldCapacity = 1u << oldLog2;
     newCapacity = 1u << newLog2;
-    if (newCapacity >= PL_DHASH_SIZE_LIMIT)
+    if (newCapacity > PL_DHASH_MAX_SIZE)
         return false;
     entrySize = table->entrySize;
-    nbytes = newCapacity * entrySize;
+    if (!SizeOfEntryStore(newCapacity, entrySize, &nbytes))
+        return false;   // overflowed
 
     newEntryStore = (char *) table->ops->allocTable(table,
                                                     nbytes + ENTRY_STORE_EXTRA);

xpcom/glue/pldhash.h

@@ -28,9 +28,15 @@ extern "C" {
 #define PL_DHASHMETER 1
 #endif
 
-/* Table size limit, do not equal or exceed. */
-#undef PL_DHASH_SIZE_LIMIT
-#define PL_DHASH_SIZE_LIMIT     ((uint32_t)1 << 24)
+/*
+ * Table size limit; do not exceed.  The max capacity used to be 1<<23 but that
+ * occasionally that wasn't enough.  Making it much bigger than 1<<26 probably
+ * isn't worthwhile -- tables that big are kind of ridiculous.  Also, the
+ * growth operation will (deliberately) fail if |capacity * entrySize|
+ * overflows a uint32_t, and entrySize is always at least 8 bytes.
+ */
+#undef PL_DHASH_MAX_SIZE
+#define PL_DHASH_MAX_SIZE     ((uint32_t)1 << 26)
 
 /* Minimum table size, or gross entry count (net is at most .75 loaded). */
 #ifndef PL_DHASH_MIN_SIZE