Bug 1439470 - Turn TreeNode(nullptr) into a "virtual" sentinel. r=njn

The code before bug 1412722 benefitted from the sentinel being an actual
node object, and some code paths ended up checking its color (always
black) or getting its right and left node, that always returned to the
sentinel.

When TreeNode currently contains a nullptr, all those lead to a null
deref if the calling code is not doing the right checks, which happens
to be the case in at least some places. Instead of relying on the
callers doing the right thing, make the TreeNode do the right thing when
it contains a nullptr, effectively acting as the sentinel in that case.

We additionally ensure that nothing in the calling code will be trying
to change the color or left/right pointers on the sentinel, which is an
extra safe net compared to the code before bug 1412722.

--HG--
extra : rebase_source : 09ab0bf8682092ef6d9a0a5921be3da787d0d548

memory/build/rb.h

@@ -144,7 +144,10 @@ public:
   void Remove(T* aNode) { Remove(TreeNode(aNode)); }
 
   // Helper class to avoid having all the tree traversal code further below
-  // have to use Trait::GetTreeNode, adding visual noise.
+  // have to use Trait::GetTreeNode and do manual null pointer checks, adding
+  // visual noise. Practically speaking TreeNode(nullptr) acts as a virtual
+  // sentinel, that loops back to itself for Left() and Right() and is always
+  // black.
   class TreeNode
   {
   public:
@@ -164,28 +167,40 @@ public:
       return *this;
     }
 
-    TreeNode Left() { return TreeNode(Trait::GetTreeNode(mNode).Left()); }
+    TreeNode Left()
+    {
+      return TreeNode(mNode ? Trait::GetTreeNode(mNode).Left() : nullptr);
+    }
 
     void SetLeft(TreeNode aNode)
     {
+      MOZ_RELEASE_ASSERT(mNode);
       Trait::GetTreeNode(mNode).SetLeft(aNode.mNode);
     }
 
-    TreeNode Right() { return TreeNode(Trait::GetTreeNode(mNode).Right()); }
+    TreeNode Right()
+    {
+      return TreeNode(mNode ? Trait::GetTreeNode(mNode).Right() : nullptr);
+    }
 
     void SetRight(TreeNode aNode)
     {
+      MOZ_RELEASE_ASSERT(mNode);
       Trait::GetTreeNode(mNode).SetRight(aNode.mNode);
     }
 
-    NodeColor Color() { return Trait::GetTreeNode(mNode).Color(); }
+    NodeColor Color()
+    {
+      return mNode ? Trait::GetTreeNode(mNode).Color() : NodeColor::Black;
+    }
 
-    bool IsRed() { return Trait::GetTreeNode(mNode).IsRed(); }
+    bool IsRed() { return Color() == NodeColor::Red; }
 
-    bool IsBlack() { return Trait::GetTreeNode(mNode).IsBlack(); }
+    bool IsBlack() { return Color() == NodeColor::Black; }
 
     void SetColor(NodeColor aColor)
     {
+      MOZ_RELEASE_ASSERT(mNode);
       Trait::GetTreeNode(mNode).SetColor(aColor);
     }