Bug 1529006 - Use Rooted for NewObjectMetadataState. r=jonco

Remove the `if (!mozilla::IsPointer<T>::value || thing)` check in
GCVariantImplementation::trace, as GCPolicy will dispatch these to
GCPointerPolicy and InternalPointerPolicy (for pointers) and StructGCPolicy (for
non-pointers).

Also use Rooted for prevState_ in AutoSetNewObjectMetadata and remove
inherit from CustomAutoRooter.

js/public/GCVariant.h

@@ -47,9 +47,7 @@ struct GCVariantImplementation<T> {
   template <typename ConcreteVariant>
   static void trace(JSTracer* trc, ConcreteVariant* v, const char* name) {
     T& thing = v->template as<T>();
-    if (!mozilla::IsPointer<T>::value || thing) {
-      GCPolicy<T>::trace(trc, &thing, name);
-    }
+    GCPolicy<T>::trace(trc, &thing, name);
   }
 
   template <typename Matcher, typename ConcreteVariant>
@@ -76,9 +74,7 @@ struct GCVariantImplementation<T, Ts...> {
   static void trace(JSTracer* trc, ConcreteVariant* v, const char* name) {
     if (v->template is<T>()) {
       T& thing = v->template as<T>();
-      if (!mozilla::IsPointer<T>::value || thing) {
-        GCPolicy<T>::trace(trc, &thing, name);
-      }
+      GCPolicy<T>::trace(trc, &thing, name);
     } else {
       Next::trace(trc, v, name);
     }

js/src/vm/Realm.cpp

@@ -303,8 +303,8 @@ void ObjectRealm::trace(JSTracer* trc) {
 void Realm::traceRoots(JSTracer* trc,
                        js::gc::GCRuntime::TraceOrMarkRuntime traceOrMark) {
   if (objectMetadataState_.is<PendingMetadata>()) {
-    TraceRoot(trc, &objectMetadataState_.as<PendingMetadata>(),
-              "on-stack object pending metadata");
+    GCPolicy<NewObjectMetadataState>::trace(trc, &objectMetadataState_,
+                                            "on-stack object pending metadata");
   }
 
   if (!JS::RuntimeHeapIsMinorCollecting()) {
@@ -924,9 +924,8 @@ mozilla::HashCodeScrambler Realm::randomHashCodeScrambler() {
 
 AutoSetNewObjectMetadata::AutoSetNewObjectMetadata(
     JSContext* cx MOZ_GUARD_OBJECT_NOTIFIER_PARAM_IN_IMPL)
-    : CustomAutoRooter(cx),
-      cx_(cx->helperThread() ? nullptr : cx),
-      prevState_(cx->realm()->objectMetadataState_) {
+    : cx_(cx->helperThread() ? nullptr : cx),
+      prevState_(cx, cx->realm()->objectMetadataState_) {
   MOZ_GUARD_OBJECT_NOTIFIER_INIT;
   if (cx_) {
     cx_->realm()->objectMetadataState_ =

js/src/vm/Realm.h

@@ -20,6 +20,7 @@
 
 #include "builtin/Array.h"
 #include "gc/Barrier.h"
+#include "js/GCVariant.h"
 #include "js/UniquePtr.h"
 #include "vm/ArrayBufferObject.h"
 #include "vm/Compartment.h"
@@ -174,6 +175,8 @@ class NewProxyCache {
 // In the presence of internal errors, we do not set the new object's metadata
 // (if it was even allocated) and reset to the previous state on the stack.
 
+// See below in namespace JS for the template specialization for
+// ImmediateMetadata and DelayMetadata.
 struct ImmediateMetadata {};
 struct DelayMetadata {};
 using PendingMetadata = JSObject*;
@@ -181,23 +184,15 @@ using PendingMetadata = JSObject*;
 using NewObjectMetadataState =
     mozilla::Variant<ImmediateMetadata, DelayMetadata, PendingMetadata>;
 
-class MOZ_RAII AutoSetNewObjectMetadata : private JS::CustomAutoRooter {
+class MOZ_RAII AutoSetNewObjectMetadata {
   MOZ_DECL_USE_GUARD_OBJECT_NOTIFIER;
 
   JSContext* cx_;
-  NewObjectMetadataState prevState_;
+  Rooted<NewObjectMetadataState> prevState_;
 
   AutoSetNewObjectMetadata(const AutoSetNewObjectMetadata& aOther) = delete;
   void operator=(const AutoSetNewObjectMetadata& aOther) = delete;
 
- protected:
-  virtual void trace(JSTracer* trc) override {
-    if (prevState_.is<PendingMetadata>()) {
-      TraceRoot(trc, &prevState_.as<PendingMetadata>(),
-                "Object pending metadata");
-    }
-  }
-
  public:
   explicit AutoSetNewObjectMetadata(
       JSContext* cx MOZ_GUARD_OBJECT_NOTIFIER_PARAM);
@@ -294,6 +289,13 @@ class ObjectRealm {
 
 }  // namespace js
 
+namespace JS {
+template <> struct GCPolicy<js::ImmediateMetadata>:
+  public IgnoreGCPolicy<js::ImmediateMetadata> {};
+template <> struct GCPolicy<js::DelayMetadata>:
+  public IgnoreGCPolicy<js::DelayMetadata> {};
+} // namespace JS
+
 class JS::Realm : public JS::shadow::Realm {
   JS::Zone* zone_;
   JSRuntime* runtime_;