Bug 1252329 - Fix interaction between AutoClearTypeInferenceStateOnOOM and compacting GC r=terrence

js/src/jit-test/tests/gc/bug-1252329.js

@@ -0,0 +1,34 @@
+// |jit-test| allow-oom
+
+if (helperThreadCount() == 0 || !('gczeal' in this))
+    quit();
+
+var lfcode = new Array();
+lfcode.push("5");
+lfcode.push(`
+gczeal(8, 2);
+try {
+    [new String, y]
+} catch (e) {}
+oomAfterAllocations(50);
+try {
+    (5).replace(r, () => {});
+} catch (x) {}
+`);
+while (true) {
+  var file = lfcode.shift(); if (file == undefined) { break; }
+  loadFile(file)
+}
+function loadFile(lfVarx) {
+    if (lfVarx.substr(-3) != ".js" && lfVarx.length != 1) {
+        switch (lfRunTypeId) {
+            case 5:
+                var lfGlobal = newGlobal();
+                for (lfLocal in this) {}
+                lfGlobal.offThreadCompileScript(lfVarx);
+                lfGlobal.runOffThreadScript();
+        }
+    } else if (!isNaN(lfVarx)) {
+        lfRunTypeId = parseInt(lfVarx);
+    }
+}

js/src/vm/ObjectGroup.cpp

@@ -1808,14 +1808,20 @@ ObjectGroupCompartment::fixupNewTableAfterMovingGC(NewTable* table)
     if (table && table->initialized()) {
         for (NewTable::Enum e(*table); !e.empty(); e.popFront()) {
             NewEntry entry = e.front();
+            ObjectGroup* group = entry.group.unbarrieredGet();
+
             bool needRekey = false;
             if (IsForwarded(entry.group.unbarrieredGet())) {
-                entry.group.set(Forwarded(entry.group.unbarrieredGet()));
+                group = Forwarded(entry.group.unbarrieredGet());
+                entry.group.set(group);
                 needRekey = true;
             }
-            TaggedProto proto = entry.group.unbarrieredGet()->proto();
+            TaggedProto proto = group->proto();
             if (proto.isObject() && IsForwarded(proto.toObject())) {
                 proto = TaggedProto(Forwarded(proto.toObject()));
+                // Update the group's proto here so that we are able to lookup
+                // entries in this table before all object pointers are updated.
+                group->proto() = proto;
                 needRekey = true;
             }
             if (entry.associated && IsForwarded(entry.associated)) {
@@ -1823,7 +1829,7 @@ ObjectGroupCompartment::fixupNewTableAfterMovingGC(NewTable* table)
                 needRekey = true;
             }
             if (needRekey) {
-                const Class* clasp = entry.group.unbarrieredGet()->clasp();
+                const Class* clasp = group->clasp();
                 if (entry.associated && entry.associated->is<JSFunction>())
                     clasp = nullptr;
                 NewEntry::Lookup lookup(clasp, proto, entry.associated);

js/src/vm/TypeInference.cpp

@@ -2912,12 +2912,15 @@ ObjectGroup::detachNewScript(bool writeBarrier, ObjectGroup* replacement)
 
     if (newScript->analyzed()) {
         ObjectGroupCompartment& objectGroups = newScript->function()->compartment()->objectGroups;
+        TaggedProto proto = this->proto();
+        if (proto.isObject() && IsForwarded(proto.toObject()))
+            proto = TaggedProto(Forwarded(proto.toObject()));
+        JSObject* associated = MaybeForwarded(newScript->function());
         if (replacement) {
             MOZ_ASSERT(replacement->newScript()->function() == newScript->function());
-            objectGroups.replaceDefaultNewGroup(nullptr, proto(), newScript->function(),
-                                                replacement);
+            objectGroups.replaceDefaultNewGroup(nullptr, proto, associated, replacement);
         } else {
-            objectGroups.removeDefaultNewGroup(nullptr, proto(), newScript->function());
+            objectGroups.removeDefaultNewGroup(nullptr, proto, associated);
         }
     } else {
         MOZ_ASSERT(!replacement);