Bug 1732162 - Fix behaviour of OOM during incremental XDR encoding. r=arai

Add the stencil to XDR before instantiating so failures (for OOM, etc) fail the
delazification in the same way a parsing OOM would. This makes code easier to
reason about and lets us keep conservative asserts.

Differential Revision: https://phabricator.services.mozilla.com/D126398

js/src/frontend/BytecodeCompiler.cpp

@@ -1061,16 +1061,6 @@ static bool CompileLazyFunction(JSContext* cx, CompilationInput& input,
   Rooted<CompilationGCOutput> gcOutput(cx);
   {
     BorrowingCompilationStencil borrowingStencil(compilationState);
-    if (!CompilationStencil::instantiateStencils(cx, input, borrowingStencil,
-                                                 gcOutput.get())) {
-      return false;
-    }
-
-    MOZ_ASSERT(lazyFlags == gcOutput.get().script->immutableFlags());
-    MOZ_ASSERT(gcOutput.get().script->outermostScope()->hasOnChain(
-                   ScopeKind::NonSyntactic) ==
-               gcOutput.get().script->immutableFlags().hasFlag(
-                   JSScript::ImmutableFlags::HasNonSyntacticScope));
 
     if (input.source->hasEncoder()) {
       MOZ_ASSERT(!js::UseOffThreadParseGlobal());
@@ -1079,6 +1069,22 @@ static bool CompileLazyFunction(JSContext* cx, CompilationInput& input,
         return false;
       }
     }
+
+    if (!CompilationStencil::instantiateStencils(cx, input, borrowingStencil,
+                                                 gcOutput.get())) {
+      return false;
+    }
+
+    // NOTE: After instantiation succeeds and bytecode is attached, the rest of
+    //       this operation should be infallible. Any failure during
+    //       delazification should restore the function back to a consistent
+    //       lazy state.
+
+    MOZ_ASSERT(lazyFlags == gcOutput.get().script->immutableFlags());
+    MOZ_ASSERT(gcOutput.get().script->outermostScope()->hasOnChain(
+                   ScopeKind::NonSyntactic) ==
+               gcOutput.get().script->immutableFlags().hasFlag(
+                   JSScript::ImmutableFlags::HasNonSyntacticScope));
   }
 
   assertException.reset();

js/src/jit-test/tests/xdr/incremental-oom.js

@@ -0,0 +1,10 @@
+// |jit-test| skip-if: !('oomTest' in this)
+
+// Delazify a function while encoding bytecode.
+oomTest(() => {
+        let code = cacheEntry(`
+                function f() { }
+                f();
+        `);
+        evaluate(code, { saveIncrementalBytecode: true });
+});