Merge mozilla-central to autoland. a=merge

browser/config/mozconfigs/win32/mingwclang

@@ -54,7 +54,7 @@ AR=llvm-ar
 RANLIB=llvm-ranlib
 
 # For Stylo
-BINDGEN_CFLAGS="-I$TOOLTOOL_DIR/clang/i686-w64-mingw32/include/c++/v1 -I$TOOLTOOL_DIR/clang/lib/clang/7.0.0/include -I$TOOLTOOL_DIR/clang/i686-w64-mingw32/include"
+BINDGEN_CFLAGS="-I$TOOLTOOL_DIR/clang/i686-w64-mingw32/include/c++/v1 -I$TOOLTOOL_DIR/clang/lib/clang/8.0.0/include -I$TOOLTOOL_DIR/clang/i686-w64-mingw32/include"
 
 # Since we use windres from binutils without the rest of tools (like cpp), we need to
 # explicitly specify clang as a preprocessor.

browser/config/mozconfigs/win64/mingwclang

@@ -54,7 +54,7 @@ AR=llvm-ar
 RANLIB=llvm-ranlib
 
 # For Stylo
-BINDGEN_CFLAGS="-I$TOOLTOOL_DIR/clang/x86_64-w64-mingw32/include/c++/v1 -I$TOOLTOOL_DIR/clang/lib/clang/7.0.0/include -I$TOOLTOOL_DIR/clang/x86_64-w64-mingw32/include"
+BINDGEN_CFLAGS="-I$TOOLTOOL_DIR/clang/x86_64-w64-mingw32/include/c++/v1 -I$TOOLTOOL_DIR/clang/lib/clang/8.0.0/include -I$TOOLTOOL_DIR/clang/x86_64-w64-mingw32/include"
 
 # Since we use windres from binutils without the rest of tools (like cpp), we need to
 # explicitly specify clang as a preprocessor.

build/build-clang/clang-trunk-mingw.json

@@ -4,12 +4,12 @@
     "build_libcxx": true,
     "build_type": "Release",
     "assertions": false,
-    "llvm_repo": "https://llvm.org/svn/llvm-project/llvm/tags/RELEASE_700/final",
-    "clang_repo": "https://llvm.org/svn/llvm-project/cfe/tags/RELEASE_700/final",
-    "lld_repo": "https://llvm.org/svn/llvm-project/lld/tags/RELEASE_700/final",
-    "compiler_repo": "https://llvm.org/svn/llvm-project/compiler-rt/tags/RELEASE_700/final",
-    "libcxx_repo": "https://llvm.org/svn/llvm-project/libcxx/tags/RELEASE_700/final",
-    "libcxxabi_repo": "https://llvm.org/svn/llvm-project/libcxxabi/tags/RELEASE_700/final",
+    "llvm_repo": "https://llvm.org/svn/llvm-project/llvm/trunk",
+    "clang_repo": "https://llvm.org/svn/llvm-project/cfe/trunk",
+    "lld_repo": "https://llvm.org/svn/llvm-project/lld/trunk",
+    "compiler_repo": "https://llvm.org/svn/llvm-project/compiler-rt/trunk",
+    "libcxx_repo": "https://llvm.org/svn/llvm-project/libcxx/trunk",
+    "libcxxabi_repo": "https://llvm.org/svn/llvm-project/libcxxabi/trunk",
     "python_path": "/usr/bin/python2.7",
     "gcc_dir": "/builds/worker/workspace/build/src/gcc",
     "cc": "/builds/worker/workspace/build/src/gcc/bin/gcc",

devtools/server/actors/replay/debugger.js

@@ -62,7 +62,11 @@ ReplayDebugger.prototype = {
   replayResumeBackward() { RecordReplayControl.resume(/* forward = */ false); },
   replayResumeForward() { RecordReplayControl.resume(/* forward = */ true); },
   replayTimeWarp: RecordReplayControl.timeWarp,
-  replayPause: RecordReplayControl.pause,
+
+  replayPause() {
+    RecordReplayControl.pause();
+    this._repaint();
+  },
 
   addDebuggee() {},
   removeAllDebuggees() {},
@@ -90,6 +94,18 @@ ReplayDebugger.prototype = {
     return this._sendRequest(request);
   },
 
+  // Update graphics according to the current state of the child process. This
+  // should be done anytime we pause and allow the user to interact with the
+  // debugger.
+  _repaint() {
+    const rv = this._sendRequestAllowDiverge({ type: "repaint" });
+    if ("width" in rv && "height" in rv) {
+      RecordReplayControl.hadRepaint(rv.width, rv.height);
+    } else {
+      RecordReplayControl.hadRepaintFailure();
+    }
+  },
+
   _setBreakpoint(handler, position, data) {
     const id = RecordReplayControl.setBreakpoint(handler, position);
     this._breakpoints.push({id, position, data});
@@ -163,10 +179,24 @@ ReplayDebugger.prototype = {
     return this._scripts[data.id];
   },
 
-  findScripts() {
-    // Note: Debugger's findScripts() method takes a query argument, which
-    // we ignore here.
-    const data = this._sendRequest({ type: "findScripts" });
+  _convertScriptQuery(query) {
+    // Make a copy of the query, converting properties referring to debugger
+    // things into their associated ids.
+    const rv = Object.assign({}, query);
+    if ("global" in query) {
+      rv.global = query.global._data.id;
+    }
+    if ("source" in query) {
+      rv.source = query.source._data.id;
+    }
+    return rv;
+  },
+
+  findScripts(query) {
+    const data = this._sendRequest({
+      type: "findScripts",
+      query: this._convertScriptQuery(query)
+    });
     return data.map(script => this._addScript(script));
   },
 
@@ -315,7 +345,8 @@ ReplayDebugger.prototype = {
   get onEnterFrame() { return this._breakpointKindGetter("EnterFrame"); },
   set onEnterFrame(handler) {
     this._breakpointKindSetter("EnterFrame", handler,
-                               () => handler.call(this, this.getNewestFrame()));
+                               () => { this._repaint();
+                                       handler.call(this, this.getNewestFrame()); });
   },
 
   get replayingOnPopFrame() {
@@ -326,7 +357,8 @@ ReplayDebugger.prototype = {
 
   set replayingOnPopFrame(handler) {
     if (handler) {
-      this._setBreakpoint(() => handler.call(this, this.getNewestFrame()),
+      this._setBreakpoint(() => { this._repaint();
+                                  handler.call(this, this.getNewestFrame()); },
                           { kind: "OnPop" }, handler);
     } else {
       this._clearMatchingBreakpoints(({position}) => {
@@ -340,7 +372,8 @@ ReplayDebugger.prototype = {
   },
   set replayingOnForcedPause(handler) {
     this._breakpointKindSetter("ForcedPause", handler,
-                               () => handler.call(this, this.getNewestFrame()));
+                               () => { this._repaint();
+                                       handler.call(this, this.getNewestFrame()); });
   },
 
   getNewConsoleMessage() {
@@ -388,7 +421,8 @@ ReplayDebuggerScript.prototype = {
   getPredecessorOffsets(pc) { return this._forward("getPredecessorOffsets", pc); },
 
   setBreakpoint(offset, handler) {
-    this._dbg._setBreakpoint(() => { handler.hit(this._dbg.getNewestFrame()); },
+    this._dbg._setBreakpoint(() => { this._dbg._repaint();
+                                     handler.hit(this._dbg.getNewestFrame()); },
                              { kind: "Break", script: this._data.id, offset },
                              handler);
   },
@@ -505,7 +539,8 @@ ReplayDebuggerFrame.prototype = {
     this._clearOnStepBreakpoints();
     offsets.forEach(offset => {
       this._dbg._setBreakpoint(
-        () => handler.call(this._dbg.getNewestFrame()),
+        () => { this._dbg._repaint();
+                handler.call(this._dbg.getNewestFrame()); },
         { kind: "OnStep",
           script: this._data.script,
           offset,
@@ -523,6 +558,7 @@ ReplayDebuggerFrame.prototype = {
   set onPop(handler) {
     if (handler) {
       this._dbg._setBreakpoint(() => {
+          this._dbg._repaint();
           const result = this._dbg._sendRequest({ type: "popFrameResult" });
           handler.call(this._dbg.getNewestFrame(),
                        this._dbg._convertCompletionValue(result));
@@ -604,7 +640,8 @@ ReplayDebuggerObject.prototype = {
 
   getOwnPropertyDescriptor(name) {
     this._ensureProperties();
-    return this._properties[name];
+    const desc = this._properties[name];
+    return desc ? this._convertPropertyDescriptor(desc) : null;
   },
 
   _ensureProperties() {
@@ -614,21 +651,24 @@ ReplayDebuggerObject.prototype = {
         id: this._data.id
       });
       this._properties = {};
-      properties.forEach(({name, desc}) => {
-        if ("value" in desc) {
-          desc.value = this._dbg._convertValue(desc.value);
-        }
-        if ("get" in desc) {
-          desc.get = this._dbg._getObject(desc.get);
-        }
-        if ("set" in desc) {
-          desc.set = this._dbg._getObject(desc.set);
-        }
-        this._properties[name] = desc;
-      });
+      properties.forEach(({name, desc}) => { this._properties[name] = desc; });
     }
   },
 
+  _convertPropertyDescriptor(desc) {
+    const rv = Object.assign({}, desc);
+    if ("value" in desc) {
+      rv.value = this._dbg._convertValue(desc.value);
+    }
+    if ("get" in desc) {
+      rv.get = this._dbg._getObject(desc.get);
+    }
+    if ("set" in desc) {
+      rv.set = this._dbg._getObject(desc.set);
+    }
+    return rv;
+  },
+
   get allocationSite() { NYI(); },
   get errorMessageName() { NYI(); },
   get errorNotes() { NYI(); },

devtools/server/actors/replay/graphics.js

@@ -14,7 +14,7 @@
 
 ChromeUtils.import("resource://gre/modules/Services.jsm");
 
-function updateWindow(window, buffer, width, height) {
+function updateWindow(window, buffer, width, height, hadFailure) {
   // Make sure the window has a canvas filling the screen.
   let canvas = window.middlemanCanvas;
   if (!canvas) {
@@ -36,10 +36,20 @@ function updateWindow(window, buffer, width, height) {
       `scale(${ 1 / scale }) translate(-${ width / scale }px, -${ height / scale }px)`;
   }
 
+  const cx = canvas.getContext("2d");
+
   const graphicsData = new Uint8Array(buffer);
-  const imageData = canvas.getContext("2d").getImageData(0, 0, width, height);
+  const imageData = cx.getImageData(0, 0, width, height);
   imageData.data.set(graphicsData);
-  canvas.getContext("2d").putImageData(imageData, 0, 0);
+  cx.putImageData(imageData, 0, 0);
+
+  // Indicate to the user when repainting failed and we are showing old painted
+  // graphics instead of the most up-to-date graphics.
+  if (hadFailure) {
+    cx.fillStyle = "red";
+    cx.font = "48px serif";
+    cx.fillText("PAINT FAILURE", 10, 50);
+  }
 
   // Make recording/replaying tabs easier to differentiate from other tabs.
   window.document.title = "RECORD/REPLAY";
@@ -48,11 +58,11 @@ function updateWindow(window, buffer, width, height) {
 // Entry point for when we have some new graphics data from the child process
 // to draw.
 // eslint-disable-next-line no-unused-vars
-function Update(buffer, width, height) {
+function Update(buffer, width, height, hadFailure) {
   try {
     // Paint to all windows we can find. Hopefully there is only one.
     for (const window of Services.ww.getWindowEnumerator()) {
-      updateWindow(window, buffer, width, height);
+      updateWindow(window, buffer, width, height, hadFailure);
     }
   } catch (e) {
     dump("Middleman Graphics Update Exception: " + e + "\n");

devtools/server/actors/replay/replay.js

@@ -523,10 +523,30 @@ function forwardToScript(name) {
 
 const gRequestHandlers = {
 
+  repaint() {
+    if (!RecordReplayControl.maybeDivergeFromRecording()) {
+      return {};
+    }
+    return RecordReplayControl.repaint();
+  },
+
   findScripts(request) {
+    const query = Object.assign({}, request.query);
+    if ("global" in query) {
+      query.global = gPausedObjects.getObject(query.global);
+    }
+    if ("source" in query) {
+      query.source = gScriptSources.getObject(query.source);
+      if (!query.source) {
+        return [];
+      }
+    }
+    const scripts = dbg.findScripts(query);
     const rv = [];
-    gScripts.forEach((id) => {
-      rv.push(getScriptData(id));
+    scripts.forEach(script => {
+      if (considerScript(script)) {
+        rv.push(getScriptData(gScripts.getId(script)));
+      }
     });
     return rv;
   },

dom/ipc/TabChild.cpp

@@ -1226,10 +1226,10 @@ TabChild::RecvShow(const ScreenIntSize& aSize,
   }
 
   // We have now done enough initialization for the record/replay system to
-  // create checkpoints. Try to create the initial checkpoint now, in case this
-  // process never paints later on (the usual place where checkpoints occur).
+  // create checkpoints. Create a checkpoint now, in case this process never
+  // paints later on (the usual place where checkpoints occur).
   if (recordreplay::IsRecordingOrReplaying()) {
-    recordreplay::child::MaybeCreateInitialCheckpoint();
+    recordreplay::child::CreateCheckpoint();
   }
 
   return IPC_OK();

gfx/2d/NativeFontResourceDWrite.cpp

@@ -272,8 +272,8 @@ NativeFontResourceDWrite::Create(uint8_t *aFontData, uint32_t aDataLength,
   }
 
   RefPtr<NativeFontResourceDWrite> fontResource =
-    new NativeFontResourceDWrite(factory, fontFile.forget(), faceType,
-                                 numberOfFaces, aNeedsCairo);
+    new NativeFontResourceDWrite(factory, fontFile.forget(), ffsRef.forget(),
+                                 faceType, numberOfFaces, aNeedsCairo);
   return fontResource.forget();
 }
 

gfx/2d/NativeFontResourceDWrite.h

@@ -40,14 +40,20 @@ public:
 private:
   NativeFontResourceDWrite(IDWriteFactory *aFactory,
                            already_AddRefed<IDWriteFontFile> aFontFile,
+                           already_AddRefed<IDWriteFontFileStream> aFontFileStream,
                            DWRITE_FONT_FACE_TYPE aFaceType,
                            uint32_t aNumberOfFaces, bool aNeedsCairo)
-    : mFactory(aFactory), mFontFile(aFontFile), mFaceType(aFaceType)
-    , mNumberOfFaces(aNumberOfFaces), mNeedsCairo(aNeedsCairo)
+    : mFactory(aFactory)
+    , mFontFile(aFontFile)
+    , mFontFileStream(aFontFileStream)
+    , mFaceType(aFaceType)
+    , mNumberOfFaces(aNumberOfFaces)
+    , mNeedsCairo(aNeedsCairo)
   {}
 
   IDWriteFactory *mFactory;
   RefPtr<IDWriteFontFile> mFontFile;
+  RefPtr<IDWriteFontFileStream> mFontFileStream;
   DWRITE_FONT_FACE_TYPE mFaceType;
   uint32_t mNumberOfFaces;
   bool mNeedsCairo;

gfx/layers/ipc/ShadowLayers.cpp

@@ -771,10 +771,6 @@ ShadowLayerForwarder::EndTransaction(const nsIntRegion& aRegionToClear,
   // finish. If it does we don't have to delay messages at all.
   GetCompositorBridgeChild()->PostponeMessagesIfAsyncPainting();
 
-  if (recordreplay::IsRecordingOrReplaying()) {
-    recordreplay::child::NotifyPaintStart();
-  }
-
   MOZ_LAYERS_LOG(("[LayersForwarder] sending transaction..."));
   RenderTraceScope rendertrace3("Forward Transaction", "000093");
   if (!mShadowManager->SendUpdate(info)) {
@@ -782,15 +778,15 @@ ShadowLayerForwarder::EndTransaction(const nsIntRegion& aRegionToClear,
     return false;
   }
 
-  if (recordreplay::IsRecordingOrReplaying()) {
-    recordreplay::child::WaitForPaintToComplete();
-  }
-
   if (startTime) {
     mPaintTiming.sendMs() = (TimeStamp::Now() - startTime.value()).ToMilliseconds();
     mShadowManager->SendRecordPaintTimes(mPaintTiming);
   }
 
+  if (recordreplay::IsRecordingOrReplaying()) {
+    recordreplay::child::NotifyPaintStart();
+  }
+
   *aSent = true;
   mIsFirstPaint = false;
   mFocusTarget = FocusTarget();

gfx/webrender/src/renderer.rs

@@ -4624,6 +4624,7 @@ impl DebugServer {
 // that we can use in wrench reftests to ensure that
 // tests are batching and/or allocating on render
 // targets as we expect them to.
+#[repr(C)]
 pub struct RendererStats {
     pub total_draw_calls: usize,
     pub alpha_target_count: usize,

gfx/webrender/src/texture_cache.rs

@@ -19,14 +19,6 @@ use std::cmp;
 use std::mem;
 use std::rc::Rc;
 
-// The fixed number of layers for the shared texture cache.
-// There is one array texture per image format, allocated lazily.
-const TEXTURE_ARRAY_LAYERS_LINEAR: usize = 4;
-const TEXTURE_ARRAY_LAYERS_NEAREST: usize = 1;
-
-// The dimensions of each layer in the texture cache.
-const TEXTURE_LAYER_DIMENSIONS: u32 = 2048;
-
 // The size of each region (page) in a texture layer.
 const TEXTURE_REGION_DIMENSIONS: u32 = 512;
 
@@ -238,25 +230,44 @@ impl TextureCache {
     pub fn new(max_texture_size: u32) -> Self {
         TextureCache {
             max_texture_size,
+            // Used primarily for cached shadow masks. There can be lots of
+            // these on some pages like francine, but most pages don't use it
+            // much.
             array_a8_linear: TextureArray::new(
                 ImageFormat::R8,
                 TextureFilter::Linear,
-                TEXTURE_ARRAY_LAYERS_LINEAR,
+                1024,
+                1,
             ),
+            // Used for experimental hdr yuv texture support, but not used in
+            // production Firefox.
             array_a16_linear: TextureArray::new(
                 ImageFormat::R16,
                 TextureFilter::Linear,
-                TEXTURE_ARRAY_LAYERS_LINEAR,
+                1024,
+                1,
             ),
+            // The primary cache for images, glyphs, etc.
             array_rgba8_linear: TextureArray::new(
                 ImageFormat::BGRA8,
                 TextureFilter::Linear,
-                TEXTURE_ARRAY_LAYERS_LINEAR,
+                2048,
+                4,
             ),
+            // Used for image-rendering: crisp. This is mostly favicons, which
+            // are small. Some other images use it too, but those tend to be
+            // larger than 512x512 and thus don't use the shared cache anyway.
+            //
+            // Ideally we'd use 512 as the dimensions here, since we don't really
+            // need more. But once a page gets something of a given size bucket
+            // assigned to it, all further allocations need to be of that size.
+            // So using 1024 gives us 4 buckets instead of 1, which in practice
+            // is probably enough.
             array_rgba8_nearest: TextureArray::new(
                 ImageFormat::BGRA8,
                 TextureFilter::Nearest,
-                TEXTURE_ARRAY_LAYERS_NEAREST,
+                1024,
+                1,
             ),
             next_id: CacheTextureId(1),
             pending_updates: TextureUpdateList::new(),
@@ -746,8 +757,8 @@ impl TextureCache {
             let update_op = TextureUpdate {
                 id: texture_id,
                 op: TextureUpdateOp::Create {
-                    width: TEXTURE_LAYER_DIMENSIONS,
-                    height: TEXTURE_LAYER_DIMENSIONS,
+                    width: texture_array.dimensions,
+                    height: texture_array.dimensions,
                     format: descriptor.format,
                     filter: texture_array.filter,
                     // TODO(gw): Creating a render target here is only used
@@ -1081,6 +1092,7 @@ impl TextureRegion {
 #[cfg_attr(feature = "replay", derive(Deserialize))]
 struct TextureArray {
     filter: TextureFilter,
+    dimensions: u32,
     layer_count: usize,
     format: ImageFormat,
     is_allocated: bool,
@@ -1092,11 +1104,13 @@ impl TextureArray {
     fn new(
         format: ImageFormat,
         filter: TextureFilter,
-        layer_count: usize
+        dimensions: u32,
+        layer_count: usize,
     ) -> Self {
         TextureArray {
             format,
             filter,
+            dimensions,
             layer_count,
             is_allocated: false,
             regions: Vec::new(),
@@ -1112,8 +1126,8 @@ impl TextureArray {
 
     fn update_profile(&self, counter: &mut ResourceProfileCounter) {
         if self.is_allocated {
-            let size = self.layer_count as u32 * TEXTURE_LAYER_DIMENSIONS *
-                TEXTURE_LAYER_DIMENSIONS * self.format.bytes_per_pixel();
+            let size = self.layer_count as u32 * self.dimensions *
+                self.dimensions * self.format.bytes_per_pixel();
             counter.set(self.layer_count as usize, size as usize);
         } else {
             counter.set(0, 0);
@@ -1132,8 +1146,8 @@ impl TextureArray {
         // This means that very rarely used image formats can be
         // added but won't allocate a cache if never used.
         if !self.is_allocated {
-            debug_assert!(TEXTURE_LAYER_DIMENSIONS % TEXTURE_REGION_DIMENSIONS == 0);
-            let regions_per_axis = TEXTURE_LAYER_DIMENSIONS / TEXTURE_REGION_DIMENSIONS;
+            debug_assert!(self.dimensions % TEXTURE_REGION_DIMENSIONS == 0);
+            let regions_per_axis = self.dimensions / TEXTURE_REGION_DIMENSIONS;
             for layer_index in 0 .. self.layer_count {
                 for y in 0 .. regions_per_axis {
                     for x in 0 .. regions_per_axis {

gfx/webrender_bindings/revision.txt

@@ -1 +1 @@
-74f265e447d2927c27d4320c676779956d39eaf0
+b648c76e2dc2cbcbd635322cdf94ab9d5320e0c1

ipc/chromium/src/base/message_pump_default.cc

@@ -60,6 +60,19 @@ void MessagePumpDefault::Run(Delegate* delegate) {
       AUTO_PROFILER_LABEL("MessagePumpDefault::Run:Wait", IDLE);
       {
         AUTO_PROFILER_THREAD_SLEEP;
+        // Some threads operating a message pump (i.e. the compositor thread)
+        // can diverge from Web Replay recordings, after which the thread needs
+        // to be explicitly notified in order to coordinate with the
+        // record/replay checkpoint system.
+        if (mozilla::recordreplay::IsRecordingOrReplaying()) {
+          // Workaround static analysis. Message pumps for which the lambda
+          // below might be called will not be destroyed.
+          void* thiz = this;
+
+          mozilla::recordreplay::MaybeWaitForCheckpointSave();
+          mozilla::recordreplay::NotifyUnrecordedWait([=]() { ((MessagePumpDefault*)thiz)->ScheduleWork(); },
+                                                      /* aOnlyWhenDiverged = */ true);
+        }
         event_.Wait();
       }
     } else {

ipc/glue/MessageChannel.cpp

@@ -995,6 +995,17 @@ MessageChannel::Echo(Message* aMsg)
 bool
 MessageChannel::Send(Message* aMsg)
 {
+    if (recordreplay::HasDivergedFromRecording() &&
+        recordreplay::child::SuppressMessageAfterDiverge(aMsg))
+    {
+        // Only certain IPDL messages are allowed to be sent in a replaying
+        // process after it has diverged from the recording, to avoid
+        // deadlocking with threads that remain idle. The browser remains
+        // paused after diverging from the recording, and other IPDL messages
+        // do not need to be sent.
+        return true;
+    }
+
     if (aMsg->size() >= kMinTelemetryMessageSize) {
         Telemetry::Accumulate(Telemetry::IPC_MESSAGE_SIZE2, aMsg->size());
     }

js/src/jit-test/tests/wasm/passive-segs-nonboundary.js

@@ -356,21 +356,21 @@ function checkMiscPrefixed(opcode, expect_failure) {
                 [funcBody(
                     {locals:[],
                      body:[0x41, 0x0, 0x41, 0x0, 0x41, 0x0, // 3 x const.i32 0
-                           MiscPrefix, opcode]})])]);
+                           MiscPrefix, ...opcode]})])]);
     if (expect_failure) {
         assertErrorMessage(() => new WebAssembly.Module(binary),
                            WebAssembly.CompileError, /unrecognized opcode/);
     } else {
-        assertEq(true, WebAssembly.validate(binary));
+        assertEq(WebAssembly.validate(binary), true);
     }
 }
 
 //-----------------------------------------------------------
 // Verification cases for memory.copy/fill opcode encodings
 
-checkMiscPrefixed(0x0a, false); // memory.copy
-checkMiscPrefixed(0x0b, false); // memory.fill
-checkMiscPrefixed(0x0f, true);  // table.copy+1, which is currently unassigned
+checkMiscPrefixed([0x0a, 0x00], false); // memory.copy, flags=0
+checkMiscPrefixed([0x0b, 0x00], false); // memory.fill, flags=0
+checkMiscPrefixed([0x0f], true);        // table.copy+1, which is currently unassigned
 
 //-----------------------------------------------------------
 // Verification cases for memory.copy/fill arguments

js/src/vm/HelperThreads.cpp

@@ -162,7 +162,8 @@ MaybeWaitForRecordReplayCheckpointSave(AutoLockHelperThreadState& locked)
 
         // Now that we are holding the helper thread state lock again,
         // notify the record/replay system that we might block soon.
-        mozilla::recordreplay::NotifyUnrecordedWait(HelperThread::WakeupAll);
+        mozilla::recordreplay::NotifyUnrecordedWait(HelperThread::WakeupAll,
+                                                    /* aOnlyWhenDiverged = */ false);
     }
 }
 

js/src/vm/TypedArrayObject-inl.h

@@ -279,17 +279,10 @@ class ElementSpecific
             return true;
         }
 
-        // Inhibit unaligned accesses on ARM (bug 1097253, a compiler bug).
-#if defined(__arm__) && MOZ_IS_GCC
-#  define JS_VOLATILE_ARM volatile
-#else
-#  define JS_VOLATILE_ARM
-#endif
-
         SharedMem<void*> data = Ops::extract(source);
         switch (source->type()) {
           case Scalar::Int8: {
-            SharedMem<JS_VOLATILE_ARM int8_t*> src = data.cast<JS_VOLATILE_ARM int8_t*>();
+            SharedMem<int8_t*> src = data.cast<int8_t*>();
             for (uint32_t i = 0; i < count; ++i) {
                 Ops::store(dest++, ConvertNumber<T>(Ops::load(src++)));
             }
@@ -297,49 +290,49 @@ class ElementSpecific
           }
           case Scalar::Uint8:
           case Scalar::Uint8Clamped: {
-            SharedMem<JS_VOLATILE_ARM uint8_t*> src = data.cast<JS_VOLATILE_ARM uint8_t*>();
+            SharedMem<uint8_t*> src = data.cast<uint8_t*>();
             for (uint32_t i = 0; i < count; ++i) {
                 Ops::store(dest++, ConvertNumber<T>(Ops::load(src++)));
             }
             break;
           }
           case Scalar::Int16: {
-            SharedMem<JS_VOLATILE_ARM int16_t*> src = data.cast<JS_VOLATILE_ARM int16_t*>();
+            SharedMem<int16_t*> src = data.cast<int16_t*>();
             for (uint32_t i = 0; i < count; ++i) {
                 Ops::store(dest++, ConvertNumber<T>(Ops::load(src++)));
             }
             break;
           }
           case Scalar::Uint16: {
-            SharedMem<JS_VOLATILE_ARM uint16_t*> src = data.cast<JS_VOLATILE_ARM uint16_t*>();
+            SharedMem<uint16_t*> src = data.cast<uint16_t*>();
             for (uint32_t i = 0; i < count; ++i) {
                 Ops::store(dest++, ConvertNumber<T>(Ops::load(src++)));
             }
             break;
           }
           case Scalar::Int32: {
-            SharedMem<JS_VOLATILE_ARM int32_t*> src = data.cast<JS_VOLATILE_ARM int32_t*>();
+            SharedMem<int32_t*> src = data.cast<int32_t*>();
             for (uint32_t i = 0; i < count; ++i) {
                 Ops::store(dest++, ConvertNumber<T>(Ops::load(src++)));
             }
             break;
           }
           case Scalar::Uint32: {
-            SharedMem<JS_VOLATILE_ARM uint32_t*> src = data.cast<JS_VOLATILE_ARM uint32_t*>();
+            SharedMem<uint32_t*> src = data.cast<uint32_t*>();
             for (uint32_t i = 0; i < count; ++i) {
                 Ops::store(dest++, ConvertNumber<T>(Ops::load(src++)));
             }
             break;
           }
           case Scalar::Float32: {
-            SharedMem<JS_VOLATILE_ARM float*> src = data.cast<JS_VOLATILE_ARM float*>();
+            SharedMem<float*> src = data.cast<float*>();
             for (uint32_t i = 0; i < count; ++i) {
                 Ops::store(dest++, ConvertNumber<T>(Ops::load(src++)));
             }
             break;
           }
           case Scalar::Float64: {
-            SharedMem<JS_VOLATILE_ARM double*> src = data.cast<JS_VOLATILE_ARM double*>();
+            SharedMem<double*> src = data.cast<double*>();
             for (uint32_t i = 0; i < count; ++i) {
                 Ops::store(dest++, ConvertNumber<T>(Ops::load(src++)));
             }
@@ -349,8 +342,6 @@ class ElementSpecific
             MOZ_CRASH("setFromTypedArray with a typed array with bogus type");
         }
 
-#undef JS_VOLATILE_ARM
-
         return true;
     }
 

js/src/wasm/WasmBaselineCompile.cpp

@@ -9239,8 +9239,15 @@ BaseCompiler::emitInstanceCall(uint32_t lineOrBytecode, const MIRTypeVector& sig
 
     popValueStackBy(numArgs);
 
-    // Note, a number of clients of emitInstanceCall currently assume that the
-    // following operation does not destroy ReturnReg.
+    // Note, many clients of emitInstanceCall currently assume that pushing the
+    // result here does not destroy ReturnReg.
+    //
+    // Furthermore, clients assume that even if retType == ExprType::Void, the
+    // callee may have returned a status result and left it in ReturnReg for us
+    // to find, and that that register will not be destroyed here (or above).
+    // In this case the callee will have a C++ declaration stating that there is
+    // a return value.  Examples include memory and table operations that are
+    // implemented as callouts.
 
     pushReturnedIfNonVoid(baselineCall, retType);
 }
@@ -10080,14 +10087,7 @@ BaseCompiler::emitStructNarrow()
         return true;
     }
 
-    // Null pointers are just passed through.
-
-    Label done;
-    Label doTest;
     RegPtr rp = popRef();
-    masm.branchTestPtr(Assembler::NonZero, rp, rp, &doTest);
-    pushRef(NULLREF_VALUE);
-    masm.jump(&done);
 
     // AnyRef -> (ref T) must first unbox; leaves rp or null
 
@@ -10097,15 +10097,11 @@ BaseCompiler::emitStructNarrow()
 
     const StructType& outputStruct = env_.types[outputType.refTypeIndex()].structType();
 
-    masm.bind(&doTest);
-
     pushI32(mustUnboxAnyref);
     pushI32(outputStruct.moduleIndex_);
     pushRef(rp);
     emitInstanceCall(lineOrBytecode, SigPIIP_, ExprType::AnyRef, SymbolicAddress::StructNarrow);
 
-    masm.bind(&done);
-
     return true;
 }
 

js/src/wasm/WasmInstance.cpp

@@ -714,13 +714,18 @@ Instance::structNew(Instance* instance, uint32_t typeIndex)
 
 /* static */ void*
 Instance::structNarrow(Instance* instance, uint32_t mustUnboxAnyref, uint32_t outputTypeIndex,
-                       void* nonnullPtr)
+                       void* maybeNullPtr)
 {
     JSContext* cx = TlsContext.get();
 
     Rooted<TypedObject*> obj(cx);
     Rooted<StructTypeDescr*> typeDescr(cx);
 
+    if (maybeNullPtr == nullptr) {
+        return maybeNullPtr;
+    }
+
+    void* nonnullPtr = maybeNullPtr;
     if (mustUnboxAnyref) {
         Rooted<NativeObject*> no(cx, static_cast<NativeObject*>(nonnullPtr));
         if (!no->is<TypedObject>()) {

js/src/wasm/WasmInstance.h

@@ -192,7 +192,8 @@ class Instance
                              uint32_t srcOffset, uint32_t len, uint32_t segIndex);
     static void postBarrier(Instance* instance, gc::Cell** location);
     static void* structNew(Instance* instance, uint32_t typeIndex);
-    static void* structNarrow(Instance* instance, uint32_t mustUnboxAnyref, uint32_t outputTypeIndex, void* ptr);
+    static void* structNarrow(Instance* instance, uint32_t mustUnboxAnyref, uint32_t outputTypeIndex,
+                              void* maybeNullPtr);
 };
 
 typedef UniquePtr<Instance> UniqueInstance;

js/src/wasm/WasmOpIter.h

@@ -2055,6 +2055,14 @@ OpIter<Policy>::readMemOrTableCopy(bool isMem, Value* dst, Value* src, Value* le
         }
     }
 
+    uint8_t memOrTableFlags;
+    if (!readFixedU8(&memOrTableFlags)) {
+        return fail(isMem ? "unable to read memory flags" : "unable to read table flags");
+    }
+    if (memOrTableFlags != 0) {
+        return fail(isMem ? "memory flags must be zero" : "table flags must be zero");
+    }
+
     if (!popWithType(ValType::I32, len)) {
         return false;
     }
@@ -2114,6 +2122,14 @@ OpIter<Policy>::readMemFill(Value* start, Value* val, Value* len)
         return fail("can't touch memory without memory");
     }
 
+    uint8_t memoryFlags;
+    if (!readFixedU8(&memoryFlags)) {
+        return fail("unable to read memory flags");
+    }
+    if (memoryFlags != 0) {
+        return fail("memory flags must be zero");
+    }
+
     if (!popWithType(ValType::I32, len)) {
         return false;
     }
@@ -2158,6 +2174,14 @@ OpIter<Policy>::readMemOrTableInit(bool isMem, uint32_t* segIndex,
         return false;
     }
 
+    uint8_t memOrTableFlags;
+    if (!readFixedU8(&memOrTableFlags)) {
+        return fail(isMem ? "unable to read memory flags" : "unable to read table flags");
+    }
+    if (memOrTableFlags != 0) {
+        return fail(isMem ? "memory flags must be zero" : "table flags must be zero");
+    }
+
     if (!readVarU32(segIndex)) {
         return false;
     }

js/src/wasm/WasmTextToBinary.cpp

@@ -6290,13 +6290,16 @@ EncodeWake(Encoder& e, AstWake& s)
 }
 
 #ifdef ENABLE_WASM_BULKMEM_OPS
+static const uint8_t DEFAULT_MEM_TABLE_FLAGS = 0;
+
 static bool
 EncodeMemOrTableCopy(Encoder& e, AstMemOrTableCopy& s)
 {
     return EncodeExpr(e, s.dest()) &&
            EncodeExpr(e, s.src()) &&
            EncodeExpr(e, s.len()) &&
-           e.writeOp(s.isMem() ? MiscOp::MemCopy : MiscOp::TableCopy);
+           e.writeOp(s.isMem() ? MiscOp::MemCopy : MiscOp::TableCopy) &&
+           e.writeFixedU8(DEFAULT_MEM_TABLE_FLAGS);
 }
 
 static bool
@@ -6312,7 +6315,8 @@ EncodeMemFill(Encoder& e, AstMemFill& s)
     return EncodeExpr(e, s.start()) &&
            EncodeExpr(e, s.val()) &&
            EncodeExpr(e, s.len()) &&
-           e.writeOp(MiscOp::MemFill);
+           e.writeOp(MiscOp::MemFill) &&
+           e.writeFixedU8(DEFAULT_MEM_TABLE_FLAGS);
 }
 
 static bool
@@ -6322,6 +6326,7 @@ EncodeMemOrTableInit(Encoder& e, AstMemOrTableInit& s)
            EncodeExpr(e, s.src()) &&
            EncodeExpr(e, s.len()) &&
            e.writeOp(s.isMem() ? MiscOp::MemInit : MiscOp::TableInit) &&
+           e.writeFixedU8(DEFAULT_MEM_TABLE_FLAGS) &&
            e.writeVarU32(s.segIndex());
 }
 #endif

layout/ipc/VsyncChild.cpp

@@ -67,14 +67,12 @@ VsyncChild::RecvNotify(const TimeStamp& aVsyncTimestamp)
   MOZ_ASSERT(NS_IsMainThread());
   MOZ_ASSERT(!mIsShutdown);
 
-  // Ignore Vsync messages sent to a recording/replaying process. Vsyncs are
-  // triggered at the top of the main thread's event loop instead.
-  if (recordreplay::IsRecordingOrReplaying()) {
-    return IPC_OK();
-  }
-
   SchedulerGroup::MarkVsyncRan();
   if (mObservingVsync && mObserver) {
+    if (recordreplay::IsRecordingOrReplaying() && !recordreplay::child::OnVsync()) {
+      return IPC_OK();
+    }
+
     mObserver->NotifyVsync(aVsyncTimestamp);
   }
   return IPC_OK();

mfbt/RecordReplay.cpp

@@ -51,7 +51,8 @@ namespace recordreplay {
   Macro(InternalRecordReplayBytes,                              \
         (void* aData, size_t aSize), (aData, aSize))            \
   Macro(NotifyUnrecordedWait,                                   \
-        (const std::function<void()>& aCallback), (aCallback))  \
+        (const std::function<void()>& aCallback, bool aOnlyWhenDiverged), \
+        (aCallback, aOnlyWhenDiverged))                         \
   Macro(MaybeWaitForCheckpointSave, (), ())                     \
   Macro(InternalInvalidateRecording, (const char* aWhy), (aWhy)) \
   Macro(InternalDestroyPLDHashTableCallbacks,                   \

mfbt/RecordReplay.h

@@ -258,12 +258,16 @@ static inline bool HasDivergedFromRecording();
 // The callback passed to NotifyUnrecordedWait will be invoked at most once
 // by the main thread whenever the main thread is waiting for other threads to
 // become idle, and at most once after the call to NotifyUnrecordedWait if the
-// main thread is already waiting for other threads to become idle.
+// main thread is already waiting for other threads to become idle. If
+// aOnlyWhenDiverged is specified, the callback will only be invoked if the
+// thread has diverged from the recording (causing all resources to be treated
+// as unrecorded).
 //
 // The callback should poke the thread so that it is no longer blocked on the
 // resource. The thread must call MaybeWaitForCheckpointSave before blocking
 // again.
-MFBT_API void NotifyUnrecordedWait(const std::function<void()>& aCallback);
+MFBT_API void NotifyUnrecordedWait(const std::function<void()>& aCallback,
+                                   bool aOnlyWhenDiverged);
 MFBT_API void MaybeWaitForCheckpointSave();
 
 // API for debugging inconsistent behavior between recording and replay.

modules/libmar/sign/mar_sign.c

@@ -534,6 +534,7 @@ extract_signature(const char *src, uint32_t sigIndex, const char * dest)
   for (i = 0; i <= sigIndex; i++) {
     /* Avoid leaking while skipping signatures */
     free(extractedSignature);
+    extractedSignature = NULL;
 
     /* skip past the signature algorithm ID */
     if (fseeko(fpSrc, sizeof(uint32_t), SEEK_CUR)) {

taskcluster/ci/build/windows.yml

@@ -919,7 +919,7 @@ win32-mingwclang/opt:
         - linux64-sccache
         - linux64-cbindgen
         - linux64-node
-        - linux64-clang-7-mingw-x86
+        - linux64-clang-trunk-mingw-x86
         - linux64-mingw32-nsis
         - linux64-mingw32-fxc2
 
@@ -955,7 +955,7 @@ win32-mingwclang/debug:
         - linux64-sccache
         - linux64-cbindgen
         - linux64-node
-        - linux64-clang-7-mingw-x86
+        - linux64-clang-trunk-mingw-x86
         - linux64-mingw32-nsis
         - linux64-mingw32-fxc2
 
@@ -990,7 +990,7 @@ win64-mingwclang/opt:
         - linux64-sccache
         - linux64-cbindgen
         - linux64-node
-        - linux64-clang-7-mingw-x64
+        - linux64-clang-trunk-mingw-x64
         - linux64-mingw32-nsis
         - linux64-mingw32-fxc2
 
@@ -1026,6 +1026,6 @@ win64-mingwclang/debug:
         - linux64-sccache
         - linux64-cbindgen
         - linux64-node
-        - linux64-clang-7-mingw-x64
+        - linux64-clang-trunk-mingw-x64
         - linux64-mingw32-nsis
         - linux64-mingw32-fxc2

taskcluster/ci/release-mark-as-started/kind.yml

@@ -0,0 +1,51 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+loader: taskgraph.loader.transform:loader
+
+transforms:
+   - taskgraph.transforms.release_mark_as_started:transforms
+   - taskgraph.transforms.task:transforms
+
+job-defaults:
+   description: mark release as started in Ship-It v1
+   worker-type:
+      by-project:
+         mozilla-beta: scriptworker-prov-v1/shipit-v1
+         mozilla-release: scriptworker-prov-v1/shipit-v1
+         mozilla-esr60: scriptworker-prov-v1/shipit-v1
+         default: scriptworker-prov-v1/shipit-dev
+   worker:
+      implementation: shipit-started
+   scopes:
+      by-project:
+         mozilla-beta:
+            - project:releng:ship-it:server:production
+            - project:releng:ship-it:action:mark-as-started
+         mozilla-release:
+            - project:releng:ship-it:server:production
+            - project:releng:ship-it:action:mark-as-started
+         mozilla-esr60:
+            - project:releng:ship-it:server:production
+            - project:releng:ship-it:action:mark-as-started
+         default:
+            - project:releng:ship-it:server:staging
+            - project:releng:ship-it:action:mark-as-started
+   run-on-projects: []
+   shipping-phase: promote
+   locales-file: browser/locales/l10n-changesets.json
+
+jobs:
+   fennec:
+      name: release-fennec_mark_as_started
+      shipping-product: fennec
+      locales-file: mobile/locales/l10n-changesets.json
+
+   firefox:
+      name: release-firefox_mark_as_started
+      shipping-product: firefox
+
+   devedition:
+      name: release-devedition_mark_as_started
+      shipping-product: devedition

taskcluster/ci/toolchain/linux.yml

@@ -70,49 +70,49 @@ linux64-clang-7:
     toolchains:
         - linux64-gcc-4.9
 
-linux64-clang-7-mingw-x86:
-    description: "MinGW-Clang 7 x86 toolchain build"
+linux64-clang-trunk-mingw-x86:
+    description: "MinGW-Clang Trunk x86 toolchain build"
     treeherder:
         kind: build
         platform: toolchains/opt
-        symbol: TMW(clang7-32)
+        symbol: TMW(clang-x86)
         tier: 2
     worker-type: aws-provisioner-v1/gecko-{level}-b-linux-xlarge
     worker:
         max-run-time: 7200
     run:
         using: toolchain-script
-        script: build-clang-7-mingw.sh
+        script: build-clang-trunk-mingw.sh
         arguments: [
             'x86'
         ]
         resources:
             - 'build/build-clang/build-clang.py'
-            - 'build/build-clang/clang-7-mingw.json'
+            - 'build/build-clang/clang-trunk-mingw.json'
             - 'taskcluster/scripts/misc/tooltool-download.sh'
         toolchain-artifact: public/build/clangmingw.tar.xz
     toolchains:
         - linux64-gcc-4.9
 
-linux64-clang-7-mingw-x64:
-    description: "MinGW-Clang 7 Pre x64 toolchain build"
+linux64-clang-trunk-mingw-x64:
+    description: "MinGW-Clang Trunk x64 toolchain build"
     treeherder:
         kind: build
         platform: toolchains/opt
-        symbol: TMW(clang7-64)
+        symbol: TMW(clang-x64)
         tier: 2
     worker-type: aws-provisioner-v1/gecko-{level}-b-linux-xlarge
     worker:
         max-run-time: 7200
     run:
         using: toolchain-script
-        script: build-clang-7-mingw.sh
+        script: build-clang-trunk-mingw.sh
         arguments: [
             'x64'
         ]
         resources:
             - 'build/build-clang/build-clang.py'
-            - 'build/build-clang/clang-7-mingw.json'
+            - 'build/build-clang/clang-trunk-mingw.json'
             - 'taskcluster/scripts/misc/tooltool-download.sh'
         toolchain-artifact: public/build/clangmingw.tar.xz
     toolchains:

taskcluster/docs/kinds.rst

@@ -295,6 +295,10 @@ release-mark-as-shipped
 -----------------------
 Marks releases as shipped in Ship-It v1
 
+release-mark-as-started
+-----------------------
+Marks releases as started in Ship-It v1
+
 release-bouncer-aliases
 -----------------------
 Update Bouncer's (download.mozilla.org) "latest" aliases.

taskcluster/scripts/misc/build-clang-trunk-mingw.sh

@@ -30,7 +30,6 @@ INSTALL_DIR=$TOOLCHAIN_DIR/build/stage3/clang
 CROSS_PREFIX_DIR=$INSTALL_DIR/$machine-w64-mingw32
 SRC_DIR=$TOOLCHAIN_DIR/src
 
-CLANG_VERSION=7.0.0
 make_flags="-j$(nproc)"
 
 mingw_version=cfd85ebed773810429bf2164c3a985895b7dbfe3
@@ -137,6 +136,7 @@ build_mingw() {
 }
 
 build_compiler_rt() {
+  CLANG_VERSION=$(basename $(dirname $(dirname $(dirname $($CC --print-libgcc-file-name -rtlib=compiler-rt)))))
   mkdir compiler-rt
   pushd compiler-rt
   cmake \
@@ -292,7 +292,7 @@ set +x
 
 cd build/build-clang
 # |mach python| sets up a virtualenv for us!
-../../mach python ./build-clang.py -c clang-7-mingw.json --skip-tar
+../../mach python ./build-clang.py -c clang-trunk-mingw.json --skip-tar
 
 set -x
 

taskcluster/taskgraph/transforms/release_mark_as_started.py

@@ -0,0 +1,57 @@
+
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+"""
+Add from parameters.yml into Balrog publishing tasks.
+"""
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import json
+
+from taskgraph.transforms.base import TransformSequence
+from taskgraph.transforms.l10n import parse_locales_file
+from taskgraph.util.schema import resolve_keyed_by
+from taskgraph.util.scriptworker import get_release_config
+
+transforms = TransformSequence()
+
+
+@transforms.add
+def make_task_description(config, jobs):
+    release_config = get_release_config(config)
+    for job in jobs:
+        resolve_keyed_by(
+            job, 'worker-type', item_name=job['name'], project=config.params['project']
+        )
+        resolve_keyed_by(
+            job, 'scopes', item_name=job['name'], project=config.params['project']
+        )
+
+        job['worker']['release-name'] = '{product}-{version}-build{build_number}'.format(
+            product=job['shipping-product'].capitalize(),
+            version=release_config['version'],
+            build_number=release_config['build_number']
+        )
+        job['worker']['product'] = job['shipping-product']
+        branch = config.params['head_repository'].split('https://hg.mozilla.org/')[1]
+        job['worker']['branch'] = branch
+
+        # locales files has different structure between mobile and desktop
+        locales_file = job['locales-file']
+        all_locales = {}
+
+        if job['shipping-product'] == 'fennec':
+            with open(locales_file, mode='r') as f:
+                all_locales = json.dumps(json.load(f))
+        else:
+            all_locales = "\n".join([
+                "{} {}".format(locale, revision)
+                for locale, revision in sorted(parse_locales_file(job['locales-file']).items())
+            ])
+
+        job['worker']['locales'] = all_locales
+        del job['locales-file']
+
+        yield job

taskcluster/taskgraph/transforms/task.py

@@ -616,6 +616,12 @@ task_description_schema = Schema({
     }, {
         Required('implementation'): 'shipit-shipped',
         Required('release-name'): basestring,
+    }, {
+        Required('implementation'): 'shipit-started',
+        Required('release-name'): basestring,
+        Required('product'): basestring,
+        Required('branch'): basestring,
+        Required('locales'): basestring,
     }, {
         Required('implementation'): 'treescript',
         Required('tags'): [Any('buildN', 'release', None)],
@@ -1251,6 +1257,23 @@ def build_ship_it_shipped_payload(config, task, task_def):
     }
 
 
+@payload_builder('shipit-started')
+def build_ship_it_started_payload(config, task, task_def):
+    worker = task['worker']
+    release_config = get_release_config(config)
+
+    task_def['payload'] = {
+        'release_name': worker['release-name'],
+        'product': worker['product'],
+        'version': release_config['version'],
+        'build_number': release_config['build_number'],
+        'branch': worker['branch'],
+        'revision': get_branch_rev(config),
+        'partials': release_config.get('partial_versions', ""),
+        'l10n_changesets': worker['locales'],
+    }
+
+
 @payload_builder('sign-and-push-addons')
 def build_sign_and_push_addons_payload(config, task, task_def):
     worker = task['worker']

taskcluster/taskgraph/util/scriptworker.py

@@ -337,6 +337,7 @@ def get_release_config(config):
                                                  'release-secondary-update-verify-config',
                                                  'release-balrog-submit-toplevel',
                                                  'release-secondary-balrog-submit-toplevel',
+                                                 'release-mark-as-started'
                                                  ):
         partial_updates = json.loads(partial_updates)
         release_config['partial_versions'] = ', '.join([

toolkit/recordreplay/BufferStream.h

@@ -0,0 +1,68 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* vim: set ts=8 sts=2 et sw=2 tw=80: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef mozilla_recordreplay_BufferStream_h
+#define mozilla_recordreplay_BufferStream_h
+
+#include "InfallibleVector.h"
+
+namespace mozilla {
+namespace recordreplay {
+
+// BufferStream provides similar functionality to Stream in File.h, allowing
+// reading or writing to a stream of data backed by an in memory buffer instead
+// of data stored on disk.
+class BufferStream
+{
+  InfallibleVector<char>* mOutput;
+
+  const char* mInput;
+  size_t mInputSize;
+
+public:
+  BufferStream(const char* aInput, size_t aInputSize)
+    : mOutput(nullptr), mInput(aInput), mInputSize(aInputSize)
+  {}
+
+  explicit BufferStream(InfallibleVector<char>* aOutput)
+    : mOutput(aOutput), mInput(nullptr), mInputSize(0)
+  {}
+
+  void WriteBytes(const void* aData, size_t aSize) {
+    MOZ_RELEASE_ASSERT(mOutput);
+    mOutput->append((char*) aData, aSize);
+  }
+
+  void WriteScalar(size_t aValue) {
+    WriteBytes(&aValue, sizeof(aValue));
+  }
+
+  void ReadBytes(void* aData, size_t aSize) {
+    if (aSize) {
+      MOZ_RELEASE_ASSERT(mInput);
+      MOZ_RELEASE_ASSERT(aSize <= mInputSize);
+      memcpy(aData, mInput, aSize);
+      mInput += aSize;
+      mInputSize -= aSize;
+    }
+  }
+
+  size_t ReadScalar() {
+    size_t rv;
+    ReadBytes(&rv, sizeof(rv));
+    return rv;
+  }
+
+  bool IsEmpty() {
+    MOZ_RELEASE_ASSERT(mInput);
+    return mInputSize == 0;
+  }
+};
+
+} // namespace recordreplay
+} // namespace mozilla
+
+#endif // mozilla_recordreplay_BufferStream_h

toolkit/recordreplay/Callback.cpp

@@ -49,6 +49,9 @@ BeginCallback(size_t aCallbackId)
   }
   thread->SetPassThrough(false);
 
+  RecordingEventSection res(thread);
+  MOZ_RELEASE_ASSERT(res.CanAccessEvents());
+
   thread->Events().RecordOrReplayThreadEvent(ThreadEvent::ExecuteCallback);
   thread->Events().WriteScalar(aCallbackId);
 }
@@ -73,7 +76,8 @@ SaveOrRestoreCallbackData(void** aData)
   MOZ_RELEASE_ASSERT(gCallbackData);
 
   Thread* thread = Thread::Current();
-  MOZ_RELEASE_ASSERT(thread->CanAccessRecording());
+  RecordingEventSection res(thread);
+  MOZ_RELEASE_ASSERT(res.CanAccessEvents());
 
   thread->Events().RecordOrReplayThreadEvent(ThreadEvent::RestoreCallbackData);
 
@@ -102,7 +106,8 @@ void
 PassThroughThreadEventsAllowCallbacks(const std::function<void()>& aFn)
 {
   Thread* thread = Thread::Current();
-  MOZ_RELEASE_ASSERT(thread->CanAccessRecording());
+  RecordingEventSection res(thread);
+  MOZ_RELEASE_ASSERT(res.CanAccessEvents());
 
   if (IsRecording()) {
     if (thread->IsMainThread()) {

toolkit/recordreplay/File.cpp

@@ -47,13 +47,7 @@ Stream::ReadBytes(void* aData, size_t aSize)
     }
 
     MOZ_RELEASE_ASSERT(mBufferPos == mBufferLength);
-
-    // If we try to read off the end of a stream then we must have hit the end
-    // of the replay for this thread.
-    while (mChunkIndex == mChunks.length()) {
-      MOZ_RELEASE_ASSERT(mName == StreamName::Event);
-      HitEndOfRecording();
-    }
+    MOZ_RELEASE_ASSERT(mChunkIndex < mChunks.length());
 
     const StreamChunkLocation& chunk = mChunks[mChunkIndex++];
 
@@ -89,6 +83,7 @@ void
 Stream::WriteBytes(const void* aData, size_t aSize)
 {
   MOZ_RELEASE_ASSERT(mFile->OpenForWriting());
+  MOZ_RELEASE_ASSERT(mName != StreamName::Event || mInRecordingEventSection);
 
   // Prevent the entire file from being flushed while we write this data.
   AutoReadSpinLock streamLock(mFile->mStreamLock);

toolkit/recordreplay/File.h

@@ -60,10 +60,12 @@ enum class StreamName
 };
 
 class File;
+class RecordingEventSection;
 
 class Stream
 {
   friend class File;
+  friend class RecordingEventSection;
 
   // File this stream belongs to.
   File* mFile;
@@ -117,6 +119,9 @@ class Stream
   // flushed.
   size_t mFlushedChunks;
 
+  // Whether there is a RecordingEventSection instance active for this stream.
+  bool mInRecordingEventSection;
+
   Stream(File* aFile, StreamName aName, size_t aNameIndex)
     : mFile(aFile)
     , mName(aName)
@@ -133,6 +138,7 @@ class Stream
     , mLastEvent((ThreadEvent) 0)
     , mChunkIndex(0)
     , mFlushedChunks(0)
+    , mInRecordingEventSection(false)
   {}
 
 public:
@@ -204,6 +210,7 @@ public:
   };
 
   friend class Stream;
+  friend class RecordingEventSection;
 
 private:
   // Open file handle, or 0 if closed.

toolkit/recordreplay/Lock.cpp

@@ -6,8 +6,6 @@
 
 #include "Lock.h"
 
-#include "mozilla/StaticMutex.h"
-
 #include "ChunkAllocator.h"
 #include "InfallibleVector.h"
 #include "SpinLock.h"
@@ -69,13 +67,12 @@ static ReadWriteSpinLock gLocksLock;
 Lock::New(void* aNativeLock)
 {
   Thread* thread = Thread::Current();
-  if (!thread || thread->PassThroughEvents() || HasDivergedFromRecording()) {
+  RecordingEventSection res(thread);
+  if (!res.CanAccessEvents()) {
     Destroy(aNativeLock); // Clean up any old lock, as below.
     return;
   }
 
-  MOZ_RELEASE_ASSERT(thread->CanAccessRecording());
-
   thread->Events().RecordOrReplayThreadEvent(ThreadEvent::CreateLock);
 
   size_t id;
@@ -152,7 +149,11 @@ void
 Lock::Enter()
 {
   Thread* thread = Thread::Current();
-  MOZ_RELEASE_ASSERT(thread->CanAccessRecording());
+
+  RecordingEventSection res(thread);
+  if (!res.CanAccessEvents()) {
+    return;
+  }
 
   // Include an event in each thread's record when a lock acquire begins. This
   // is not required by the replay but is used to check that lock acquire order
@@ -165,8 +166,9 @@ Lock::Enter()
   if (IsRecording()) {
     acquires->mAcquires->WriteScalar(thread->Id());
   } else {
-    // Wait until this thread is next in line to acquire the lock.
-    while (thread->Id() != acquires->mNextOwner) {
+    // Wait until this thread is next in line to acquire the lock, or until it
+    // has been instructed to diverge from the recording.
+    while (thread->Id() != acquires->mNextOwner && !thread->MaybeDivergeFromRecording()) {
       Thread::Wait();
     }
   }
@@ -175,10 +177,11 @@ Lock::Enter()
 void
 Lock::Exit()
 {
-  if (IsReplaying()) {
+  Thread* thread = Thread::Current();
+  if (IsReplaying() && !thread->HasDivergedFromRecording()) {
     // Notify the next owner before releasing the lock.
     LockAcquires* acquires = gLockAcquires.Get(mId);
-    acquires->ReadAndNotifyNextOwner(Thread::Current());
+    acquires->ReadAndNotifyNextOwner(thread);
   }
 }
 
@@ -188,9 +191,7 @@ struct AtomicLock : public detail::MutexImpl
   using detail::MutexImpl::unlock;
 };
 
-// Lock which is held during code sections that run atomically. This is a
-// PRLock instead of an OffTheBooksMutex because the latter performs atomic
-// operations during initialization.
+// Lock which is held during code sections that run atomically.
 static AtomicLock* gAtomicLock = nullptr;
 
 /* static */ void

toolkit/recordreplay/MiddlemanCall.cpp

@@ -0,0 +1,426 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* vim: set ts=8 sts=2 et sw=2 tw=80: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "MiddlemanCall.h"
+
+#include <unordered_map>
+
+namespace mozilla {
+namespace recordreplay {
+
+// In a replaying or middleman process, all middleman calls that have been
+// encountered, indexed by their ID.
+static StaticInfallibleVector<MiddlemanCall*> gMiddlemanCalls;
+
+// In a replaying or middleman process, association between values produced by
+// a middleman call and the call itself.
+typedef std::unordered_map<const void*, MiddlemanCall*> MiddlemanCallMap;
+static MiddlemanCallMap* gMiddlemanCallMap;
+
+// In a middleman process, any buffers allocated for performed calls.
+static StaticInfallibleVector<void*> gAllocatedBuffers;
+
+// Lock protecting middleman call state.
+static Monitor* gMonitor;
+
+void
+InitializeMiddlemanCalls()
+{
+  MOZ_RELEASE_ASSERT(IsRecordingOrReplaying() || IsMiddleman());
+
+  gMiddlemanCallMap = new MiddlemanCallMap();
+  gMonitor = new Monitor();
+}
+
+// Apply the ReplayInput phase to aCall and any calls it depends on that have
+// not been sent to the middleman yet, filling aOutgoingCalls with the set of
+// such calls.
+static bool
+GatherDependentCalls(InfallibleVector<MiddlemanCall*>& aOutgoingCalls, MiddlemanCall* aCall)
+{
+  MOZ_RELEASE_ASSERT(!aCall->mSent);
+  aCall->mSent = true;
+
+  CallArguments arguments;
+  aCall->mArguments.CopyTo(&arguments);
+
+  InfallibleVector<MiddlemanCall*> dependentCalls;
+
+  MiddlemanCallContext cx(aCall, &arguments, MiddlemanCallPhase::ReplayInput);
+  cx.mDependentCalls = &dependentCalls;
+  gRedirections[aCall->mCallId].mMiddlemanCall(cx);
+  if (cx.mFailed) {
+    return false;
+  }
+
+  for (MiddlemanCall* dependent : dependentCalls) {
+    if (!dependent->mSent && !GatherDependentCalls(aOutgoingCalls, dependent)) {
+      return false;
+    }
+  }
+
+  aOutgoingCalls.append(aCall);
+  return true;
+}
+
+bool
+SendCallToMiddleman(size_t aCallId, CallArguments* aArguments, bool aDiverged)
+{
+  MOZ_RELEASE_ASSERT(IsReplaying());
+
+  const Redirection& redirection = gRedirections[aCallId];
+  MOZ_RELEASE_ASSERT(redirection.mMiddlemanCall);
+
+  Maybe<MonitorAutoLock> lock;
+  lock.emplace(*gMonitor);
+
+  // Allocate and fill in a new MiddlemanCall.
+  size_t id = gMiddlemanCalls.length();
+  MiddlemanCall* newCall = new MiddlemanCall();
+  gMiddlemanCalls.emplaceBack(newCall);
+  newCall->mId = id;
+  newCall->mCallId = aCallId;
+  newCall->mArguments.CopyFrom(aArguments);
+
+  // Perform the ReplayPreface phase on the new call.
+  {
+    MiddlemanCallContext cx(newCall, aArguments, MiddlemanCallPhase::ReplayPreface);
+    redirection.mMiddlemanCall(cx);
+    if (cx.mFailed) {
+      delete newCall;
+      gMiddlemanCalls.popBack();
+      return false;
+    }
+  }
+
+  // Other phases will not run if we have not diverged from the recording.
+  // Any outputs for the call have been handled by the SaveOutput hook.
+  if (!aDiverged) {
+    return true;
+  }
+
+  // Perform the ReplayInput phase on the new call and any others it depends on.
+  InfallibleVector<MiddlemanCall*> outgoingCalls;
+  if (!GatherDependentCalls(outgoingCalls, newCall)) {
+    for (MiddlemanCall* call : outgoingCalls) {
+      call->mSent = false;
+    }
+    return false;
+  }
+
+  // Encode all calls we are sending to the middleman.
+  InfallibleVector<char> inputData;
+  BufferStream inputStream(&inputData);
+  for (MiddlemanCall* call : outgoingCalls) {
+    call->EncodeInput(inputStream);
+  }
+
+  // Perform the calls synchronously in the middleman.
+  InfallibleVector<char> outputData;
+  if (!child::SendMiddlemanCallRequest(inputData.begin(), inputData.length(), &outputData)) {
+    // This thread is not allowed to perform middleman calls anymore. Release
+    // the lock and block until the process rewinds.
+    lock.reset();
+    Thread::WaitForever();
+  }
+
+  // Decode outputs for the calls just sent, and perform the ReplayOutput phase
+  // on any older dependent calls we sent.
+  BufferStream outputStream(outputData.begin(), outputData.length());
+  for (MiddlemanCall* call : outgoingCalls) {
+    call->DecodeOutput(outputStream);
+
+    if (call != newCall) {
+      CallArguments oldArguments;
+      call->mArguments.CopyTo(&oldArguments);
+      MiddlemanCallContext cx(call, &oldArguments, MiddlemanCallPhase::ReplayOutput);
+      cx.mReplayOutputIsOld = true;
+      gRedirections[call->mCallId].mMiddlemanCall(cx);
+    }
+  }
+
+  // Perform the ReplayOutput phase to fill in outputs for the current call.
+  newCall->mArguments.CopyTo(aArguments);
+  MiddlemanCallContext cx(newCall, aArguments, MiddlemanCallPhase::ReplayOutput);
+  redirection.mMiddlemanCall(cx);
+  return true;
+}
+
+void
+ProcessMiddlemanCall(const char* aInputData, size_t aInputSize,
+                     InfallibleVector<char>* aOutputData)
+{
+  MOZ_RELEASE_ASSERT(IsMiddleman());
+
+  BufferStream inputStream(aInputData, aInputSize);
+  BufferStream outputStream(aOutputData);
+
+  while (!inputStream.IsEmpty()) {
+    MiddlemanCall* call = new MiddlemanCall();
+    call->DecodeInput(inputStream);
+
+    const Redirection& redirection = gRedirections[call->mCallId];
+    MOZ_RELEASE_ASSERT(gRedirections[call->mCallId].mMiddlemanCall);
+
+    CallArguments arguments;
+    call->mArguments.CopyTo(&arguments);
+
+    {
+      MiddlemanCallContext cx(call, &arguments, MiddlemanCallPhase::MiddlemanInput);
+      redirection.mMiddlemanCall(cx);
+    }
+
+    RecordReplayInvokeCall(call->mCallId, &arguments);
+
+    {
+      MiddlemanCallContext cx(call, &arguments, MiddlemanCallPhase::MiddlemanOutput);
+      redirection.mMiddlemanCall(cx);
+    }
+
+    call->mArguments.CopyFrom(&arguments);
+    call->EncodeOutput(outputStream);
+
+    while (call->mId >= gMiddlemanCalls.length()) {
+      gMiddlemanCalls.emplaceBack(nullptr);
+    }
+    MOZ_RELEASE_ASSERT(!gMiddlemanCalls[call->mId]);
+    gMiddlemanCalls[call->mId] = call;
+  }
+}
+
+void*
+MiddlemanCallContext::AllocateBytes(size_t aSize)
+{
+  void* rv = malloc(aSize);
+
+  // In a middleman process, any buffers we allocate live until the calls are
+  // reset. In a replaying process, the buffers will either live forever
+  // (if they are allocated in the ReplayPreface phase, to match the lifetime
+  // of the MiddlemanCall itself) or will be recovered when we rewind after we
+  // are done with our divergence from the recording (any other phase).
+  if (IsMiddleman()) {
+    gAllocatedBuffers.append(rv);
+  }
+
+  return rv;
+}
+
+void
+ResetMiddlemanCalls()
+{
+  MOZ_RELEASE_ASSERT(IsMiddleman());
+
+  for (MiddlemanCall* call : gMiddlemanCalls) {
+    if (call) {
+      CallArguments arguments;
+      call->mArguments.CopyTo(&arguments);
+
+      MiddlemanCallContext cx(call, &arguments, MiddlemanCallPhase::MiddlemanRelease);
+      gRedirections[call->mCallId].mMiddlemanCall(cx);
+
+      delete call;
+    }
+  }
+
+  gMiddlemanCalls.clear();
+  for (auto buffer : gAllocatedBuffers) {
+    free(buffer);
+  }
+  gAllocatedBuffers.clear();
+}
+
+///////////////////////////////////////////////////////////////////////////////
+// System Values
+///////////////////////////////////////////////////////////////////////////////
+
+static void
+AddMiddlemanCallValue(const void* aThing, MiddlemanCall* aCall)
+{
+  gMiddlemanCallMap->erase(aThing);
+  gMiddlemanCallMap->insert(MiddlemanCallMap::value_type(aThing, aCall));
+}
+
+static MiddlemanCall*
+LookupMiddlemanCall(const void* aThing)
+{
+  MiddlemanCallMap::const_iterator iter = gMiddlemanCallMap->find(aThing);
+  if (iter != gMiddlemanCallMap->end()) {
+    return iter->second;
+  }
+  return nullptr;
+}
+
+static const void*
+GetMiddlemanCallValue(size_t aId)
+{
+  MOZ_RELEASE_ASSERT(IsMiddleman());
+  MOZ_RELEASE_ASSERT(aId < gMiddlemanCalls.length() &&
+                     gMiddlemanCalls[aId] &&
+                     gMiddlemanCalls[aId]->mMiddlemanValue.isSome());
+  return gMiddlemanCalls[aId]->mMiddlemanValue.ref();
+}
+
+bool
+Middleman_SystemInput(MiddlemanCallContext& aCx, const void** aThingPtr)
+{
+  MOZ_RELEASE_ASSERT(aCx.AccessPreface());
+
+  if (!*aThingPtr) {
+    // Null values are handled by the normal argument copying logic.
+    return true;
+  }
+
+  Maybe<size_t> callId;
+  if (aCx.mPhase == MiddlemanCallPhase::ReplayPreface) {
+    // Determine any middleman call this object came from, before the pointer
+    // has a chance to be clobbered by another call between this and the
+    // ReplayInput phase.
+    MiddlemanCall* call = LookupMiddlemanCall(*aThingPtr);
+    if (call) {
+      callId.emplace(call->mId);
+    }
+  }
+  aCx.ReadOrWritePrefaceBytes(&callId, sizeof(callId));
+
+  switch (aCx.mPhase) {
+  case MiddlemanCallPhase::ReplayPreface:
+    return true;
+  case MiddlemanCallPhase::ReplayInput:
+    if (callId.isSome()) {
+      aCx.WriteInputScalar(callId.ref());
+      aCx.mDependentCalls->append(gMiddlemanCalls[callId.ref()]);
+      return true;
+    }
+    return false;
+  case MiddlemanCallPhase::MiddlemanInput:
+    if (callId.isSome()) {
+      size_t callIndex = aCx.ReadInputScalar();
+      *aThingPtr = GetMiddlemanCallValue(callIndex);
+      return true;
+    }
+    return false;
+  default:
+    MOZ_CRASH("Bad phase");
+  }
+}
+
+// Pointer system values are preserved during the replay so that null tests
+// and equality tests work as expected. We additionally mangle the
+// pointers here by setting one of the two highest bits, depending on whether
+// the pointer came from the recording or from the middleman. This avoids
+// accidentally conflating pointers that happen to have the same value but
+// which originate from different processes.
+static const void*
+MangleSystemValue(const void* aValue, bool aFromRecording)
+{
+  return (const void*) ((size_t)aValue | (1ULL << (aFromRecording ? 63 : 62)));
+}
+
+void
+Middleman_SystemOutput(MiddlemanCallContext& aCx, const void** aOutput, bool aUpdating)
+{
+  if (!*aOutput) {
+    return;
+  }
+
+  switch (aCx.mPhase) {
+  case MiddlemanCallPhase::ReplayPreface:
+    if (!HasDivergedFromRecording()) {
+      // If we haven't diverged from the recording, use the output value saved
+      // in the recording.
+      if (!aUpdating) {
+        *aOutput = MangleSystemValue(*aOutput, true);
+      }
+      aCx.mCall->SetRecordingValue(*aOutput);
+      AddMiddlemanCallValue(*aOutput, aCx.mCall);
+    }
+    break;
+  case MiddlemanCallPhase::MiddlemanOutput:
+    aCx.mCall->SetMiddlemanValue(*aOutput);
+    AddMiddlemanCallValue(*aOutput, aCx.mCall);
+    break;
+  case MiddlemanCallPhase::ReplayOutput: {
+    if (!aUpdating) {
+      *aOutput = MangleSystemValue(*aOutput, false);
+    }
+    aCx.mCall->SetMiddlemanValue(*aOutput);
+
+    // Associate the value produced by the middleman with this call. If the
+    // call previously went through the ReplayPreface phase when we did not
+    // diverge from the recording, we will associate values from both the
+    // recording and middleman processes with this call. If a call made after
+    // diverging produced the same value as a call made before diverging, use
+    // the value saved in the recording for the first call, so that equality
+    // tests on the value work as expected.
+    MiddlemanCall* previousCall = LookupMiddlemanCall(*aOutput);
+    if (previousCall) {
+      if (previousCall->mRecordingValue.isSome()) {
+        *aOutput = previousCall->mRecordingValue.ref();
+      }
+    } else {
+      AddMiddlemanCallValue(*aOutput, aCx.mCall);
+    }
+    break;
+  }
+  default:
+    return;
+  }
+}
+
+///////////////////////////////////////////////////////////////////////////////
+// MiddlemanCall
+///////////////////////////////////////////////////////////////////////////////
+
+void
+MiddlemanCall::EncodeInput(BufferStream& aStream) const
+{
+  aStream.WriteScalar(mId);
+  aStream.WriteScalar(mCallId);
+  aStream.WriteBytes(&mArguments, sizeof(CallRegisterArguments));
+  aStream.WriteScalar(mPreface.length());
+  aStream.WriteBytes(mPreface.begin(), mPreface.length());
+  aStream.WriteScalar(mInput.length());
+  aStream.WriteBytes(mInput.begin(), mInput.length());
+}
+
+void
+MiddlemanCall::DecodeInput(BufferStream& aStream)
+{
+  mId = aStream.ReadScalar();
+  mCallId = aStream.ReadScalar();
+  aStream.ReadBytes(&mArguments, sizeof(CallRegisterArguments));
+  size_t prefaceLength = aStream.ReadScalar();
+  mPreface.appendN(0, prefaceLength);
+  aStream.ReadBytes(mPreface.begin(), prefaceLength);
+  size_t inputLength = aStream.ReadScalar();
+  mInput.appendN(0, inputLength);
+  aStream.ReadBytes(mInput.begin(), inputLength);
+}
+
+void
+MiddlemanCall::EncodeOutput(BufferStream& aStream) const
+{
+  aStream.WriteBytes(&mArguments, sizeof(CallRegisterArguments));
+  aStream.WriteScalar(mOutput.length());
+  aStream.WriteBytes(mOutput.begin(), mOutput.length());
+}
+
+void
+MiddlemanCall::DecodeOutput(BufferStream& aStream)
+{
+  // Only update the return value when decoding arguments, so that we don't
+  // clobber the call's arguments with any changes made in the middleman.
+  CallRegisterArguments newArguments;
+  aStream.ReadBytes(&newArguments, sizeof(CallRegisterArguments));
+  mArguments.CopyRvalFrom(&newArguments);
+
+  size_t outputLength = aStream.ReadScalar();
+  mOutput.appendN(0, outputLength);
+  aStream.ReadBytes(mOutput.begin(), outputLength);
+}
+
+} // namespace recordreplay
+} // namespace mozilla

toolkit/recordreplay/MiddlemanCall.h

@@ -0,0 +1,464 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* vim: set ts=8 sts=2 et sw=2 tw=80: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef mozilla_recordreplay_MiddlemanCall_h
+#define mozilla_recordreplay_MiddlemanCall_h
+
+#include "BufferStream.h"
+#include "ProcessRedirect.h"
+#include "mozilla/Maybe.h"
+
+namespace mozilla {
+namespace recordreplay {
+
+// Middleman Calls Overview
+//
+// With few exceptions, replaying processes do not interact with the underlying
+// system or call the actual versions of redirected system library functions.
+// This is problematic after diverging from the recording, as then the diverged
+// thread cannot interact with its recording either.
+//
+// Middleman calls are used in a replaying process after diverging from the
+// recording to perform calls in the middleman process instead. Inputs are
+// gathered and serialized in the replaying process, then sent to the middleman
+// process. The middleman calls the function, and its outputs are serialized
+// for reading by the replaying process.
+//
+// Calls that might need to be sent to the middleman are processed in phases,
+// per the MiddlemanCallPhase enum below. The timeline of a middleman call is
+// as follows:
+//
+// - Any redirection with a middleman call hook can potentially be sent to the
+//   middleman. In a replaying process, whenever such a call is encountered,
+//   the hook is invoked in the ReplayPreface phase to capture any input data
+//   that must be examined at the time of the call itself.
+//
+// - If the thread has not diverged from the recording, the call is remembered
+//   but no further action is necessary yet.
+//
+// - If the thread has diverged from the recording, the call needs to go
+//   through the remaining phases. The ReplayInput phase captures any
+//   additional inputs to the call, potentially including values produced by
+//   other middleman calls.
+//
+// - The transitive closure of these call dependencies is produced, and all
+//   calls found go through the ReplayInput phase. The resulting data is sent
+//   to the middleman process, which goes through the MiddlemanInput phase
+//   to decode those inputs.
+//
+// - The middleman performs each of the calls it has been given, and their
+//   outputs are encoded in the MiddlemanOutput phase. These outputs are sent
+//   to the replaying process in a response and decoded in the ReplayOutput
+//   phase, which can then resume execution.
+//
+// - The replaying process holds onto information about calls it has sent until
+//   it rewinds to a point before it diverged from the recording. This rewind
+//   will --- without any special action required --- wipe out information on
+//   all calls sent to the middleman, and retain any data gathered in the
+//   ReplayPreface phase for calls that were made prior to the rewind target.
+//
+// - Information about calls and all resources held are retained in the
+//   middleman process are retained until a replaying process asks for them to
+//   be reset, which happens any time the replaying process first diverges from
+//   the recording. The MiddlemanRelease phase is used to release any system
+//   resources held.
+
+// Ways of processing calls that can be sent to the middleman.
+enum class MiddlemanCallPhase
+{
+  // When replaying, a call is being performed that might need to be sent to
+  // the middleman later.
+  ReplayPreface,
+
+  // A call for which inputs have been gathered is now being sent to the
+  // middleman. This is separate from ReplayPreface because capturing inputs
+  // might need to dereference pointers that could be bogus values originating
+  // from the recording. Waiting to dereference these pointers until we know
+  // the call needs to be sent to the middleman avoids needing to understand
+  // the inputs to all call sites of general purpose redirections such as
+  // CFArrayCreate.
+  ReplayInput,
+
+  // In the middleman process, a call from the replaying process is being
+  // performed.
+  MiddlemanInput,
+
+  // In the middleman process, a call from the replaying process was just
+  // performed, and its outputs need to be saved.
+  MiddlemanOutput,
+
+  // Back in the replaying process, the outputs from a call have been received
+  // from the middleman.
+  ReplayOutput,
+
+  // In the middleman process, release any system resources held after this
+  // call.
+  MiddlemanRelease,
+};
+
+struct MiddlemanCall
+{
+  // Unique ID for this call.
+  size_t mId;
+
+  // ID of the redirection being invoked.
+  size_t mCallId;
+
+  // All register arguments and return values are preserved when sending the
+  // call back and forth between processes.
+  CallRegisterArguments mArguments;
+
+  // Written in ReplayPrefaceInput, read in ReplayInput and MiddlemanInput.
+  InfallibleVector<char> mPreface;
+
+  // Written in ReplayInput, read in MiddlemanInput.
+  InfallibleVector<char> mInput;
+
+  // Written in MiddlemanOutput, read in ReplayOutput.
+  InfallibleVector<char> mOutput;
+
+  // In a replaying process, whether this call has been sent to the middleman.
+  bool mSent;
+
+  // In a replaying process, any value associated with this call that was
+  // included in the recording, when the call was made before diverging from
+  // the recording.
+  Maybe<const void*> mRecordingValue;
+
+  // In a replaying or middleman process, any value associated with this call
+  // that was produced by the middleman itself.
+  Maybe<const void*> mMiddlemanValue;
+
+  MiddlemanCall()
+    : mId(0), mCallId(0), mSent(false)
+  {}
+
+  void EncodeInput(BufferStream& aStream) const;
+  void DecodeInput(BufferStream& aStream);
+
+  void EncodeOutput(BufferStream& aStream) const;
+  void DecodeOutput(BufferStream& aStream);
+
+  void SetRecordingValue(const void* aValue) {
+    MOZ_RELEASE_ASSERT(mRecordingValue.isNothing());
+    mRecordingValue.emplace(aValue);
+  }
+
+  void SetMiddlemanValue(const void* aValue) {
+    MOZ_RELEASE_ASSERT(mMiddlemanValue.isNothing());
+    mMiddlemanValue.emplace(aValue);
+  }
+};
+
+// Information needed to process one of the phases of a middleman call,
+// in either the replaying or middleman process.
+struct MiddlemanCallContext
+{
+  // Call being operated on.
+  MiddlemanCall* mCall;
+
+  // Complete arguments and return value information for the call.
+  CallArguments* mArguments;
+
+  // Current processing phase.
+  MiddlemanCallPhase mPhase;
+
+  // During the ReplayPreface or ReplayInput phases, whether capturing input
+  // data has failed. In such cases the call cannot be sent to the middleman
+  // and, if the thread has diverged from the recording, an unhandled
+  // divergence and associated rewind will occur.
+  bool mFailed;
+
+  // During the ReplayInput phase, this can be used to fill in any middleman
+  // calls whose output the current one depends on.
+  InfallibleVector<MiddlemanCall*>* mDependentCalls;
+
+  // Streams of data that can be accessed during the various phases. Streams
+  // need to be read or written from at the same points in the phases which use
+  // them, so that callbacks operating on these streams can be composed without
+  // issues.
+
+  // The preface is written during ReplayPreface, and read during both
+  // ReplayInput and MiddlemanInput.
+  Maybe<BufferStream> mPrefaceStream;
+
+  // Inputs are written during ReplayInput, and read during MiddlemanInput.
+  Maybe<BufferStream> mInputStream;
+
+  // Outputs are written during MiddlemanOutput, and read during ReplayOutput.
+  Maybe<BufferStream> mOutputStream;
+
+  // During the ReplayOutput phase, this is set if the call was made sometime
+  // in the past and pointers referred to in the arguments may no longer be
+  // valid.
+  bool mReplayOutputIsOld;
+
+  MiddlemanCallContext(MiddlemanCall* aCall, CallArguments* aArguments, MiddlemanCallPhase aPhase)
+    : mCall(aCall), mArguments(aArguments), mPhase(aPhase),
+      mFailed(false), mDependentCalls(nullptr), mReplayOutputIsOld(false)
+  {
+    switch (mPhase) {
+    case MiddlemanCallPhase::ReplayPreface:
+      mPrefaceStream.emplace(&mCall->mPreface);
+      break;
+    case MiddlemanCallPhase::ReplayInput:
+      mPrefaceStream.emplace(mCall->mPreface.begin(), mCall->mPreface.length());
+      mInputStream.emplace(&mCall->mInput);
+      break;
+    case MiddlemanCallPhase::MiddlemanInput:
+      mPrefaceStream.emplace(mCall->mPreface.begin(), mCall->mPreface.length());
+      mInputStream.emplace(mCall->mInput.begin(), mCall->mInput.length());
+      break;
+    case MiddlemanCallPhase::MiddlemanOutput:
+      mOutputStream.emplace(&mCall->mOutput);
+      break;
+    case MiddlemanCallPhase::ReplayOutput:
+      mOutputStream.emplace(mCall->mOutput.begin(), mCall->mOutput.length());
+      break;
+    case MiddlemanCallPhase::MiddlemanRelease:
+      break;
+    }
+  }
+
+  void MarkAsFailed() {
+    MOZ_RELEASE_ASSERT(mPhase == MiddlemanCallPhase::ReplayPreface ||
+                       mPhase == MiddlemanCallPhase::ReplayInput);
+    mFailed = true;
+  }
+
+  void WriteInputBytes(const void* aBuffer, size_t aSize) {
+    MOZ_RELEASE_ASSERT(mPhase == MiddlemanCallPhase::ReplayInput);
+    mInputStream.ref().WriteBytes(aBuffer, aSize);
+  }
+
+  void WriteInputScalar(size_t aValue) {
+    MOZ_RELEASE_ASSERT(mPhase == MiddlemanCallPhase::ReplayInput);
+    mInputStream.ref().WriteScalar(aValue);
+  }
+
+  void ReadInputBytes(void* aBuffer, size_t aSize) {
+    MOZ_RELEASE_ASSERT(mPhase == MiddlemanCallPhase::MiddlemanInput);
+    mInputStream.ref().ReadBytes(aBuffer, aSize);
+  }
+
+  size_t ReadInputScalar() {
+    MOZ_RELEASE_ASSERT(mPhase == MiddlemanCallPhase::MiddlemanInput);
+    return mInputStream.ref().ReadScalar();
+  }
+
+  bool AccessInput() {
+    return mInputStream.isSome();
+  }
+
+  void ReadOrWriteInputBytes(void* aBuffer, size_t aSize) {
+    switch (mPhase) {
+    case MiddlemanCallPhase::ReplayInput:
+      WriteInputBytes(aBuffer, aSize);
+      break;
+    case MiddlemanCallPhase::MiddlemanInput:
+      ReadInputBytes(aBuffer, aSize);
+      break;
+    default:
+      MOZ_CRASH();
+    }
+  }
+
+  bool AccessPreface() {
+    return mPrefaceStream.isSome();
+  }
+
+  void ReadOrWritePrefaceBytes(void* aBuffer, size_t aSize) {
+    switch (mPhase) {
+    case MiddlemanCallPhase::ReplayPreface:
+      mPrefaceStream.ref().WriteBytes(aBuffer, aSize);
+      break;
+    case MiddlemanCallPhase::ReplayInput:
+    case MiddlemanCallPhase::MiddlemanInput:
+      mPrefaceStream.ref().ReadBytes(aBuffer, aSize);
+      break;
+    default:
+      MOZ_CRASH();
+    }
+  }
+
+  void ReadOrWritePrefaceBuffer(void** aBufferPtr, size_t aSize) {
+    switch (mPhase) {
+    case MiddlemanCallPhase::ReplayPreface:
+      mPrefaceStream.ref().WriteBytes(*aBufferPtr, aSize);
+      break;
+    case MiddlemanCallPhase::ReplayInput:
+    case MiddlemanCallPhase::MiddlemanInput:
+      *aBufferPtr = AllocateBytes(aSize);
+      mPrefaceStream.ref().ReadBytes(*aBufferPtr, aSize);
+      break;
+    default:
+      MOZ_CRASH();
+    }
+  }
+
+  bool AccessOutput() {
+    return mOutputStream.isSome();
+  }
+
+  void ReadOrWriteOutputBytes(void* aBuffer, size_t aSize) {
+    switch (mPhase) {
+    case MiddlemanCallPhase::MiddlemanOutput:
+      mOutputStream.ref().WriteBytes(aBuffer, aSize);
+      break;
+    case MiddlemanCallPhase::ReplayOutput:
+      mOutputStream.ref().ReadBytes(aBuffer, aSize);
+      break;
+    default:
+      MOZ_CRASH();
+    }
+  }
+
+  void ReadOrWriteOutputBuffer(void** aBuffer, size_t aSize) {
+    if (*aBuffer) {
+      if (mPhase == MiddlemanCallPhase::MiddlemanInput || mReplayOutputIsOld) {
+        *aBuffer = AllocateBytes(aSize);
+      }
+
+      if (AccessOutput()) {
+        ReadOrWriteOutputBytes(*aBuffer, aSize);
+      }
+    }
+  }
+
+  // Allocate some memory associated with the call, which will be released in
+  // the replaying process on a rewind and in the middleman process when the
+  // call state is reset.
+  void* AllocateBytes(size_t aSize);
+};
+
+// Notify the system about a call to a redirection with a middleman call hook.
+// aDiverged is set if the current thread has diverged from the recording and
+// any outputs for the call must be filled in; otherwise, they already have
+// been filled in using data from the recording. Returns false if the call was
+// unable to be processed.
+bool SendCallToMiddleman(size_t aCallId, CallArguments* aArguments, bool aDiverged);
+
+// In the middleman process, perform one or more calls encoded in aInputData
+// and encode their outputs to aOutputData.
+void ProcessMiddlemanCall(const char* aInputData, size_t aInputSize,
+                          InfallibleVector<char>* aOutputData);
+
+// In the middleman process, reset all call state.
+void ResetMiddlemanCalls();
+
+///////////////////////////////////////////////////////////////////////////////
+// Middleman Call Helpers
+///////////////////////////////////////////////////////////////////////////////
+
+// Capture the contents of an input buffer at BufferArg with element count at CountArg.
+template <size_t BufferArg, size_t CountArg, typename ElemType>
+static inline void
+Middleman_Buffer(MiddlemanCallContext& aCx)
+{
+  if (aCx.AccessPreface()) {
+    auto& buffer = aCx.mArguments->Arg<BufferArg, void*>();
+    auto byteSize = aCx.mArguments->Arg<CountArg, size_t>() * sizeof(ElemType);
+    aCx.ReadOrWritePrefaceBuffer(&buffer, byteSize);
+  }
+}
+
+// Capture the contents of a fixed size input buffer.
+template <size_t BufferArg, size_t ByteSize>
+static inline void
+Middleman_BufferFixedSize(MiddlemanCallContext& aCx)
+{
+  if (aCx.AccessPreface()) {
+    auto& buffer = aCx.mArguments->Arg<BufferArg, void*>();
+    if (buffer) {
+      aCx.ReadOrWritePrefaceBuffer(&buffer, ByteSize);
+    }
+  }
+}
+
+// Capture a C string argument.
+template <size_t StringArg>
+static inline void
+Middleman_CString(MiddlemanCallContext& aCx)
+{
+  if (aCx.AccessPreface()) {
+    auto& buffer = aCx.mArguments->Arg<StringArg, char*>();
+    size_t len = (aCx.mPhase == MiddlemanCallPhase::ReplayPreface) ? strlen(buffer) + 1 : 0;
+    aCx.ReadOrWritePrefaceBytes(&len, sizeof(len));
+    aCx.ReadOrWritePrefaceBuffer((void**) &buffer, len);
+  }
+}
+
+// Capture the data written to an output buffer at BufferArg with element count at CountArg.
+template <size_t BufferArg, size_t CountArg, typename ElemType>
+static inline void
+Middleman_WriteBuffer(MiddlemanCallContext& aCx)
+{
+  auto& buffer = aCx.mArguments->Arg<BufferArg, void*>();
+  auto count = aCx.mArguments->Arg<CountArg, size_t>();
+  aCx.ReadOrWriteOutputBuffer(&buffer, count * sizeof(ElemType));
+}
+
+// Capture the data written to a fixed size output buffer.
+template <size_t BufferArg, size_t ByteSize>
+static inline void
+Middleman_WriteBufferFixedSize(MiddlemanCallContext& aCx)
+{
+  auto& buffer = aCx.mArguments->Arg<BufferArg, void*>();
+  aCx.ReadOrWriteOutputBuffer(&buffer, ByteSize);
+}
+
+// Capture return values that are too large for register storage.
+template <size_t ByteSize>
+static inline void
+Middleman_OversizeRval(MiddlemanCallContext& aCx)
+{
+  Middleman_WriteBufferFixedSize<0, ByteSize>(aCx);
+}
+
+// Capture a byte count of stack argument data.
+template <size_t ByteSize>
+static inline void
+Middleman_StackArgumentData(MiddlemanCallContext& aCx)
+{
+  if (aCx.AccessPreface()) {
+    auto stack = aCx.mArguments->StackAddress<0>();
+    aCx.ReadOrWritePrefaceBytes(stack, ByteSize);
+  }
+}
+
+static inline void
+Middleman_NoOp(MiddlemanCallContext& aCx)
+{
+}
+
+template <MiddlemanCallFn Fn0,
+          MiddlemanCallFn Fn1,
+          MiddlemanCallFn Fn2 = Middleman_NoOp,
+          MiddlemanCallFn Fn3 = Middleman_NoOp,
+          MiddlemanCallFn Fn4 = Middleman_NoOp>
+static inline void
+Middleman_Compose(MiddlemanCallContext& aCx)
+{
+  Fn0(aCx);
+  Fn1(aCx);
+  Fn2(aCx);
+  Fn3(aCx);
+  Fn4(aCx);
+}
+
+// Helper for capturing inputs that are produced by other middleman calls.
+// Returns false in the ReplayInput or MiddlemanInput phases if the input
+// system value could not be found.
+bool Middleman_SystemInput(MiddlemanCallContext& aCx, const void** aThingPtr);
+
+// Helper for capturing output system values that might be consumed by other
+// middleman calls.
+void Middleman_SystemOutput(MiddlemanCallContext& aCx, const void** aOutput, bool aUpdating = false);
+
+} // namespace recordreplay
+} // namespace mozilla
+
+#endif // mozilla_recordreplay_MiddlemanCall_h

toolkit/recordreplay/ProcessRecordReplay.cpp

@@ -51,6 +51,9 @@ char* gRecordingFilename;
 // Current process ID.
 static int gPid;
 
+// ID of the process which produced the recording.
+static int gRecordingPid;
+
 // Whether to spew record/replay messages to stderr.
 static bool gSpewEnabled;
 
@@ -111,6 +114,7 @@ RecordReplayInterface_Initialize(int aArgc, char* aArgv[])
   EarlyInitializeRedirections();
 
   if (!IsRecordingOrReplaying()) {
+    InitializeMiddlemanCalls();
     return;
   }
 
@@ -144,6 +148,7 @@ RecordReplayInterface_Initialize(int aArgc, char* aArgv[])
   Thread::SpawnAllThreads();
   InitializeCountdownThread();
   SetupDirtyMemoryHandler();
+  InitializeMiddlemanCalls();
 
   // Don't create a stylo thread pool when recording or replaying.
   putenv((char*) "STYLO_THREADS=1");
@@ -152,6 +157,7 @@ RecordReplayInterface_Initialize(int aArgc, char* aArgv[])
 
   Lock::InitializeLocks();
   InitializeRewindState();
+  gRecordingPid = RecordReplayValue(gPid);
 
   gInitialized = true;
 }
@@ -160,12 +166,11 @@ MOZ_EXPORT size_t
 RecordReplayInterface_InternalRecordReplayValue(size_t aValue)
 {
   Thread* thread = Thread::Current();
-  if (thread->PassThroughEvents()) {
+  RecordingEventSection res(thread);
+  if (!res.CanAccessEvents()) {
     return aValue;
   }
-  EnsureNotDivergedFromRecording();
 
-  MOZ_RELEASE_ASSERT(thread->CanAccessRecording());
   thread->Events().RecordOrReplayThreadEvent(ThreadEvent::Value);
   thread->Events().RecordOrReplayValue(&aValue);
   return aValue;
@@ -175,12 +180,11 @@ MOZ_EXPORT void
 RecordReplayInterface_InternalRecordReplayBytes(void* aData, size_t aSize)
 {
   Thread* thread = Thread::Current();
-  if (thread->PassThroughEvents()) {
+  RecordingEventSection res(thread);
+  if (!res.CanAccessEvents()) {
     return;
   }
-  EnsureNotDivergedFromRecording();
 
-  MOZ_RELEASE_ASSERT(thread->CanAccessRecording());
   thread->Events().RecordOrReplayThreadEvent(ThreadEvent::Bytes);
   thread->Events().CheckInput(aSize);
   thread->Events().RecordOrReplayBytes(aData, aSize);
@@ -325,6 +329,12 @@ ThreadEventName(ThreadEvent aEvent)
   return gRedirections[callId].mName;
 }
 
+int
+GetRecordingPid()
+{
+  return gRecordingPid;
+}
+
 ///////////////////////////////////////////////////////////////////////////////
 // Record/Replay Assertions
 ///////////////////////////////////////////////////////////////////////////////
@@ -335,10 +345,10 @@ MOZ_EXPORT void
 RecordReplayInterface_InternalRecordReplayAssert(const char* aFormat, va_list aArgs)
 {
   Thread* thread = Thread::Current();
-  if (thread->PassThroughEvents() || thread->HasDivergedFromRecording()) {
+  RecordingEventSection res(thread);
+  if (!res.CanAccessEvents()) {
     return;
   }
-  MOZ_RELEASE_ASSERT(thread->CanAccessRecording());
 
   // Add the asserted string to the recording.
   char text[1024];
@@ -352,10 +362,10 @@ MOZ_EXPORT void
 RecordReplayInterface_InternalRecordReplayAssertBytes(const void* aData, size_t aSize)
 {
   Thread* thread = Thread::Current();
-  if (thread->PassThroughEvents() || thread->HasDivergedFromRecording()) {
+  RecordingEventSection res(thread);
+  if (!res.CanAccessEvents()) {
     return;
   }
-  MOZ_RELEASE_ASSERT(thread->CanAccessRecording());
 
   thread->Events().RecordOrReplayThreadEvent(ThreadEvent::AssertBytes);
   thread->Events().CheckInput(aData, aSize);

toolkit/recordreplay/ProcessRecordReplay.h

@@ -232,6 +232,9 @@ MOZ_MakeRecordReplayPrinter(PrintSpew, true)
 
 #undef MOZ_MakeRecordReplayPrinter
 
+// Get the ID of the process that produced the recording.
+int GetRecordingPid();
+
 ///////////////////////////////////////////////////////////////////////////////
 // Profiling
 ///////////////////////////////////////////////////////////////////////////////

toolkit/recordreplay/ProcessRedirect.cpp

@@ -7,6 +7,7 @@
 #include "ProcessRedirect.h"
 
 #include "InfallibleVector.h"
+#include "MiddlemanCall.h"
 #include "mozilla/Sprintf.h"
 
 #include <dlfcn.h>
@@ -27,6 +28,24 @@ namespace recordreplay {
 // Redirection Skeleton
 ///////////////////////////////////////////////////////////////////////////////
 
+static bool
+CallPreambleHook(PreambleFn aPreamble, size_t aCallId, CallArguments* aArguments)
+{
+  PreambleResult result = aPreamble(aArguments);
+  switch (result) {
+  case PreambleResult::Veto:
+    return true;
+  case PreambleResult::PassThrough: {
+    AutoEnsurePassThroughThreadEvents pt;
+    RecordReplayInvokeCall(aCallId, aArguments);
+    return true;
+  }
+  case PreambleResult::Redirect:
+    return false;
+  }
+  Unreachable();
+}
+
 extern "C" {
 
 __attribute__((used)) int
@@ -37,42 +56,64 @@ RecordReplayInterceptCall(int aCallId, CallArguments* aArguments)
   // Call the preamble to see if this call needs special handling, even when
   // events have been passed through.
   if (redirection.mPreamble) {
-    PreambleResult result = redirection.mPreamble(aArguments);
-    switch (result) {
-    case PreambleResult::Veto:
+    if (CallPreambleHook(redirection.mPreamble, aCallId, aArguments)) {
       return 0;
-    case PreambleResult::PassThrough: {
-      AutoEnsurePassThroughThreadEvents pt;
-      RecordReplayInvokeCall(aCallId, aArguments);
-      return 0;
-    }
-    case PreambleResult::Redirect:
-      break;
     }
   }
 
   Thread* thread = Thread::Current();
+  Maybe<RecordingEventSection> res;
+  res.emplace(thread);
 
-  // When events are passed through, invoke the call with the original stack
-  // and register state.
-  if (!thread || thread->PassThroughEvents()) {
-    // RecordReplayRedirectCall will load the function to call from the
-    // return value slot.
-    aArguments->Rval<uint8_t*>() = redirection.mOriginalFunction;
-    return 1;
+  if (!res.ref().CanAccessEvents()) {
+    // When events are passed through, invoke the call with the original stack
+    // and register state.
+    if (!thread || thread->PassThroughEvents()) {
+      // RecordReplayRedirectCall will load the function to call from the
+      // return value slot.
+      aArguments->Rval<uint8_t*>() = redirection.mOriginalFunction;
+      return 1;
+    }
+
+    MOZ_RELEASE_ASSERT(thread->HasDivergedFromRecording());
+
+    // After we have diverged from the recording, we can't access the thread's
+    // recording anymore.
+
+    // If the redirection has a middleman preamble hook, call it to see if it
+    // can handle this call. The middleman preamble hook is separate from the
+    // normal preamble hook because entering the RecordingEventSection can
+    // cause the current thread to diverge from the recording; testing for
+    // HasDivergedFromRecording() does not work reliably in the normal preamble.
+    if (redirection.mMiddlemanPreamble) {
+      if (CallPreambleHook(redirection.mMiddlemanPreamble, aCallId, aArguments)) {
+        return 0;
+      }
+    }
+
+    // If the redirection has a middleman call hook, try to perform the call in
+    // the middleman instead.
+    if (redirection.mMiddlemanCall) {
+      if (SendCallToMiddleman(aCallId, aArguments, /* aPopulateOutput = */ true)) {
+        return 0;
+      }
+    }
+
+    // Calling any redirection which performs the standard steps will cause
+    // debugger operations that have diverged from the recording to fail.
+    EnsureNotDivergedFromRecording();
+    Unreachable();
   }
 
-  // Calling any redirection which performs the standard steps will cause
-  // debugger operations that have diverged from the recording to fail.
-  EnsureNotDivergedFromRecording();
-
-  MOZ_RELEASE_ASSERT(thread->CanAccessRecording());
-
   if (IsRecording()) {
     // Call the original function, passing through events while we do so.
+    // Destroy the RecordingEventSection so that we don't prevent the file
+    // from being flushed in case we end up blocking.
+    res.reset();
     thread->SetPassThrough(true);
     RecordReplayInvokeCall(aCallId, aArguments);
     thread->SetPassThrough(false);
+    res.emplace(thread);
   }
 
   // Save any system error in case we want to record/replay it.
@@ -86,6 +127,13 @@ RecordReplayInterceptCall(int aCallId, CallArguments* aArguments)
     redirection.mSaveOutput(thread->Events(), aArguments, &error);
   }
 
+  // Save information about any potential middleman calls encountered if we
+  // haven't diverged from the recording, in case we diverge and later calls
+  // access data produced by this one.
+  if (IsReplaying() && redirection.mMiddlemanCall) {
+    (void) SendCallToMiddleman(aCallId, aArguments, /* aDiverged = */ false);
+  }
+
   RestoreError(error);
   return 0;
 }

toolkit/recordreplay/ProcessRedirect.h

@@ -13,7 +13,6 @@
 #include "ProcessRecordReplay.h"
 #include "ProcessRewind.h"
 #include "Thread.h"
-#include "ipc/Channel.h"
 
 #include "mozilla/Assertions.h"
 #include "mozilla/Atomics.h"
@@ -74,18 +73,11 @@ namespace recordreplay {
 // Function Redirections
 ///////////////////////////////////////////////////////////////////////////////
 
-// Capture the arguments that can be passed to a redirection, and provide
-// storage to specify the redirection's return value. We only need to capture
-// enough argument data here for calls made directly from Gecko code,
-// i.e. where events are not passed through. Calls made while events are passed
-// through are performed with the same stack and register state as when they
-// were initially invoked.
-//
-// Arguments and return value indexes refer to the register contents as passed
-// to the function originally. For functions with complex or floating point
-// arguments and return values, the right index to use might be different than
-// expected, per the requirements of the System V x64 ABI.
-struct CallArguments
+struct CallArguments;
+
+// All argument and return value data that is stored in registers and whose
+// values are preserved when calling a redirected function.
+struct CallRegisterArguments
 {
 protected:
   size_t arg0;      // 0
@@ -101,6 +93,28 @@ protected:
   size_t rval1;     // 80
   double floatrval0; // 88
   double floatrval1; // 96
+                     // Size: 104
+
+public:
+  void CopyFrom(const CallRegisterArguments* aArguments);
+  void CopyTo(CallRegisterArguments* aArguments) const;
+  void CopyRvalFrom(const CallRegisterArguments* aArguments);
+};
+
+// Capture the arguments that can be passed to a redirection, and provide
+// storage to specify the redirection's return value. We only need to capture
+// enough argument data here for calls made directly from Gecko code,
+// i.e. where events are not passed through. Calls made while events are passed
+// through are performed with the same stack and register state as when they
+// were initially invoked.
+//
+// Arguments and return value indexes refer to the register contents as passed
+// to the function originally. For functions with complex or floating point
+// arguments and return values, the right index to use might be different than
+// expected, per the requirements of the System V x64 ABI.
+struct CallArguments : public CallRegisterArguments
+{
+protected:
   size_t stack[64]; // 104
                     // Size: 616
 
@@ -121,6 +135,12 @@ public:
     }
   }
 
+  template <size_t Offset>
+  size_t* StackAddress() {
+    static_assert(Offset % sizeof(size_t) == 0, "Bad stack offset");
+    return &stack[Offset / sizeof(size_t)];
+  }
+
   template <typename T, size_t Index = 0>
   T& Rval() {
     static_assert(sizeof(T) == sizeof(size_t), "Size must match");
@@ -142,6 +162,27 @@ public:
   }
 };
 
+inline void
+CallRegisterArguments::CopyFrom(const CallRegisterArguments* aArguments)
+{
+  memcpy(this, aArguments, sizeof(CallRegisterArguments));
+}
+
+inline void
+CallRegisterArguments::CopyTo(CallRegisterArguments* aArguments) const
+{
+  memcpy(aArguments, this, sizeof(CallRegisterArguments));
+}
+
+inline void
+CallRegisterArguments::CopyRvalFrom(const CallRegisterArguments* aArguments)
+{
+  rval0 = aArguments->rval0;
+  rval1 = aArguments->rval1;
+  floatrval0 = aArguments->floatrval0;
+  floatrval1 = aArguments->floatrval1;
+}
+
 // Generic type for a system error code.
 typedef ssize_t ErrorType;
 
@@ -164,6 +205,15 @@ enum class PreambleResult {
   PassThrough
 };
 
+// Signature for a function that is called on entry to a redirection and can
+// modify its behavior.
+typedef PreambleResult (*PreambleFn)(CallArguments* aArguments);
+
+// Signature for a function that conveys data about a call to or from the
+// middleman process.
+struct MiddlemanCallContext;
+typedef void (*MiddlemanCallFn)(MiddlemanCallContext& aCx);
+
 // Information about a system library API function which is being redirected.
 struct Redirection
 {
@@ -187,7 +237,15 @@ struct Redirection
   SaveOutputFn mSaveOutput;
 
   // If specified, will be called upon entry to the redirected call.
-  PreambleResult (*mPreamble)(CallArguments* aArguments);
+  PreambleFn mPreamble;
+
+  // If specified, will be called while replaying and diverged from the
+  // recording to perform this call in the middleman process.
+  MiddlemanCallFn mMiddlemanCall;
+
+  // Additional preamble that is only called while replaying and diverged from
+  // the recording.
+  PreambleFn mMiddlemanPreamble;
 };
 
 // All platform specific redirections, indexed by the call event.
@@ -468,15 +526,6 @@ RR_WriteBufferViaRval(Stream& aEvents, CallArguments* aArguments, ErrorType* aEr
   aEvents.RecordOrReplayBytes(buf, rval + Offset);
 }
 
-// Insert an atomic access while recording/replaying so that calls to this
-// function replay in the same order they occurred in while recording. This is
-// used for functions that are used in inter-thread synchronization.
-static inline void
-RR_OrderCall(Stream& aEvents, CallArguments* aArguments, ErrorType* aError)
-{
-  AutoOrderedAtomicAccess();
-}
-
 // Record/replay a scalar return value.
 static inline void
 RR_ScalarRval(Stream& aEvents, CallArguments* aArguments, ErrorType* aError)
@@ -520,7 +569,7 @@ template <size_t ReturnValue>
 static inline PreambleResult
 Preamble_Veto(CallArguments* aArguments)
 {
-  aArguments->Rval<size_t>() = 0;
+  aArguments->Rval<size_t>() = ReturnValue;
   return PreambleResult::Veto;
 }
 
@@ -531,7 +580,7 @@ Preamble_VetoIfNotPassedThrough(CallArguments* aArguments)
   if (AreThreadEventsPassedThrough()) {
     return PreambleResult::PassThrough;
   }
-  aArguments->Rval<size_t>() = 0;
+  aArguments->Rval<size_t>() = ReturnValue;
   return PreambleResult::Veto;
 }
 

toolkit/recordreplay/ProcessRewind.cpp

@@ -8,6 +8,7 @@
 
 #include "nsString.h"
 #include "ipc/ChildInternal.h"
+#include "ipc/ParentInternal.h"
 #include "mozilla/dom/ScriptSettings.h"
 #include "mozilla/StaticMutex.h"
 #include "InfallibleVector.h"
@@ -125,6 +126,7 @@ NewCheckpoint(bool aTemporary)
 {
   MOZ_RELEASE_ASSERT(Thread::CurrentIsMainThread());
   MOZ_RELEASE_ASSERT(!AreThreadEventsPassedThrough());
+  MOZ_RELEASE_ASSERT(!HasDivergedFromRecording());
   MOZ_RELEASE_ASSERT(IsReplaying() || !aTemporary);
 
   navigation::BeforeCheckpoint();
@@ -192,7 +194,13 @@ DivergeFromRecording()
 
   Thread* thread = Thread::Current();
   MOZ_RELEASE_ASSERT(thread->IsMainThread());
-  thread->DivergeFromRecording();
+
+  if (!thread->HasDivergedFromRecording()) {
+    // Reset middleman call state whenever we first diverge from the recording.
+    child::SendResetMiddlemanCalls();
+
+    thread->DivergeFromRecording();
+  }
 
   gUnhandledDivergeAllowed = true;
 }
@@ -218,9 +226,18 @@ DisallowUnhandledDivergeFromRecording()
 void
 EnsureNotDivergedFromRecording()
 {
+  // If we have diverged from the recording and encounter an operation we can't
+  // handle, rewind to the last checkpoint.
   AssertEventsAreNotPassedThrough();
   if (HasDivergedFromRecording()) {
     MOZ_RELEASE_ASSERT(gUnhandledDivergeAllowed);
+
+    // Crash instead of rewinding in the painting stress mode, for finding
+    // areas where middleman calls do not cover all painting logic.
+    if (parent::InRepaintStressMode()) {
+      MOZ_CRASH("Recording divergence in repaint stress mode");
+    }
+
     PrintSpew("Unhandled recording divergence, restoring checkpoint...\n");
     RestoreCheckpointAndResume(gRewindInfo->mSavedCheckpoints.back().mCheckpoint);
     Unreachable();

toolkit/recordreplay/Thread.cpp

@@ -234,10 +234,12 @@ Thread::SpawnThread(Thread* aThread)
 /* static */ NativeThreadId
 Thread::StartThread(Callback aStart, void* aArgument, bool aNeedsJoin)
 {
-  EnsureNotDivergedFromRecording();
-
   Thread* thread = Thread::Current();
-  MOZ_RELEASE_ASSERT(thread->CanAccessRecording());
+  RecordingEventSection res(thread);
+  if (!res.CanAccessEvents()) {
+    EnsureNotDivergedFromRecording();
+    Unreachable();
+  }
 
   MonitorAutoLock lock(*gMonitor);
 
@@ -387,7 +389,15 @@ Thread::WaitForIdleThreads()
       Thread* thread = GetById(i);
       if (!thread->mIdle) {
         done = false;
-        if (thread->mUnrecordedWaitCallback && !thread->mUnrecordedWaitNotified) {
+
+        // Check if there is a callback we can invoke to get this thread to
+        // make progress. The mUnrecordedWaitOnlyWhenDiverged flag is used to
+        // avoid perturbing the behavior of threads that may or may not be
+        // waiting on an unrecorded resource, depending on whether they have
+        // diverged from the recording yet.
+        if (thread->mUnrecordedWaitCallback && !thread->mUnrecordedWaitNotified &&
+            (!thread->mUnrecordedWaitOnlyWhenDiverged ||
+             thread->WillDivergeFromRecordingSoon())) {
           // Set this flag before releasing the idle lock. Otherwise it's
           // possible the thread could call NotifyUnrecordedWait while we
           // aren't holding the lock, and we would set the flag afterwards
@@ -395,10 +405,11 @@ Thread::WaitForIdleThreads()
           thread->mUnrecordedWaitNotified = true;
 
           // Release the idle lock here to avoid any risk of deadlock.
+          std::function<void()> callback = thread->mUnrecordedWaitCallback;
           {
             MonitorAutoUnlock unlock(*gMonitor);
             AutoPassThroughThreadEvents pt;
-            thread->mUnrecordedWaitCallback();
+            callback();
           }
 
           // Releasing the global lock means that we need to start over
@@ -436,7 +447,7 @@ Thread::ResumeIdleThreads()
 }
 
 void
-Thread::NotifyUnrecordedWait(const std::function<void()>& aCallback)
+Thread::NotifyUnrecordedWait(const std::function<void()>& aCallback, bool aOnlyWhenDiverged)
 {
   MonitorAutoLock lock(*gMonitor);
   if (mUnrecordedWaitCallback) {
@@ -449,6 +460,7 @@ Thread::NotifyUnrecordedWait(const std::function<void()>& aCallback)
   }
 
   mUnrecordedWaitCallback = aCallback;
+  mUnrecordedWaitOnlyWhenDiverged = aOnlyWhenDiverged;
 
   // The main thread might be able to make progress now by calling the routine
   // if it is waiting for idle replay threads.
@@ -470,9 +482,10 @@ Thread::MaybeWaitForCheckpointSave()
 extern "C" {
 
 MOZ_EXPORT void
-RecordReplayInterface_NotifyUnrecordedWait(const std::function<void()>& aCallback)
+RecordReplayInterface_NotifyUnrecordedWait(const std::function<void()>& aCallback,
+                                           bool aOnlyWhenDiverged)
 {
-  Thread::Current()->NotifyUnrecordedWait(aCallback);
+  Thread::Current()->NotifyUnrecordedWait(aCallback, aOnlyWhenDiverged);
 }
 
 MOZ_EXPORT void

toolkit/recordreplay/Thread.h

@@ -98,6 +98,10 @@ private:
   // recorded events cannot be accessed.
   bool mDivergedFromRecording;
 
+  // Whether this thread should diverge from the recording at the next
+  // opportunity. This can be set from any thread.
+  Atomic<bool, SequentiallyConsistent, Behavior::DontPreserve> mShouldDivergeFromRecording;
+
   // Start routine and argument which the thread is currently executing. This
   // is cleared after the routine finishes and another start routine may be
   // assigned to the thread. mNeedsJoin specifies whether the thread must be
@@ -130,6 +134,7 @@ private:
   // and whether the callback has been invoked yet while the main thread is
   // waiting for threads to become idle. Protected by the thread monitor.
   std::function<void()> mUnrecordedWaitCallback;
+  bool mUnrecordedWaitOnlyWhenDiverged;
   bool mUnrecordedWaitNotified;
 
 public:
@@ -179,6 +184,26 @@ public:
     return mDivergedFromRecording;
   }
 
+  // Mark this thread as needing to diverge from the recording soon, and wake
+  // it up in case it can make progress now. The mShouldDivergeFromRecording
+  // flag is separate from mDivergedFromRecording so that the thread can only
+  // begin diverging from the recording at calls to MaybeDivergeFromRecording.
+  void SetShouldDivergeFromRecording() {
+    MOZ_RELEASE_ASSERT(CurrentIsMainThread());
+    mShouldDivergeFromRecording = true;
+    Notify(mId);
+  }
+  bool WillDivergeFromRecordingSoon() {
+    MOZ_RELEASE_ASSERT(CurrentIsMainThread());
+    return mShouldDivergeFromRecording;
+  }
+  bool MaybeDivergeFromRecording() {
+    if (mShouldDivergeFromRecording) {
+      mDivergedFromRecording = true;
+    }
+    return mDivergedFromRecording;
+  }
+
   // Return whether this thread may read or write to its recorded event stream.
   bool CanAccessRecording() const {
     return !PassThroughEvents() && !AreEventsDisallowed() && !HasDivergedFromRecording();
@@ -253,7 +278,8 @@ public:
   static void WaitForeverNoIdle();
 
   // See RecordReplay.h.
-  void NotifyUnrecordedWait(const std::function<void()>& aCallback);
+  void NotifyUnrecordedWait(const std::function<void()>& aCallback,
+                            bool aOnlyWhenDiverged);
   static void MaybeWaitForCheckpointSave();
 
   // Wait for all other threads to enter the idle state necessary for saving
@@ -290,6 +316,60 @@ public:
   }
 };
 
+// Mark a region of code where a thread's event stream can be accessed.
+// This class has several properties:
+//
+// - When recording, all writes to the thread's event stream occur atomically
+//   within the class: the end of the stream cannot be hit at an intermediate
+//   point.
+//
+// - When replaying, this checks for the end of the stream, and blocks the
+//   thread if necessary.
+//
+// - When replaying, this is a point where the thread can begin diverging from
+//   the recording. Checks for divergence should occur after the constructor
+//   finishes.
+class MOZ_RAII RecordingEventSection
+{
+  Thread* mThread;
+
+public:
+  explicit RecordingEventSection(Thread* aThread)
+    : mThread(aThread)
+  {
+    if (!aThread || !aThread->CanAccessRecording()) {
+      return;
+    }
+    if (IsRecording()) {
+      MOZ_RELEASE_ASSERT(!aThread->Events().mInRecordingEventSection);
+      aThread->Events().mFile->mStreamLock.ReadLock();
+      aThread->Events().mInRecordingEventSection = true;
+    } else {
+      while (!aThread->MaybeDivergeFromRecording() && aThread->Events().AtEnd()) {
+        HitEndOfRecording();
+      }
+    }
+  }
+
+  ~RecordingEventSection() {
+    if (!mThread || !mThread->CanAccessRecording()) {
+      return;
+    }
+    if (IsRecording()) {
+      mThread->Events().mFile->mStreamLock.ReadUnlock();
+      mThread->Events().mInRecordingEventSection = false;
+    }
+  }
+
+  bool CanAccessEvents() {
+    if (!mThread || mThread->PassThroughEvents() || mThread->HasDivergedFromRecording()) {
+      return false;
+    }
+    MOZ_RELEASE_ASSERT(mThread->CanAccessRecording());
+    return true;
+  }
+};
+
 } // namespace recordreplay
 } // namespace mozilla
 

toolkit/recordreplay/Trigger.cpp

@@ -59,6 +59,7 @@ MOZ_EXPORT void
 RecordReplayInterface_RegisterTrigger(void* aObj, const std::function<void()>& aCallback)
 {
   MOZ_RELEASE_ASSERT(aObj);
+  MOZ_RELEASE_ASSERT(!AreThreadEventsPassedThrough());
 
   Thread* thread = Thread::Current();
   if (thread->HasDivergedFromRecording()) {
@@ -84,6 +85,9 @@ RecordReplayInterface_RegisterTrigger(void* aObj, const std::function<void()>& a
     }
   }
 
+  RecordingEventSection res(thread);
+  MOZ_RELEASE_ASSERT(res.CanAccessEvents());
+
   thread->Events().RecordOrReplayThreadEvent(ThreadEvent::RegisterTrigger);
   thread->Events().CheckInput(id);
 }
@@ -157,7 +161,10 @@ MOZ_EXPORT void
 RecordReplayInterface_ExecuteTriggers()
 {
   Thread* thread = Thread::Current();
-  MOZ_RELEASE_ASSERT(thread->CanAccessRecording());
+  RecordingEventSection res(thread);
+  if (!res.CanAccessEvents()) {
+    return;
+  }
 
   if (IsRecording()) {
     // Invoke the callbacks for any triggers waiting for execution, including

toolkit/recordreplay/ipc/Channel.cpp

@@ -12,6 +12,7 @@
 
 #include "MainThreadUtils.h"
 #include "nsXULAppAPI.h"
+#include "base/eintr_wrapper.h"
 #include "base/process_util.h"
 #include "mozilla/dom/ContentChild.h"
 #include "mozilla/ipc/FileDescriptor.h"
@@ -65,6 +66,7 @@ Channel::Channel(size_t aId, bool aMiddlemanRecording, const MessageHandler& aHa
   , mInitialized(false)
   , mConnectionFd(0)
   , mFd(0)
+  , mMessageBuffer(nullptr)
   , mMessageBytes(0)
 {
   MOZ_RELEASE_ASSERT(NS_IsMainThread());
@@ -148,7 +150,9 @@ Channel::ThreadMain(void* aChannelArg)
 void
 Channel::SendMessage(const Message& aMsg)
 {
-  MOZ_RELEASE_ASSERT(NS_IsMainThread() || aMsg.mType == MessageType::FatalError);
+  MOZ_RELEASE_ASSERT(NS_IsMainThread() ||
+                     aMsg.mType == MessageType::FatalError ||
+                     aMsg.mType == MessageType::MiddlemanCallRequest);
 
   // Block until the channel is initialized.
   if (!mInitialized) {
@@ -179,27 +183,29 @@ Channel::SendMessage(const Message& aMsg)
 Message*
 Channel::WaitForMessage()
 {
-  if (!mMessageBuffer.length()) {
-    mMessageBuffer.appendN(0, PageSize);
+  if (!mMessageBuffer) {
+    mMessageBuffer = (MessageBuffer*) AllocateMemory(sizeof(MessageBuffer), MemoryKind::Generic);
+    mMessageBuffer->appendN(0, PageSize);
   }
 
   size_t messageSize = 0;
   while (true) {
     if (mMessageBytes >= sizeof(Message)) {
-      Message* msg = (Message*) mMessageBuffer.begin();
+      Message* msg = (Message*) mMessageBuffer->begin();
       messageSize = msg->mSize;
+      MOZ_RELEASE_ASSERT(messageSize >= sizeof(Message));
       if (mMessageBytes >= messageSize) {
         break;
       }
     }
 
     // Make sure the buffer is large enough for the entire incoming message.
-    if (messageSize > mMessageBuffer.length()) {
-      mMessageBuffer.appendN(0, messageSize - mMessageBuffer.length());
+    if (messageSize > mMessageBuffer->length()) {
+      mMessageBuffer->appendN(0, messageSize - mMessageBuffer->length());
     }
 
-    ssize_t nbytes = HANDLE_EINTR(recv(mFd, &mMessageBuffer[mMessageBytes],
-                                       mMessageBuffer.length() - mMessageBytes, 0));
+    ssize_t nbytes = HANDLE_EINTR(recv(mFd, &mMessageBuffer->begin()[mMessageBytes],
+                                       mMessageBuffer->length() - mMessageBytes, 0));
     if (nbytes < 0) {
       MOZ_RELEASE_ASSERT(errno == EAGAIN);
       continue;
@@ -215,12 +221,12 @@ Channel::WaitForMessage()
     mMessageBytes += nbytes;
   }
 
-  Message* res = ((Message*)mMessageBuffer.begin())->Clone();
+  Message* res = ((Message*)mMessageBuffer->begin())->Clone();
 
   // Remove the message we just received from the incoming buffer.
   size_t remaining = mMessageBytes - messageSize;
   if (remaining) {
-    memmove(mMessageBuffer.begin(), &mMessageBuffer[messageSize], remaining);
+    memmove(mMessageBuffer->begin(), &mMessageBuffer->begin()[messageSize], remaining);
   }
   mMessageBytes = remaining;
 

toolkit/recordreplay/ipc/Channel.h

@@ -98,6 +98,9 @@ namespace recordreplay {
   /* Set whether to save a particular checkpoint. */           \
   _Macro(SetSaveCheckpoint)                                    \
                                                                \
+  /* Respond to a MiddlemanCallRequest message. */             \
+  _Macro(MiddlemanCallResponse)                                \
+                                                               \
   /* Messages sent from the child process to the middleman. */ \
                                                                \
   /* Sent in response to a FlushRecording, telling the middleman that the flush */ \
@@ -119,6 +122,13 @@ namespace recordreplay {
   /* Send a response to a DebuggerRequest message. */          \
   _Macro(DebuggerResponse)                                     \
                                                                \
+  /* Call a system function from the middleman process which the child has */ \
+  /* encountered after diverging from the recording. */        \
+  _Macro(MiddlemanCallRequest)                                 \
+                                                               \
+  /* Reset all information generated by previous MiddlemanCallRequest messages. */ \
+  _Macro(ResetMiddlemanCalls)                                  \
+                                                               \
   /* Notify that the 'AlwaysMarkMajorCheckpoints' directive was invoked. */ \
   _Macro(AlwaysMarkMajorCheckpoints)
 
@@ -164,6 +174,7 @@ public:
   bool CanBeSentWhileUnpaused() const {
     return mType == MessageType::CreateCheckpoint
         || mType == MessageType::SetDebuggerRunsInMiddleman
+        || mType == MessageType::MiddlemanCallResponse
         || mType == MessageType::Terminate;
   }
 
@@ -367,11 +378,15 @@ static const gfx::SurfaceFormat gSurfaceFormat = gfx::SurfaceFormat::R8G8B8X8;
 
 struct PaintMessage : public Message
 {
+  // Checkpoint whose state is being painted.
+  uint32_t mCheckpointId;
+
   uint32_t mWidth;
   uint32_t mHeight;
 
-  PaintMessage(uint32_t aWidth, uint32_t aHeight)
+  PaintMessage(uint32_t aCheckpointId, uint32_t aWidth, uint32_t aHeight)
     : Message(MessageType::Paint, sizeof(*this))
+    , mCheckpointId(aCheckpointId)
     , mWidth(aWidth)
     , mHeight(aHeight)
   {}
@@ -418,6 +433,29 @@ struct HitBreakpointMessage : public Message
 
 typedef EmptyMessage<MessageType::AlwaysMarkMajorCheckpoints> AlwaysMarkMajorCheckpointsMessage;
 
+template <MessageType Type>
+struct BinaryMessage : public Message
+{
+  explicit BinaryMessage(uint32_t aSize)
+    : Message(Type, aSize)
+  {}
+
+  const char* BinaryData() const { return Data<BinaryMessage<Type>, char>(); }
+  size_t BinaryDataSize() const { return DataSize<BinaryMessage<Type>, char>(); }
+
+  static BinaryMessage<Type>*
+  New(const char* aData, size_t aDataSize) {
+    BinaryMessage<Type>* res = NewWithData<BinaryMessage<Type>, char>(aDataSize);
+    MOZ_RELEASE_ASSERT(res->BinaryDataSize() == aDataSize);
+    PodCopy(res->Data<BinaryMessage<Type>, char>(), aData, aDataSize);
+    return res;
+  }
+};
+
+typedef BinaryMessage<MessageType::MiddlemanCallRequest> MiddlemanCallRequestMessage;
+typedef BinaryMessage<MessageType::MiddlemanCallResponse> MiddlemanCallResponseMessage;
+typedef EmptyMessage<MessageType::ResetMiddlemanCalls> ResetMiddlemanCallsMessage;
+
 class Channel
 {
 public:
@@ -445,7 +483,8 @@ private:
   Monitor mMonitor;
 
   // Buffer for message data received from the other side of the channel.
-  InfallibleVector<char, 0, AllocPolicy<MemoryKind::Generic>> mMessageBuffer;
+  typedef InfallibleVector<char, 0, AllocPolicy<MemoryKind::Generic>> MessageBuffer;
+  MessageBuffer* mMessageBuffer;
 
   // The number of bytes of data already in the message buffer.
   size_t mMessageBytes;

toolkit/recordreplay/ipc/ChildIPC.cpp

@@ -16,7 +16,10 @@
 #include "ipc/Channel.h"
 #include "mac/handler/exception_handler.h"
 #include "mozilla/dom/ContentChild.h"
+#include "mozilla/layers/CompositorBridgeChild.h"
 #include "mozilla/layers/ImageDataSerializer.h"
+#include "mozilla/layers/ImageDataSerializer.h"
+#include "mozilla/layers/LayerTransactionChild.h"
 #include "mozilla/Sprintf.h"
 #include "mozilla/VsyncDispatcher.h"
 
@@ -64,6 +67,9 @@ static IntroductionMessage* gIntroductionMessage;
 // When recording, whether developer tools server code runs in the middleman.
 static bool gDebuggerRunsInMiddleman;
 
+// Any response received to the last MiddlemanCallRequest message.
+static MiddlemanCallResponseMessage* gCallResponseMessage;
+
 // Processing routine for incoming channel messages.
 static void
 ChannelMessageHandler(Message* aMsg)
@@ -159,6 +165,14 @@ ChannelMessageHandler(Message* aMsg)
       });
     break;
   }
+  case MessageType::MiddlemanCallResponse: {
+    MonitorAutoLock lock(*gMonitor);
+    MOZ_RELEASE_ASSERT(!gCallResponseMessage);
+    gCallResponseMessage = (MiddlemanCallResponseMessage*) aMsg;
+    aMsg = nullptr; // Avoid freeing the message below.
+    gMonitor->NotifyAll();
+    break;
+  }
   default:
     MOZ_CRASH();
   }
@@ -187,6 +201,7 @@ ListenForCheckpointThreadMain(void*)
   }
 }
 
+// Shared memory block for graphics data.
 void* gGraphicsShmem;
 
 void
@@ -325,9 +340,11 @@ DebuggerRunsInMiddleman()
 }
 
 void
-MaybeCreateInitialCheckpoint()
+CreateCheckpoint()
 {
-  NewCheckpoint(/* aTemporary = */ false);
+  if (!HasDivergedFromRecording()) {
+    NewCheckpoint(/* aTemporary = */ false);
+  }
 }
 
 void
@@ -398,7 +415,7 @@ SetVsyncObserver(VsyncObserver* aObserver)
   gVsyncObserver = aObserver;
 }
 
-void
+static void
 NotifyVsyncObserver()
 {
   if (gVsyncObserver) {
@@ -406,30 +423,68 @@ NotifyVsyncObserver()
   }
 }
 
+// Whether an update has been sent to the compositor for a normal paint, and we
+// haven't reached PaintFromMainThread yet. This is used to preserve the
+// invariant that there can be at most one paint performed between two
+// checkpoints, other than repaints triggered by the debugger.
+static bool gHasActivePaint;
+
+bool
+OnVsync()
+{
+  // In the repainting stress mode, we create a new checkpoint on every vsync
+  // message received from the UI process. When we notify the parent about the
+  // new checkpoint it will trigger a repaint to make sure that all layout and
+  // painting activity can occur when diverged from the recording.
+  if (parent::InRepaintStressMode()) {
+    CreateCheckpoint();
+  }
+
+  // After a paint starts, ignore incoming vsyncs until the paint completes.
+  return !gHasActivePaint;
+}
+
 ///////////////////////////////////////////////////////////////////////////////
 // Painting
 ///////////////////////////////////////////////////////////////////////////////
 
-// Graphics memory is only written on the compositor thread and read on the
-// main thread and by the middleman. The gPendingPaint flag is used to
-// synchronize access, so that data is not read until the paint has completed.
-static Maybe<PaintMessage> gPaintMessage;
-static bool gPendingPaint;
-
-// Target buffer for the draw target created by the child process widget.
+// Target buffer for the draw target created by the child process widget, which
+// the compositor thread writes to.
 static void* gDrawTargetBuffer;
 static size_t gDrawTargetBufferSize;
 
+// Dimensions of the last paint which the compositor performed.
+static size_t gPaintWidth, gPaintHeight;
+
+// How many updates have been sent to the compositor thread and haven't been
+// processed yet. This can briefly become negative if the main thread sends an
+// update and the compositor processes it before the main thread reaches
+// NotifyPaintStart. Outside of this window, the compositor can only write to
+// gDrawTargetBuffer or update gPaintWidth/gPaintHeight if this is non-zero.
+static Atomic<int32_t, SequentiallyConsistent, Behavior::DontPreserve> gNumPendingPaints;
+
+// ID of the compositor thread.
+static Atomic<size_t, SequentiallyConsistent, Behavior::DontPreserve> gCompositorThreadId;
+
 already_AddRefed<gfx::DrawTarget>
 DrawTargetForRemoteDrawing(LayoutDeviceIntSize aSize)
 {
   MOZ_RELEASE_ASSERT(!NS_IsMainThread());
 
+  // Keep track of the compositor thread ID.
+  size_t threadId = Thread::Current()->Id();
+  if (gCompositorThreadId) {
+    MOZ_RELEASE_ASSERT(threadId == gCompositorThreadId);
+  } else {
+    gCompositorThreadId = threadId;
+  }
+
   if (aSize.IsEmpty()) {
     return nullptr;
   }
 
-  gPaintMessage = Some(PaintMessage(aSize.width, aSize.height));
+  gPaintWidth = aSize.width;
+  gPaintHeight = aSize.height;
 
   gfx::IntSize size(aSize.width, aSize.height);
   size_t bufferSize = layers::ImageDataSerializer::ComputeRGBBufferSize(size, gSurfaceFormat);
@@ -458,42 +513,135 @@ NotifyPaintStart()
 {
   MOZ_RELEASE_ASSERT(NS_IsMainThread());
 
-  NewCheckpoint(/* aTemporary = */ false);
+  // A new paint cannot be triggered until the last one finishes and has been
+  // sent to the middleman.
+  MOZ_RELEASE_ASSERT(HasDivergedFromRecording() || !gHasActivePaint);
 
-  gPendingPaint = true;
+  gNumPendingPaints++;
+  gHasActivePaint = true;
+
+  CreateCheckpoint();
 }
 
-void
-WaitForPaintToComplete()
+static void
+PaintFromMainThread()
 {
   MOZ_RELEASE_ASSERT(NS_IsMainThread());
 
-  MonitorAutoLock lock(*gMonitor);
-  while (gPendingPaint) {
-    gMonitor->Wait();
-  }
-  if (IsActiveChild() && gPaintMessage.isSome()) {
+  // There cannot not be any other in flight paints.
+  MOZ_RELEASE_ASSERT(!gNumPendingPaints);
+
+  // Clear the active flag now that we have completed the paint.
+  MOZ_RELEASE_ASSERT(gHasActivePaint);
+  gHasActivePaint = false;
+
+  if (IsActiveChild() && gDrawTargetBuffer) {
     memcpy(gGraphicsShmem, gDrawTargetBuffer, gDrawTargetBufferSize);
-    gChannel->SendMessage(gPaintMessage.ref());
+    gChannel->SendMessage(PaintMessage(navigation::LastNormalCheckpoint(),
+                                       gPaintWidth, gPaintHeight));
   }
 }
 
 void
 NotifyPaintComplete()
 {
-  MOZ_RELEASE_ASSERT(!NS_IsMainThread());
+  MOZ_RELEASE_ASSERT(Thread::Current()->Id() == gCompositorThreadId);
 
-  MonitorAutoLock lock(*gMonitor);
-  MOZ_RELEASE_ASSERT(gPendingPaint);
-  gPendingPaint = false;
-  gMonitor->Notify();
+  // Notify the main thread in case it is waiting for this paint to complete.
+  {
+    MonitorAutoLock lock(*gMonitor);
+    if (--gNumPendingPaints == 0) {
+      gMonitor->Notify();
+    }
+  }
+
+  // Notify the middleman about the completed paint from the main thread.
+  NS_DispatchToMainThread(NewRunnableFunction("PaintFromMainThread", PaintFromMainThread));
+}
+
+void
+Repaint(size_t* aWidth, size_t* aHeight)
+{
+  MOZ_RELEASE_ASSERT(NS_IsMainThread());
+  MOZ_RELEASE_ASSERT(HasDivergedFromRecording());
+
+  // Don't try to repaint if the first normal paint hasn't occurred yet.
+  if (!gCompositorThreadId) {
+    *aWidth = 0;
+    *aHeight = 0;
+    return;
+  }
+
+  // Ignore the request to repaint if the compositor thread has already
+  // diverged from the recording. In this case we have already done a repaint
+  // and the last graphics we sent will still be correct.
+  Thread* compositorThread = Thread::GetById(gCompositorThreadId);
+  if (!compositorThread->WillDivergeFromRecordingSoon()) {
+    // Create an artifical vsync to see if graphics have changed since the last
+    // paint and a new paint is needed.
+    NotifyVsyncObserver();
+
+    if (gNumPendingPaints) {
+      // Allow the compositor to diverge from the recording so it can perform
+      // any paint we just triggered, or finish any in flight paint that that
+      // existed at the point we are paused at.
+      Thread::GetById(gCompositorThreadId)->SetShouldDivergeFromRecording();
+
+      // Wait for the compositor to finish all in flight paints, including any
+      // one we just triggered.
+      MonitorAutoLock lock(*gMonitor);
+      while (gNumPendingPaints) {
+        gMonitor->Wait();
+      }
+    }
+  }
+
+  if (gDrawTargetBuffer) {
+    memcpy(gGraphicsShmem, gDrawTargetBuffer, gDrawTargetBufferSize);
+    *aWidth = gPaintWidth;
+    *aHeight = gPaintHeight;
+  } else {
+    *aWidth = 0;
+    *aHeight = 0;
+  }
+}
+
+static bool
+CompositorCanPerformMiddlemanCalls()
+{
+  // After repainting finishes the compositor is not allowed to send call
+  // requests to the middleman anymore.
+  return !!gNumPendingPaints;
+}
+
+bool
+SuppressMessageAfterDiverge(IPC::Message* aMsg)
+{
+  MOZ_RELEASE_ASSERT(HasDivergedFromRecording());
+
+  // Only messages necessary for compositing can be sent after the sending
+  // thread has diverged from the recording. Sending other messages can risk
+  // deadlocking when a necessary lock is held by an idle thread (we probably
+  // need a more robust way to deal with this problem).
+
+  IPC::Message::msgid_t type = aMsg->type();
+  if (type >= layers::PLayerTransaction::PLayerTransactionStart &&
+      type <= layers::PLayerTransaction::PLayerTransactionEnd) {
+    return false;
+  }
+
+  if (type == layers::PCompositorBridge::Msg_PTextureConstructor__ID) {
+    return false;
+  }
+
+  return true;
 }
 
 ///////////////////////////////////////////////////////////////////////////////
 // Checkpoint Messages
 ///////////////////////////////////////////////////////////////////////////////
 
-// When recording, the time when the last HitCheckpoint message was sent.
+// The time when the last HitCheckpoint message was sent.
 static double gLastCheckpointTime;
 
 // When recording and we are idle, the time when we became idle.
@@ -534,7 +682,7 @@ HitCheckpoint(size_t aId, bool aRecordingEndpoint)
 }
 
 ///////////////////////////////////////////////////////////////////////////////
-// Debugger Messages
+// Message Helpers
 ///////////////////////////////////////////////////////////////////////////////
 
 void
@@ -558,6 +706,52 @@ HitBreakpoint(bool aRecordingEndpoint, const uint32_t* aBreakpoints, size_t aNum
     });
 }
 
+bool
+SendMiddlemanCallRequest(const char* aInputData, size_t aInputSize,
+                         InfallibleVector<char>* aOutputData)
+{
+  Thread* thread = Thread::Current();
+
+  // Middleman calls can only be made from the main and compositor threads.
+  // These two threads cannot simultaneously send call requests or other
+  // messages, as doing so will race both here and in Channel::SendMessage.
+  // CompositorCanPerformMiddlemanCalls() ensures that the main thread is
+  // not actively sending messages at times when the compositor performs
+  // middleman calls.
+  MOZ_RELEASE_ASSERT(thread->IsMainThread() || thread->Id() == gCompositorThreadId);
+
+  if (thread->Id() == gCompositorThreadId && !CompositorCanPerformMiddlemanCalls()) {
+    return false;
+  }
+
+  MonitorAutoLock lock(*gMonitor);
+
+  MOZ_RELEASE_ASSERT(!gCallResponseMessage);
+
+  MiddlemanCallRequestMessage* msg = MiddlemanCallRequestMessage::New(aInputData, aInputSize);
+  gChannel->SendMessage(*msg);
+  free(msg);
+
+  while (!gCallResponseMessage) {
+    gMonitor->Wait();
+  }
+
+  aOutputData->append(gCallResponseMessage->BinaryData(), gCallResponseMessage->BinaryDataSize());
+
+  free(gCallResponseMessage);
+  gCallResponseMessage = nullptr;
+
+  gMonitor->Notify();
+  return true;
+}
+
+void
+SendResetMiddlemanCalls()
+{
+  MOZ_RELEASE_ASSERT(NS_IsMainThread());
+  gChannel->SendMessage(ResetMiddlemanCallsMessage());
+}
+
 } // namespace child
 } // namespace recordreplay
 } // namespace mozilla

toolkit/recordreplay/ipc/ChildIPC.h

@@ -11,6 +11,8 @@
 #include "mozilla/gfx/2D.h"
 #include "Units.h"
 
+namespace IPC { class Message; }
+
 namespace mozilla {
 
 class VsyncObserver;
@@ -29,42 +31,35 @@ void InitRecordingOrReplayingProcess(int* aArgc, char*** aArgv);
 base::ProcessId MiddlemanProcessId();
 base::ProcessId ParentProcessId();
 
-// Create a normal checkpoint, if no such checkpoint has been created yet.
-void MaybeCreateInitialCheckpoint();
+// Create a normal checkpoint, if execution has not diverged from the recording.
+void CreateCheckpoint();
 
 ///////////////////////////////////////////////////////////////////////////////
 // Painting Coordination
 ///////////////////////////////////////////////////////////////////////////////
 
-// In child processes, paints do not occur in response to vsyncs from the UI
-// process: when a page is updating rapidly these events occur sporadically and
-// cause the tab's graphics to not accurately reflect the tab's state at that
-// point in time. When viewing a normal tab this is no problem because the tab
-// will be painted with the correct graphics momentarily, but when the tab can
-// be rewound and paused this behavior is visible.
-//
-// This API is used to trigger artificial vsyncs whenever the page is updated.
-// SetVsyncObserver is used to tell the child code about any singleton vsync
-// observer that currently exists, and NotifyVsyncObserver is used to trigger
-// a vsync on that observer at predictable points, e.g. the top of the main
-// thread's event loop.
+// Tell the child code about any singleton vsync observer that currently
+// exists. This is used to trigger artifical vsyncs that paint the current
+// graphics when paused.
 void SetVsyncObserver(VsyncObserver* aObserver);
-void NotifyVsyncObserver();
 
-// Similarly to the vsync handling above, in order to ensure that the tab's
-// graphics accurately reflect its state, we want to perform paints
-// synchronously after a vsync has occurred. When a paint is about to happen,
-// the main thread calls NotifyPaintStart, and after the compositor thread has
-// been informed about the update the main thread calls WaitForPaintToComplete
-// to block until the compositor thread has finished painting and called
-// NotifyPaintComplete.
+// Called before processing incoming vsyncs from the UI process. Returns false
+// if the vsync should be ignored.
+bool OnVsync();
+
+// Tell the child code about any ongoing painting activity. When a paint is
+// about to happen, the main thread calls NotifyPaintStart, and when the
+// compositor thread finishes the paint it calls NotifyPaintComplete.
 void NotifyPaintStart();
-void WaitForPaintToComplete();
 void NotifyPaintComplete();
 
 // Get a draw target which the compositor thread can paint to.
 already_AddRefed<gfx::DrawTarget> DrawTargetForRemoteDrawing(LayoutDeviceIntSize aSize);
 
+// Called to ignore IPDL messages sent after diverging from the recording,
+// except for those needed for compositing.
+bool SuppressMessageAfterDiverge(IPC::Message* aMsg);
+
 } // namespace child
 } // namespace recordreplay
 } // namespace mozilla

toolkit/recordreplay/ipc/ChildInternal.h

@@ -7,8 +7,10 @@
 #ifndef mozilla_recordreplay_ChildInternal_h
 #define mozilla_recordreplay_ChildInternal_h
 
+#include "Channel.h"
 #include "ChildIPC.h"
 #include "JSControl.h"
+#include "MiddlemanCall.h"
 #include "Monitor.h"
 
 namespace mozilla {
@@ -62,6 +64,10 @@ js::ExecutionPoint CurrentExecutionPoint(const js::BreakpointPosition& aPosition
 // executing into an ExecutionPoint.
 js::ExecutionPoint TimeWarpTargetExecutionPoint(ProgressCounter aTarget);
 
+// Synchronously paint the current contents into the graphics shared memory
+// object, returning the size of the painted area via aWidth/aHeight.
+void Repaint(size_t* aWidth, size_t* aHeight);
+
 // Called when running forward, immediately before hitting a normal or
 // temporary checkpoint.
 void BeforeCheckpoint();
@@ -70,6 +76,9 @@ void BeforeCheckpoint();
 // when running forward or immediately after rewinding.
 void AfterCheckpoint(const CheckpointId& aCheckpoint);
 
+// Get the ID of the last normal checkpoint.
+size_t LastNormalCheckpoint();
+
 } // namespace navigation
 
 namespace child {
@@ -116,6 +125,11 @@ void EndIdleTime();
 // Whether the middleman runs developer tools server code.
 bool DebuggerRunsInMiddleman();
 
+// Send messages operating on middleman calls.
+bool SendMiddlemanCallRequest(const char* aInputData, size_t aInputSize,
+                              InfallibleVector<char>* aOutputData);
+void SendResetMiddlemanCalls();
+
 } // namespace child
 
 } // namespace recordreplay

toolkit/recordreplay/ipc/ChildNavigation.cpp

@@ -698,13 +698,15 @@ PausedPhase::MaybeDivergeFromRecording()
     return false;
   }
 
+  size_t index = mRequestIndex;
+
   if (!EnsureTemporaryCheckpoint()) {
     // One of the premature exit cases was hit in EnsureTemporaryCheckpoint.
     // Don't allow any operations that can diverge from the recording.
     return false;
   }
 
-  if (mRequests[mRequestIndex].mUnhandledDivergence) {
+  if (mRequests[index].mUnhandledDivergence) {
     // We tried to process this request before and had an unhandled divergence.
     // Disallow the request handler from doing anything that might diverge from
     // the recording.
@@ -1109,6 +1111,12 @@ AfterCheckpoint(const CheckpointId& aCheckpoint)
   gNavigation->AfterCheckpoint(aCheckpoint);
 }
 
+size_t
+LastNormalCheckpoint()
+{
+  return gNavigation->LastCheckpoint().mNormal;
+}
+
 void
 DebuggerRequest(js::CharBuffer* aRequestBuffer)
 {
@@ -1190,9 +1198,13 @@ extern "C" {
 MOZ_EXPORT ProgressCounter
 RecordReplayInterface_NewTimeWarpTarget()
 {
+  if (AreThreadEventsDisallowed()) {
+    return 0;
+  }
+
   // NewTimeWarpTarget() must be called at consistent points between recording
   // and replaying.
-  recordreplay::RecordReplayAssert("NewTimeWarpTarget");
+  RecordReplayAssert("NewTimeWarpTarget");
 
   if (!gNavigation) {
     return 0;

toolkit/recordreplay/ipc/ChildProcess.cpp

@@ -295,6 +295,7 @@ ChildProcessInfo::SendMessage(const Message& aMsg)
   case MessageType::RunToPoint:
   case MessageType::DebuggerRequest:
   case MessageType::SetBreakpoint:
+  case MessageType::MiddlemanCallResponse:
     mMessages.emplaceBack(aMsg.Clone());
     break;
   default:
@@ -407,8 +408,11 @@ ChildProcessInfo::OnIncomingRecoveryMessage(const Message& aMsg)
   }
   case MessageType::HitBreakpoint:
   case MessageType::DebuggerResponse:
+  case MessageType::MiddlemanCallRequest:
     SendNextRecoveryMessage();
     break;
+  case MessageType::ResetMiddlemanCalls:
+    break;
   default:
     MOZ_CRASH("Unexpected message during recovery");
   }

toolkit/recordreplay/ipc/DisabledIPC.cpp

@@ -40,7 +40,7 @@ ParentProcessId()
   MOZ_CRASH();
 }
 
-void MaybeCreateInitialCheckpoint()
+void CreateCheckpoint()
 {
   MOZ_CRASH();
 }
@@ -51,8 +51,8 @@ SetVsyncObserver(VsyncObserver* aObserver)
   MOZ_CRASH();
 }
 
-void
-NotifyVsyncObserver()
+bool
+OnVsync()
 {
   MOZ_CRASH();
 }
@@ -69,14 +69,14 @@ NotifyPaintComplete()
   MOZ_CRASH();
 }
 
-void
-WaitForPaintToComplete()
+already_AddRefed<gfx::DrawTarget>
+DrawTargetForRemoteDrawing(LayoutDeviceIntSize aSize)
 {
   MOZ_CRASH();
 }
 
-already_AddRefed<gfx::DrawTarget>
-DrawTargetForRemoteDrawing(LayoutDeviceIntSize aSize)
+bool
+SuppressMessageAfterDiverge(IPC::Message* aMsg)
 {
   MOZ_CRASH();
 }

toolkit/recordreplay/ipc/JSControl.cpp

@@ -401,6 +401,37 @@ Middleman_MaybeSwitchToReplayingChild(JSContext* aCx, unsigned aArgc, Value* aVp
   return true;
 }
 
+static bool
+Middleman_HadRepaint(JSContext* aCx, unsigned aArgc, Value* aVp)
+{
+  CallArgs args = CallArgsFromVp(aArgc, aVp);
+
+  if (!args.get(0).isNumber() || !args.get(1).isNumber()) {
+    JS_ReportErrorASCII(aCx, "Bad width/height");
+    return false;
+  }
+
+  size_t width = args.get(0).toNumber();
+  size_t height = args.get(1).toNumber();
+
+  PaintMessage message(CheckpointId::Invalid, width, height);
+  parent::UpdateGraphicsInUIProcess(&message);
+
+  args.rval().setUndefined();
+  return true;
+}
+
+static bool
+Middleman_HadRepaintFailure(JSContext* aCx, unsigned aArgc, Value* aVp)
+{
+  CallArgs args = CallArgsFromVp(aArgc, aVp);
+
+  parent::UpdateGraphicsInUIProcess(nullptr);
+
+  args.rval().setUndefined();
+  return true;
+}
+
 ///////////////////////////////////////////////////////////////////////////////
 // Devtools Sandbox
 ///////////////////////////////////////////////////////////////////////////////
@@ -823,6 +854,26 @@ RecordReplay_TimeWarpTargetExecutionPoint(JSContext* aCx, unsigned aArgc, Value*
   return true;
 }
 
+static bool
+RecordReplay_Repaint(JSContext* aCx, unsigned aArgc, Value* aVp)
+{
+  CallArgs args = CallArgsFromVp(aArgc, aVp);
+
+  size_t width, height;
+  child::Repaint(&width, &height);
+
+  RootedObject obj(aCx, JS_NewObject(aCx, nullptr));
+  if (!obj ||
+      !JS_DefineProperty(aCx, obj, "width", (double) width, JSPROP_ENUMERATE) ||
+      !JS_DefineProperty(aCx, obj, "height", (double) height, JSPROP_ENUMERATE))
+  {
+    return false;
+  }
+
+  args.rval().setObject(*obj);
+  return true;
+}
+
 static bool
 RecordReplay_Dump(JSContext* aCx, unsigned aArgc, Value* aVp)
 {
@@ -859,6 +910,8 @@ static const JSFunctionSpec gMiddlemanMethods[] = {
   JS_FN("setBreakpoint", Middleman_SetBreakpoint, 2, 0),
   JS_FN("clearBreakpoint", Middleman_ClearBreakpoint, 1, 0),
   JS_FN("maybeSwitchToReplayingChild", Middleman_MaybeSwitchToReplayingChild, 0, 0),
+  JS_FN("hadRepaint", Middleman_HadRepaint, 2, 0),
+  JS_FN("hadRepaintFailure", Middleman_HadRepaintFailure, 0, 0),
   JS_FS_END
 };
 
@@ -870,6 +923,7 @@ static const JSFunctionSpec gRecordReplayMethods[] = {
   JS_FN("getContent", RecordReplay_GetContent, 1, 0),
   JS_FN("currentExecutionPoint", RecordReplay_CurrentExecutionPoint, 1, 0),
   JS_FN("timeWarpTargetExecutionPoint", RecordReplay_TimeWarpTargetExecutionPoint, 1, 0),
+  JS_FN("repaint", RecordReplay_Repaint, 0, 0),
   JS_FN("dump", RecordReplay_Dump, 1, 0),
   JS_FS_END
 };

toolkit/recordreplay/ipc/ParentForwarding.cpp

@@ -77,11 +77,6 @@ HandleMessageInMiddleman(ipc::Side aSide, const IPC::Message& aMessage)
       // Preferences are initialized via the SetXPCOMProcessAttributes message.
       PreferencesLoaded();
     }
-    if (type == dom::PBrowser::Msg_RenderLayers__ID) {
-      // Graphics are being loaded or unloaded for a tab, so update what we are
-      // showing to the UI process according to the last paint performed.
-      UpdateGraphicsInUIProcess(nullptr);
-    }
     return false;
   }
 
@@ -115,6 +110,13 @@ HandleMessageInMiddleman(ipc::Side aSide, const IPC::Message& aMessage)
 static bool
 AlwaysForwardMessage(const IPC::Message& aMessage)
 {
+  // Always forward messages in repaint stress mode, as the active child is
+  // almost always a replaying child and lost messages make it hard to load
+  // pages completely.
+  if (InRepaintStressMode()) {
+    return true;
+  }
+
   IPC::Message::msgid_t type = aMessage.type();
 
   // Forward close messages so that the tab shuts down properly even if it is

toolkit/recordreplay/ipc/ParentGraphics.cpp

@@ -69,8 +69,6 @@ SendGraphicsMemoryToChild()
   MOZ_RELEASE_ASSERT(kr == KERN_SUCCESS);
 }
 
-static Maybe<PaintMessage> gLastPaint;
-
 // Global object for the sandbox used to paint graphics data in this process.
 static JS::PersistentRootedObject* gGraphicsSandbox;
 
@@ -109,17 +107,47 @@ InitGraphicsSandbox()
 // Buffer used to transform graphics memory, if necessary.
 static void* gBufferMemory;
 
+// The dimensions of the data in the graphics shmem buffer.
+static size_t gLastPaintWidth, gLastPaintHeight;
+
+// Explicit Paint messages received from the child need to be handled with
+// care to make sure we show correct graphics. Each Paint message is for the
+// the process state at the most recent checkpoint in the past. When running
+// (forwards or backwards) between the checkpoint and the Paint message,
+// we could pause at a breakpoint and repaint the graphics at that point,
+// reflecting the process state at a point later than at the checkpoint.
+// In this case the Paint message's graphics will be stale. To avoid showing
+// its graphics, we wait until both the Paint and the checkpoint itself have
+// been hit, with no intervening repaint.
+
+// The last explicit paint message received from the child, if there has not
+// been an intervening repaint.
+static UniquePtr<PaintMessage> gLastExplicitPaint;
+
+// The last checkpoint the child reached, if there has not been an intervening
+// repaint.
+static size_t gLastCheckpoint;
+
 void
 UpdateGraphicsInUIProcess(const PaintMessage* aMsg)
 {
   MOZ_RELEASE_ASSERT(NS_IsMainThread());
 
   if (aMsg) {
-    gLastPaint = Some(*aMsg);
-  } else if (!gLastPaint.isSome()) {
+    gLastPaintWidth = aMsg->mWidth;
+    gLastPaintHeight = aMsg->mHeight;
+  }
+
+  if (!gLastPaintWidth || !gLastPaintHeight) {
     return;
   }
 
+  bool hadFailure = !aMsg;
+
+  // Clear out the last explicit paint information. This may delete aMsg.
+  gLastExplicitPaint = nullptr;
+  gLastCheckpoint = CheckpointId::Invalid;
+
   // Make sure there is a sandbox which is running the graphics JS module.
   if (!gGraphicsSandbox) {
     InitGraphicsSandbox();
@@ -128,8 +156,8 @@ UpdateGraphicsInUIProcess(const PaintMessage* aMsg)
   AutoSafeJSContext cx;
   JSAutoRealm ar(cx, *gGraphicsSandbox);
 
-  size_t width = gLastPaint.ref().mWidth;
-  size_t height = gLastPaint.ref().mHeight;
+  size_t width = gLastPaintWidth;
+  size_t height = gLastPaintHeight;
   size_t stride = layers::ImageDataSerializer::ComputeRGBStride(gSurfaceFormat, width);
 
   // Make sure the width and height are appropriately sized.
@@ -160,10 +188,11 @@ UpdateGraphicsInUIProcess(const PaintMessage* aMsg)
     JS_NewArrayBufferWithExternalContents(cx, width * height * 4, memory);
   MOZ_RELEASE_ASSERT(bufferObject);
 
-  JS::AutoValueArray<3> args(cx);
+  JS::AutoValueArray<4> args(cx);
   args[0].setObject(*bufferObject);
   args[1].setInt32(width);
   args[2].setInt32(height);
+  args[3].setBoolean(hadFailure);
 
   // Call into the graphics module to update the canvas it manages.
   RootedValue rval(cx);
@@ -172,6 +201,41 @@ UpdateGraphicsInUIProcess(const PaintMessage* aMsg)
   }
 }
 
+static void
+MaybeTriggerExplicitPaint()
+{
+  if (gLastExplicitPaint && gLastExplicitPaint->mCheckpointId == gLastCheckpoint) {
+    UpdateGraphicsInUIProcess(gLastExplicitPaint.get());
+  }
+}
+
+void
+MaybeUpdateGraphicsAtPaint(const PaintMessage& aMsg)
+{
+  gLastExplicitPaint.reset(new PaintMessage(aMsg));
+  MaybeTriggerExplicitPaint();
+}
+
+void
+MaybeUpdateGraphicsAtCheckpoint(size_t aCheckpointId)
+{
+  gLastCheckpoint = aCheckpointId;
+  MaybeTriggerExplicitPaint();
+}
+
+bool
+InRepaintStressMode()
+{
+  static bool checked = false;
+  static bool rv;
+  if (!checked) {
+    AutoEnsurePassThroughThreadEvents pt;
+    rv = TestEnv("MOZ_RECORD_REPLAY_REPAINT_STRESS");
+    checked = true;
+  }
+  return rv;
+}
+
 } // namespace parent
 } // namespace recordreplay
 } // namespace mozilla

toolkit/recordreplay/ipc/ParentIPC.cpp

@@ -226,6 +226,7 @@ static void RecvHitBreakpoint(const HitBreakpointMessage& aMsg);
 static void RecvDebuggerResponse(const DebuggerResponseMessage& aMsg);
 static void RecvRecordingFlushed();
 static void RecvAlwaysMarkMajorCheckpoints();
+static void RecvMiddlemanCallRequest(const MiddlemanCallRequestMessage& aMsg);
 
 // The role taken by the active child.
 class ChildRoleActive final : public ChildRole
@@ -250,7 +251,7 @@ public:
   void OnIncomingMessage(const Message& aMsg) override {
     switch (aMsg.mType) {
     case MessageType::Paint:
-      UpdateGraphicsInUIProcess((const PaintMessage*) &aMsg);
+      MaybeUpdateGraphicsAtPaint((const PaintMessage&) aMsg);
       break;
     case MessageType::HitCheckpoint:
       RecvHitCheckpoint((const HitCheckpointMessage&) aMsg);
@@ -267,6 +268,12 @@ public:
     case MessageType::AlwaysMarkMajorCheckpoints:
       RecvAlwaysMarkMajorCheckpoints();
       break;
+    case MessageType::MiddlemanCallRequest:
+      RecvMiddlemanCallRequest((const MiddlemanCallRequestMessage&) aMsg);
+      break;
+    case MessageType::ResetMiddlemanCalls:
+      ResetMiddlemanCalls();
+      break;
     default:
       MOZ_CRASH("Unexpected message");
     }
@@ -970,11 +977,45 @@ static bool gResumeForwardOrBackward = false;
 // Hit any breakpoints installed for forced pauses.
 static void HitForcedPauseBreakpoints(bool aRecordingBoundary);
 
+static void
+MaybeSendRepaintMessage()
+{
+  // In repaint stress mode, we want to trigger a repaint at every checkpoint,
+  // so before resuming after the child pauses at each checkpoint, send it a
+  // repaint message. There might not be a debugger open, so manually craft the
+  // same message which the debugger would send to trigger a repaint and parse
+  // the result.
+  if (InRepaintStressMode()) {
+    MaybeSwitchToReplayingChild();
+
+    const char16_t contents[] = u"{\"type\":\"repaint\"}";
+
+    js::CharBuffer request, response;
+    request.append(contents, ArrayLength(contents) - 1);
+    SendRequest(request, &response);
+
+    AutoSafeJSContext cx;
+    JS::RootedValue value(cx);
+    if (JS_ParseJSON(cx, response.begin(), response.length(), &value)) {
+      MOZ_RELEASE_ASSERT(value.isObject());
+      JS::RootedObject obj(cx, &value.toObject());
+      RootedValue width(cx), height(cx);
+      if (JS_GetProperty(cx, obj, "width", &width) && width.isNumber() && width.toNumber() &&
+          JS_GetProperty(cx, obj, "height", &height) && height.isNumber() && height.toNumber()) {
+        PaintMessage message(CheckpointId::Invalid, width.toNumber(), height.toNumber());
+        UpdateGraphicsInUIProcess(&message);
+      }
+    }
+  }
+}
+
 void
 Resume(bool aForward)
 {
   gActiveChild->WaitUntilPaused();
 
+  MaybeSendRepaintMessage();
+
   // Set the preferred direction of travel.
   gResumeForwardOrBackward = false;
   gChildExecuteForward = aForward;
@@ -1125,6 +1166,7 @@ static void
 RecvHitCheckpoint(const HitCheckpointMessage& aMsg)
 {
   UpdateCheckpointTimes(aMsg);
+  MaybeUpdateGraphicsAtCheckpoint(aMsg.mCheckpointId);
 
   // Resume either forwards or backwards. Break the resume off into a separate
   // runnable, to avoid starving any code already on the stack and waiting for
@@ -1199,6 +1241,18 @@ HitForcedPauseBreakpoints(bool aRecordingBoundary)
   }
 }
 
+static void
+RecvMiddlemanCallRequest(const MiddlemanCallRequestMessage& aMsg)
+{
+  InfallibleVector<char> outputData;
+  ProcessMiddlemanCall(aMsg.BinaryData(), aMsg.BinaryDataSize(), &outputData);
+
+  MiddlemanCallResponseMessage* response =
+    MiddlemanCallResponseMessage::New(outputData.begin(), outputData.length());
+  gActiveChild->SendMessage(*response);
+  free(response);
+}
+
 } // namespace parent
 } // namespace recordreplay
 } // namespace mozilla

toolkit/recordreplay/ipc/ParentInternal.h

@@ -86,9 +86,15 @@ void InitializeGraphicsMemory();
 void SendGraphicsMemoryToChild();
 
 // Update the graphics painted in the UI process, per painting data received
-// from a child process, or null for the last paint performed.
+// from a child process, or null if a repaint was triggered and failed due to
+// an unhandled recording divergence.
 void UpdateGraphicsInUIProcess(const PaintMessage* aMsg);
 
+// If necessary, update graphics after the active child sends a paint message
+// or reaches a checkpoint.
+void MaybeUpdateGraphicsAtPaint(const PaintMessage& aMsg);
+void MaybeUpdateGraphicsAtCheckpoint(size_t aCheckpointId);
+
 // ID for the mach message sent from a child process to the middleman to
 // request a port for the graphics shmem.
 static const int32_t GraphicsHandshakeMessageId = 42;
@@ -100,6 +106,13 @@ static const int32_t GraphicsMemoryMessageId = 43;
 // Fixed size of the graphics shared memory buffer.
 static const size_t GraphicsMemorySize = 4096 * 4096 * 4;
 
+// Return whether the environment variable activating repaint stress mode is
+// set. This makes various changes in both the middleman and child processes to
+// trigger a child to diverge from the recording and repaint on every vsync,
+// making sure that repainting can handle all the system interactions that
+// occur while painting the current tab.
+bool InRepaintStressMode();
+
 ///////////////////////////////////////////////////////////////////////////////
 // Child Processes
 ///////////////////////////////////////////////////////////////////////////////

toolkit/recordreplay/moz.build

@@ -26,6 +26,7 @@ if CONFIG['OS_ARCH'] == 'Darwin' and CONFIG['NIGHTLY_BUILD']:
         'ipc/ParentIPC.cpp',
         'Lock.cpp',
         'MemorySnapshot.cpp',
+        'MiddlemanCall.cpp',
         'ProcessRecordReplay.cpp',
         'ProcessRedirectDarwin.cpp',
         'ProcessRewind.cpp',

xpcom/threads/ThreadEventTarget.cpp

@@ -134,6 +134,14 @@ ThreadEventTarget::Dispatch(already_AddRefed<nsIRunnable> aEvent, uint32_t aFlag
     return NS_ERROR_ILLEGAL_DURING_SHUTDOWN;
   }
 
+  // Don't dispatch runnables to other threads when replaying and diverged from
+  // the recording, to avoid deadlocking with other idle threads. The browser
+  // remains paused after diverging from the recording, and threads will not
+  // run their event loops.
+  if (recordreplay::HasDivergedFromRecording()) {
+    return NS_ERROR_FAILURE;
+  }
+
 #ifdef MOZ_TASK_TRACER
   nsCOMPtr<nsIRunnable> tracedRunnable = CreateTracedRunnable(event.take());
   (static_cast<TracedRunnable*>(tracedRunnable.get()))->DispatchTask();

xpcom/threads/nsThread.cpp

@@ -1123,13 +1123,6 @@ nsThread::ProcessNextEvent(bool aMayWait, bool* aResult)
     return NS_ERROR_NOT_SAME_THREAD;
   }
 
-  // When recording or replaying, vsync observers are notified whenever
-  // processing events on the main thread. Waiting for explicit vsync messages
-  // from the UI process can result in paints happening at unexpected times.
-  if (recordreplay::IsRecordingOrReplaying() && mIsMainThread == MAIN_THREAD) {
-    recordreplay::child::NotifyVsyncObserver();
-  }
-
   // The toplevel event loop normally blocks waiting for the next event, but
   // if we're trying to shut this thread down, we must exit the event loop when
   // the event queue is empty.