Bug 1653737 - Add output time to VsyncEvent. r=nical

This timestamp is provided by the system on macOS, and estimated as "the next
vsync" on all other platforms.

On macOS, the output time increments in very consistent amounts. The timestamp
is independent of when exactly the vsync callback ends up running, so it is less
vulnerable to unfortunate thread scheduling.
This makes it a more reliable source for picking video frames, for example.

Differential Revision: https://phabricator.services.mozilla.com/D83828

gfx/layers/ipc/LayersMessageUtils.h

@@ -61,11 +61,13 @@ struct ParamTraits<mozilla::VsyncEvent> {
   static void Write(Message* msg, const paramType& param) {
     WriteParam(msg, param.mId);
     WriteParam(msg, param.mTime);
+    WriteParam(msg, param.mOutputTime);
   }
   static bool Read(const Message* msg, PickleIterator* iter,
                    paramType* result) {
     return ReadParam(msg, iter, &result->mId) &&
-           ReadParam(msg, iter, &result->mTime);
+           ReadParam(msg, iter, &result->mTime) &&
+           ReadParam(msg, iter, &result->mOutputTime);
   }
 };
 

gfx/thebes/SoftwareVsyncSource.cpp

@@ -47,7 +47,9 @@ void SoftwareDisplay::EnableVsync() {
   }
 
   MOZ_ASSERT(IsInSoftwareVsyncThread());
-  NotifyVsync(mozilla::TimeStamp::Now());
+  TimeStamp vsyncTime = TimeStamp::Now();
+  TimeStamp outputTime = vsyncTime + mVsyncRate;
+  NotifyVsync(vsyncTime, outputTime);
 }
 
 void SoftwareDisplay::DisableVsync() {
@@ -79,7 +81,8 @@ bool SoftwareDisplay::IsInSoftwareVsyncThread() {
   return mVsyncThread->thread_id() == PlatformThread::CurrentId();
 }
 
-void SoftwareDisplay::NotifyVsync(mozilla::TimeStamp aVsyncTimestamp) {
+void SoftwareDisplay::NotifyVsync(const mozilla::TimeStamp& aVsyncTimestamp,
+                                  const mozilla::TimeStamp& aOutputTimestamp) {
   MOZ_ASSERT(IsInSoftwareVsyncThread());
 
   mozilla::TimeStamp displayVsyncTime = aVsyncTimestamp;
@@ -93,7 +96,7 @@ void SoftwareDisplay::NotifyVsync(mozilla::TimeStamp aVsyncTimestamp) {
     displayVsyncTime = now;
   }
 
-  Display::NotifyVsync(displayVsyncTime);
+  Display::NotifyVsync(displayVsyncTime, aOutputTimestamp);
 
   // Prevent skew by still scheduling based on the original
   // vsync timestamp
@@ -111,9 +114,12 @@ void SoftwareDisplay::ScheduleNextVsync(mozilla::TimeStamp aVsyncTimestamp) {
     nextVsync = mozilla::TimeStamp::Now();
   }
 
-  mCurrentVsyncTask = NewCancelableRunnableMethod<mozilla::TimeStamp>(
-      "SoftwareDisplay::NotifyVsync", this, &SoftwareDisplay::NotifyVsync,
-      nextVsync);
+  TimeStamp outputTime = nextVsync + mVsyncRate;
+
+  mCurrentVsyncTask =
+      NewCancelableRunnableMethod<mozilla::TimeStamp, mozilla::TimeStamp>(
+          "SoftwareDisplay::NotifyVsync", this, &SoftwareDisplay::NotifyVsync,
+          nextVsync, outputTime);
 
   RefPtr<Runnable> addrefedTask = mCurrentVsyncTask;
   mVsyncThread->message_loop()->PostDelayedTask(addrefedTask.forget(),

gfx/thebes/SoftwareVsyncSource.h

@@ -21,7 +21,8 @@ class SoftwareDisplay final : public mozilla::gfx::VsyncSource::Display {
   void DisableVsync() override;
   bool IsVsyncEnabled() override;
   bool IsInSoftwareVsyncThread();
-  void NotifyVsync(mozilla::TimeStamp aVsyncTimestamp) override;
+  void NotifyVsync(const mozilla::TimeStamp& aVsyncTimestamp,
+                   const mozilla::TimeStamp& aOutputTimestamp) override;
   mozilla::TimeDuration GetVsyncRate() override;
   void ScheduleNextVsync(mozilla::TimeStamp aVsyncTimestamp);
   void Shutdown() override;

gfx/thebes/VsyncSource.cpp

@@ -88,7 +88,8 @@ VsyncSource::Display::~Display() {
   MOZ_ASSERT(mEnabledCompositorVsyncDispatchers.Length() == 0);
 }
 
-void VsyncSource::Display::NotifyVsync(TimeStamp aVsyncTimestamp) {
+void VsyncSource::Display::NotifyVsync(const TimeStamp& aVsyncTimestamp,
+                                       const TimeStamp& aOutputTimestamp) {
   // Called on the vsync thread
   MutexAutoLock lock(mDispatcherLock);
 
@@ -107,7 +108,7 @@ void VsyncSource::Display::NotifyVsync(TimeStamp aVsyncTimestamp) {
       (mLastVsyncIdSentToMainThread == mLastMainThreadProcessedVsyncId);
 
   mVsyncId = mVsyncId.Next();
-  const VsyncEvent event(mVsyncId, aVsyncTimestamp);
+  const VsyncEvent event(mVsyncId, aVsyncTimestamp, aOutputTimestamp);
 
   for (size_t i = 0; i < mEnabledCompositorVsyncDispatchers.Length(); i++) {
     mEnabledCompositorVsyncDispatchers[i]->NotifyVsync(event);

gfx/thebes/VsyncSource.h

@@ -39,16 +39,23 @@ class VsyncSource {
    public:
     Display();
 
-    // Notified when this display's vsync occurs, on the vsync thread
-    // The aVsyncTimestamp should normalize to the Vsync time that just occured
-    // However, different platforms give different vsync notification times.
-    // OSX - The vsync timestamp of the upcoming frame, in the future
+    // Notified when this display's vsync callback occurs, on the vsync thread
+    // Different platforms give different aVsyncTimestamp values.
+    // macOS: TimeStamp::Now() or the output time of the previous vsync
+    //        callback, whichever is older.
     // Windows: It's messy, see gfxWindowsPlatform.
     // Android: TODO
-    // All platforms should normalize to the vsync that just occured.
-    // Large parts of Gecko assume TimeStamps should not be in the future such
-    // as animations
-    virtual void NotifyVsync(TimeStamp aVsyncTimestamp);
+    //
+    // @param aVsyncTimestamp  The time of the Vsync that just occured. Needs to
+    //     be at or before the time of the NotifyVsync call.
+    // @param aOutputTimestamp  The estimated timestamp at which drawing will
+    //     appear on the screen, if the drawing happens within a certain
+    //     (unknown) budget. Useful for Audio/Video sync. On platforms where
+    //     this timestamp is provided by the system (macOS), it is a much more
+    //     stable and consistent timing source than the time at which the vsync
+    //     callback is called.
+    virtual void NotifyVsync(const TimeStamp& aVsyncTimestamp,
+                             const TimeStamp& aOutputTimestamp);
     void NotifyGenericObservers(VsyncEvent aEvent);
 
     RefPtr<RefreshTimerVsyncDispatcher> GetRefreshTimerVsyncDispatcher();
@@ -126,9 +133,11 @@ class VsyncSource {
 struct VsyncEvent {
   VsyncId mId;
   TimeStamp mTime;
+  TimeStamp mOutputTime;  // estimate
 
-  VsyncEvent(const VsyncId& aId, const TimeStamp& aTime)
-      : mId(aId), mTime(aTime) {}
+  VsyncEvent(const VsyncId& aId, const TimeStamp& aVsyncTime,
+             const TimeStamp& aOutputTime)
+      : mId(aId), mTime(aVsyncTime), mOutputTime(aOutputTime) {}
   VsyncEvent() = default;
 };
 

gfx/thebes/gfxAndroidPlatform.cpp

@@ -297,7 +297,10 @@ class AndroidVsyncSource final : public VsyncSource {
     using Base::DisposeNative;
 
     static void NotifyVsync() {
-      GetDisplayInstance().NotifyVsync(TimeStamp::Now());
+      Display& display = GetDisplayInstance();
+      TimeStamp vsyncTime = TimeStamp::Now();
+      TimeStamp outputTime = vsyncTime + display.GetVsyncRate();
+      display.NotifyVsync(vsyncTime, outputTime);
     }
   };
 

gfx/thebes/gfxPlatformGtk.cpp

@@ -666,7 +666,8 @@ class GtkVsyncSource final : public VsyncSource {
         }
 
         lastVsync = TimeStamp::Now();
-        NotifyVsync(lastVsync);
+        TimeStamp outputTime = lastVsync + GetVsyncRate();
+        NotifyVsync(lastVsync, outputTime);
       }
     }
 

gfx/thebes/gfxPlatformMac.cpp

@@ -518,10 +518,10 @@ static CVReturn VsyncCallback(CVDisplayLinkRef aDisplayLink,
   // Executed on OS X hardware vsync thread
   OSXVsyncSource::OSXDisplay* display =
       (OSXVsyncSource::OSXDisplay*)aDisplayLinkContext;
-  int64_t nextVsyncTimestamp = aOutputTime->hostTime;
 
-  mozilla::TimeStamp nextVsync =
-      mozilla::TimeStamp::FromSystemTime(nextVsyncTimestamp);
+  mozilla::TimeStamp outputTime =
+      mozilla::TimeStamp::FromSystemTime(aOutputTime->hostTime);
+  mozilla::TimeStamp nextVsync = outputTime;
   mozilla::TimeStamp previousVsync = display->mPreviousTimestamp;
   mozilla::TimeStamp now = TimeStamp::Now();
 
@@ -540,7 +540,7 @@ static CVReturn VsyncCallback(CVDisplayLinkRef aDisplayLink,
 
   display->mPreviousTimestamp = nextVsync;
 
-  display->NotifyVsync(previousVsync);
+  display->NotifyVsync(previousVsync, outputTime);
   return kCVReturnSuccess;
 }
 

gfx/thebes/gfxWindowsPlatform.cpp

@@ -1820,7 +1820,7 @@ class D3DVsyncSource final : public VsyncSource {
         // Large parts of gecko assume that the refresh driver timestamp
         // must be <= Now() and cannot be in the future.
         MOZ_ASSERT(vsync <= TimeStamp::Now());
-        Display::NotifyVsync(vsync);
+        Display::NotifyVsync(vsync, vsync + mVsyncRate);
 
         // DwmComposition can be dynamically enabled/disabled
         // so we have to check every time that it's available.

widget/gtk/WaylandVsyncSource.cpp

@@ -72,7 +72,9 @@ void WaylandVsyncSource::WaylandDisplay::Refresh() {
   // Vsync is enabled, but we don't have a callback configured. Set one up so
   // we can get to work.
   SetupFrameCallback();
-  NotifyVsync(TimeStamp::Now());
+  TimeStamp vsyncTimestamp = TimeStamp::Now();
+  TimeStamp outputTimestamp = vsyncTimestamp + GetVsyncRate();
+  NotifyVsync(vsyncTimestamp, outputTimestamp);
 }
 
 void WaylandVsyncSource::WaylandDisplay::EnableMonitor() {
@@ -127,7 +129,9 @@ void WaylandVsyncSource::WaylandDisplay::FrameCallback() {
     SetupFrameCallback();
   }
 
-  NotifyVsync(TimeStamp::Now());
+  TimeStamp vsyncTimestamp = TimeStamp::Now();
+  TimeStamp outputTimestamp = vsyncTimestamp + GetVsyncRate();
+  NotifyVsync(vsyncTimestamp, outputTimestamp);
 }
 
 void WaylandVsyncSource::WaylandDisplay::EnableVsync() {