Bug 1199121. Part 1 - add the ability to handle video-only streams to AudioSinkWrapper. r=kinetik.

Note AudioSinkWrapper can now report correct playback position when the media is video-only or audio-only.
We will handle the case where audio ends before video in next patch where it needs to switch to system clock when audio reaches the end.

dom/media/MediaDecoderStateMachine.cpp

@@ -1761,15 +1761,17 @@ MediaDecoderStateMachine::StartAudioSink()
     return;
   }
 
-  if (HasAudio() && !mAudioSink->IsStarted()) {
+  if (!mAudioSink->IsStarted()) {
     mAudioCompleted = false;
     mAudioSink->Start(GetMediaTime(), mInfo);
 
-    mAudioSinkPromise.Begin(
-      mAudioSink->OnEnded(TrackInfo::kAudioTrack)->Then(
+    auto promise = mAudioSink->OnEnded(TrackInfo::kAudioTrack);
+    if (promise) {
+      mAudioSinkPromise.Begin(promise->Then(
         OwnerThread(), __func__, this,
         &MediaDecoderStateMachine::OnAudioSinkComplete,
         &MediaDecoderStateMachine::OnAudioSinkError));
+    }
   }
 }
 
@@ -2620,12 +2622,8 @@ int64_t MediaDecoderStateMachine::GetClock(TimeStamp* aTimeStamp) const
   } else {
     if (mAudioCaptured) {
       clock_time = GetStreamClock();
-    } else if (HasAudio() && !mAudioCompleted) {
-      clock_time = GetAudioClock();
     } else {
-      t = TimeStamp::Now();
-      // Audio is disabled on this system. Sync to the system clock.
-      clock_time = GetVideoStreamPosition(t);
+      clock_time = mAudioSink->GetPosition(&t);
     }
     NS_ASSERTION(GetMediaTime() <= clock_time, "Clock should go forwards.");
   }
@@ -2998,17 +2996,6 @@ MediaDecoderStateMachine::LogicalPlaybackRateChanged()
     return;
   }
 
-  // AudioStream will handle playback rate change when we have audio.
-  // Do nothing while we are not playing. Change in playback rate will
-  // take effect next time we start playing again.
-  if (!HasAudio() && IsPlaying()) {
-    // Remember how much time we've spent in playing the media
-    // for playback rate will change from now on.
-    TimeStamp now = TimeStamp::Now();
-    mPlayDuration = GetVideoStreamPosition(now);
-    SetPlayStartTime(now);
-  }
-
   mPlaybackRate = mLogicalPlaybackRate;
   mAudioSink->SetPlaybackRate(mPlaybackRate);
 
@@ -3049,7 +3036,6 @@ void MediaDecoderStateMachine::OnAudioSinkComplete()
   MOZ_ASSERT(!mAudioCaptured, "Should be disconnected when capturing audio.");
 
   mAudioSinkPromise.Complete();
-  ResyncAudioClock();
   mAudioCompleted = true;
 }
 
@@ -3060,7 +3046,6 @@ void MediaDecoderStateMachine::OnAudioSinkError()
   MOZ_ASSERT(!mAudioCaptured, "Should be disconnected when capturing audio.");
 
   mAudioSinkPromise.Complete();
-  ResyncAudioClock();
   mAudioCompleted = true;
 
   // Make the best effort to continue playback when there is video.

dom/media/mediasink/AudioSinkWrapper.cpp

@@ -64,11 +64,41 @@ AudioSinkWrapper::GetEndTime(TrackType aType) const
 }
 
 int64_t
-AudioSinkWrapper::GetPosition() const
+AudioSinkWrapper::GetVideoPosition(TimeStamp aNow) const
+{
+  AssertOwnerThread();
+  MOZ_ASSERT(!mPlayStartTime.IsNull());
+  // Time elapsed since we started playing.
+  int64_t delta = (aNow - mPlayStartTime).ToMicroseconds();
+  // Take playback rate into account.
+  return mPlayDuration + delta * mParams.playbackRate;
+}
+
+int64_t
+AudioSinkWrapper::GetPosition(TimeStamp* aTimeStamp) const
 {
   AssertOwnerThread();
   MOZ_ASSERT(mIsStarted, "Must be called after playback starts.");
-  return mAudioSink->GetPosition();
+
+  int64_t pos = -1;
+  TimeStamp t = TimeStamp::Now();
+
+  if (mAudioSink) {
+    // Rely on the audio sink to report playback position when we have one.
+    pos = mAudioSink->GetPosition();
+  } else if (!mPlayStartTime.IsNull()) {
+    // Calculate playback position using system clock if we are still playing.
+    pos = GetVideoPosition(t);
+  } else {
+    // Return how long we've played if we are not playing.
+    pos = mPlayDuration;
+  }
+
+  if (aTimeStamp) {
+    *aTimeStamp = t;
+  }
+
+  return pos;
 }
 
 bool
@@ -94,8 +124,17 @@ AudioSinkWrapper::SetPlaybackRate(double aPlaybackRate)
   AssertOwnerThread();
   mParams.playbackRate = aPlaybackRate;
   if (mAudioSink) {
+    // Pass the playback rate to the audio sink. The underlying AudioStream
+    // will handle playback rate changes and report correct audio position.
     mAudioSink->SetPlaybackRate(aPlaybackRate);
+  } else if (!mPlayStartTime.IsNull()) {
+    // Adjust playback duration and start time when we are still playing.
+    TimeStamp now = TimeStamp::Now();
+    mPlayDuration = GetVideoPosition(now);
+    mPlayStartTime = now;
   }
+  // Do nothing when not playing. Changes in playback rate will be taken into
+  // account by GetVideoPosition().
 }
 
 void
@@ -121,6 +160,17 @@ AudioSinkWrapper::SetPlaying(bool aPlaying)
   if (mAudioSink) {
     mAudioSink->SetPlaying(aPlaying);
   }
+
+  if (aPlaying) {
+    MOZ_ASSERT(mPlayStartTime.IsNull());
+    mPlayStartTime = TimeStamp::Now();
+  } else {
+    // Remember how long we've played.
+    mPlayDuration = GetPosition();
+    // mPlayStartTime must be updated later since GetPosition()
+    // depends on the value of mPlayStartTime.
+    mPlayStartTime = TimeStamp();
+  }
 }
 
 void
@@ -130,10 +180,14 @@ AudioSinkWrapper::Start(int64_t aStartTime, const MediaInfo& aInfo)
   MOZ_ASSERT(!mIsStarted, "playback already started.");
 
   mIsStarted = true;
+  mPlayDuration = aStartTime;
+  mPlayStartTime = TimeStamp::Now();
 
-  mAudioSink = mCreator->Create();
-  mEndPromise = mAudioSink->Init();
-  SetPlaybackParams(mParams);
+  if (aInfo.HasAudio()) {
+    mAudioSink = mCreator->Create();
+    mEndPromise = mAudioSink->Init();
+    SetPlaybackParams(mParams);
+  }
 }
 
 void
@@ -143,9 +197,12 @@ AudioSinkWrapper::Stop()
   MOZ_ASSERT(mIsStarted, "playback not started.");
 
   mIsStarted = false;
-  mAudioSink->Shutdown();
-  mAudioSink = nullptr;
-  mEndPromise = nullptr;
+
+  if (mAudioSink) {
+    mAudioSink->Shutdown();
+    mAudioSink = nullptr;
+    mEndPromise = nullptr;
+  }
 }
 
 bool

dom/media/mediasink/AudioSinkWrapper.h

@@ -10,6 +10,7 @@
 #include "mozilla/AbstractThread.h"
 #include "mozilla/dom/AudioChannelBinding.h"
 #include "mozilla/nsRefPtr.h"
+#include "mozilla/TimeStamp.h"
 #include "mozilla/UniquePtr.h"
 
 #include "MediaSink.h"
@@ -50,6 +51,8 @@ public:
     : mOwnerThread(aOwnerThread)
     , mCreator(new CreatorImpl<Function>(aFunc))
     , mIsStarted(false)
+    // Give an insane value to facilitate debug if used before playback starts.
+    , mPlayDuration(INT64_MAX)
   {}
 
   const PlaybackParams& GetPlaybackParams() const override;
@@ -57,7 +60,7 @@ public:
 
   nsRefPtr<GenericPromise> OnEnded(TrackType aType) override;
   int64_t GetEndTime(TrackType aType) const override;
-  int64_t GetPosition() const override;
+  int64_t GetPosition(TimeStamp* aTimeStamp = nullptr) const override;
   bool HasUnplayedFrames(TrackType aType) const override;
 
   void SetVolume(double aVolume) override;
@@ -78,6 +81,8 @@ private:
     MOZ_ASSERT(mOwnerThread->IsCurrentThreadIn());
   }
 
+  int64_t GetVideoPosition(TimeStamp aNow) const;
+
   const nsRefPtr<AbstractThread> mOwnerThread;
   UniquePtr<Creator> mCreator;
   nsRefPtr<AudioSink> mAudioSink;
@@ -85,6 +90,9 @@ private:
 
   bool mIsStarted;
   PlaybackParams mParams;
+
+  TimeStamp mPlayStartTime;
+  int64_t mPlayDuration;
 };
 
 } // namespace media

dom/media/mediasink/MediaSink.h

@@ -13,6 +13,9 @@
 #include "MediaInfo.h"
 
 namespace mozilla {
+
+class TimeStamp;
+
 namespace media {
 
 /**
@@ -64,8 +67,10 @@ public:
   // Return playback position of the media.
   // Since A/V sync is always maintained by this sink, there is no need to
   // specify whether we want to get audio or video position.
+  // aTimeStamp returns the timeStamp corresponding to the returned position
+  // which is used by the compositor to derive the render time of video frames.
   // Must be called after playback starts.
-  virtual int64_t GetPosition() const = 0;
+  virtual int64_t GetPosition(TimeStamp* aTimeStamp = nullptr) const = 0;
 
   // Return true if there are data consumed but not played yet.
   // Can be called in any state.