gecko-dev/dom/media/MediaStreamGraph.h

1416 строки
51 KiB
C
Исходник Обычный вид История

/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*-*/
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this file,
* You can obtain one at http://mozilla.org/MPL/2.0/. */
#ifndef MOZILLA_MEDIASTREAMGRAPH_H_
#define MOZILLA_MEDIASTREAMGRAPH_H_
#include "AudioStream.h"
#include "MainThreadUtils.h"
#include "MediaStreamTypes.h"
#include "StreamTracks.h"
#include "VideoSegment.h"
#include "mozilla/LinkedList.h"
#include "mozilla/Mutex.h"
#include "mozilla/TaskQueue.h"
#include "nsAutoPtr.h"
#include "nsAutoRef.h"
#include "nsIRunnable.h"
#include "nsTArray.h"
#include <speex/speex_resampler.h>
class nsIRunnable;
class nsIGlobalObject;
class nsPIDOMWindowInner;
template <>
class nsAutoRefTraits<SpeexResamplerState> : public nsPointerRefTraits<SpeexResamplerState>
{
public:
static void Release(SpeexResamplerState* aState) { speex_resampler_destroy(aState); }
};
namespace mozilla {
extern LazyLogModule gMediaStreamGraphLog;
namespace dom {
enum class AudioContextOperation;
}
namespace media {
template<typename V, typename E> class Pledge;
}
/*
* MediaStreamGraph is a framework for synchronized audio/video processing
* and playback. It is designed to be used by other browser components such as
* HTML media elements, media capture APIs, real-time media streaming APIs,
* multitrack media APIs, and advanced audio APIs.
*
* The MediaStreamGraph uses a dedicated thread to process media --- the media
* graph thread. This ensures that we can process media through the graph
* without blocking on main-thread activity. The media graph is only modified
* on the media graph thread, to ensure graph changes can be processed without
* interfering with media processing. All interaction with the media graph
* thread is done with message passing.
*
* APIs that modify the graph or its properties are described as "control APIs".
* These APIs are asynchronous; they queue graph changes internally and
* those changes are processed all-at-once by the MediaStreamGraph. The
* MediaStreamGraph monitors the main thread event loop via nsIAppShell::RunInStableState
* to ensure that graph changes from a single event loop task are always
* processed all together. Control APIs should only be used on the main thread,
* currently; we may be able to relax that later.
*
* To allow precise synchronization of times in the control API, the
* MediaStreamGraph maintains a "media timeline". Control APIs that take or
* return times use that timeline. Those times never advance during
* an event loop task. This time is returned by MediaStreamGraph::GetCurrentTime().
*
* Media decoding, audio processing and media playback use thread-safe APIs to
* the media graph to ensure they can continue while the main thread is blocked.
*
* When the graph is changed, we may need to throw out buffered data and
* reprocess it. This is triggered automatically by the MediaStreamGraph.
*/
class AudioNodeEngine;
class AudioNodeExternalInputStream;
class AudioNodeStream;
class MediaInputPort;
class MediaStream;
class MediaStreamGraph;
class MediaStreamGraphImpl;
class ProcessedMediaStream;
class SourceMediaStream;
class AudioDataListenerInterface {
protected:
// Protected destructor, to discourage deletion outside of Release():
virtual ~AudioDataListenerInterface() {}
public:
/* These are for cubeb audio input & output streams: */
/**
* Output data to speakers, for use as the "far-end" data for echo
* cancellation. This is not guaranteed to be in any particular size
* chunks.
*/
virtual void NotifyOutputData(MediaStreamGraph* aGraph,
AudioDataValue* aBuffer, size_t aFrames,
TrackRate aRate, uint32_t aChannels) = 0;
/**
* Input data from a microphone (or other audio source. This is not
* guaranteed to be in any particular size chunks.
*/
virtual void NotifyInputData(MediaStreamGraph* aGraph,
const AudioDataValue* aBuffer, size_t aFrames,
TrackRate aRate, uint32_t aChannels) = 0;
/**
* Called when the underlying audio device has changed.
*/
virtual void DeviceChanged() = 0;
};
class AudioDataListener : public AudioDataListenerInterface {
protected:
// Protected destructor, to discourage deletion outside of Release():
virtual ~AudioDataListener() {}
public:
NS_INLINE_DECL_THREADSAFE_REFCOUNTING(AudioDataListener)
};
/**
* This is a base class for main-thread listener callbacks.
* This callback is invoked on the main thread when the main-thread-visible
* state of a stream has changed.
*
* These methods are called with the media graph monitor held, so
* reentry into general media graph methods is not possible.
* You should do something non-blocking and non-reentrant (e.g. dispatch an
* event) and return. DispatchFromMainThreadAfterNextStreamStateUpdate
* would be a good choice.
* The listener is allowed to synchronously remove itself from the stream, but
* not add or remove any other listeners.
*/
class MainThreadMediaStreamListener {
public:
virtual void NotifyMainThreadStreamFinished() = 0;
};
/**
* Helper struct used to keep track of memory usage by AudioNodes.
*/
struct AudioNodeSizes
{
AudioNodeSizes() : mStream(0), mEngine(0), mNodeType() {}
size_t mStream;
size_t mEngine;
const char* mNodeType;
};
class AudioNodeEngine;
class AudioNodeExternalInputStream;
class AudioNodeStream;
class AudioSegment;
class DirectMediaStreamTrackListener;
class MediaInputPort;
class MediaStreamGraphImpl;
class MediaStreamListener;
class MediaStreamTrackListener;
class MediaStreamVideoSink;
class ProcessedMediaStream;
class SourceMediaStream;
class TrackUnionStream;
/**
* Helper struct for binding a track listener to a specific TrackID.
*/
template<typename Listener>
struct TrackBound
{
RefPtr<Listener> mListener;
TrackID mTrackID;
};
/**
* A stream of synchronized audio and video data. All (not blocked) streams
* progress at the same rate --- "real time". Streams cannot seek. The only
* operation readers can perform on a stream is to read the next data.
*
* Consumers of a stream can be reading from it at different offsets, but that
* should only happen due to the order in which consumers are being run.
* Those offsets must not diverge in the long term, otherwise we would require
* unbounded buffering.
*
* Streams can be in a "blocked" state. While blocked, a stream does not
* produce data. A stream can be explicitly blocked via the control API,
* or implicitly blocked by whatever's generating it (e.g. an underrun in the
* source resource), or implicitly blocked because something consuming it
* blocks, or implicitly because it has finished.
*
* A stream can be in a "finished" state. "Finished" streams are permanently
* blocked.
*
* Transitions into and out of the "blocked" and "finished" states are managed
* by the MediaStreamGraph on the media graph thread.
*
* We buffer media data ahead of the consumers' reading offsets. It is possible
* to have buffered data but still be blocked.
*
* Any stream can have its audio and video playing when requested. The media
* stream graph plays audio by constructing audio output streams as necessary.
* Video is played by setting video frames into an MediaStreamVideoSink at the right
* time. To ensure video plays in sync with audio, make sure that the same
* stream is playing both the audio and video.
*
* The data in a stream is managed by StreamTracks. It consists of a set of
* tracks of various types that can start and end over time.
*
* Streams are explicitly managed. The client creates them via
* MediaStreamGraph::CreateInput/ProcessedMediaStream, and releases them by calling
* Destroy() when no longer needed (actual destruction will be deferred).
* The actual object is owned by the MediaStreamGraph. The basic idea is that
* main thread objects will keep Streams alive as long as necessary (using the
* cycle collector to clean up whenever needed).
*
* We make them refcounted only so that stream-related messages with MediaStream*
* pointers can be sent to the main thread safely.
*
* The lifetimes of MediaStreams are controlled from the main thread.
* For MediaStreams exposed to the DOM, the lifetime is controlled by the DOM
* wrapper; the DOM wrappers own their associated MediaStreams. When a DOM
* wrapper is destroyed, it sends a Destroy message for the associated
* MediaStream and clears its reference (the last main-thread reference to
* the object). When the Destroy message is processed on the graph manager
* thread we immediately release the affected objects (disentangling them
* from other objects as necessary).
*
* This could cause problems for media processing if a MediaStream is
* destroyed while a downstream MediaStream is still using it. Therefore
* the DOM wrappers must keep upstream MediaStreams alive as long as they
* could be being used in the media graph.
*
* At any time, however, a set of MediaStream wrappers could be
* collected via cycle collection. Destroy messages will be sent
* for those objects in arbitrary order and the MediaStreamGraph has to be able
* to handle this.
*/
// GetCurrentTime is defined in winbase.h as zero argument macro forwarding to
// GetTickCount() and conflicts with MediaStream::GetCurrentTime.
#ifdef GetCurrentTime
#undef GetCurrentTime
#endif
class MediaStream : public mozilla::LinkedListElement<MediaStream>
{
public:
NS_INLINE_DECL_THREADSAFE_REFCOUNTING(MediaStream)
explicit MediaStream();
protected:
// Protected destructor, to discourage deletion outside of Release():
virtual ~MediaStream();
public:
/**
* Returns the graph that owns this stream.
*/
MediaStreamGraphImpl* GraphImpl();
MediaStreamGraph* Graph();
/**
* Sets the graph that owns this stream. Should only be called once.
*/
void SetGraphImpl(MediaStreamGraphImpl* aGraph);
void SetGraphImpl(MediaStreamGraph* aGraph);
/**
* Returns sample rate of the graph.
*/
TrackRate GraphRate() { return mTracks.GraphRate(); }
// Control API.
// Since a stream can be played multiple ways, we need to combine independent
// volume settings. The aKey parameter is used to keep volume settings
// separate. Since the stream is always playing the same contents, only
// a single audio output stream is used; the volumes are combined.
// Currently only the first enabled audio track is played.
// XXX change this so all enabled audio tracks are mixed and played.
virtual void AddAudioOutput(void* aKey);
virtual void SetAudioOutputVolume(void* aKey, float aVolume);
virtual void RemoveAudioOutput(void* aKey);
// Since a stream can be played multiple ways, we need to be able to
// play to multiple MediaStreamVideoSinks.
// Only the first enabled video track is played.
virtual void AddVideoOutput(MediaStreamVideoSink* aSink,
TrackID aID = TRACK_ANY);
virtual void RemoveVideoOutput(MediaStreamVideoSink* aSink,
TrackID aID = TRACK_ANY);
// Explicitly suspend. Useful for example if a media element is pausing
// and we need to stop its stream emitting its buffered data. As soon as the
// Suspend message reaches the graph, the stream stops processing. It
// ignores its inputs and produces silence/no video until Resumed. Its
// current time does not advance.
virtual void Suspend();
virtual void Resume();
// Events will be dispatched by calling methods of aListener.
virtual void AddListener(MediaStreamListener* aListener);
virtual void RemoveListener(MediaStreamListener* aListener);
virtual void AddTrackListener(MediaStreamTrackListener* aListener,
TrackID aTrackID);
virtual void RemoveTrackListener(MediaStreamTrackListener* aListener,
TrackID aTrackID);
/**
* Adds aListener to the source stream of track aTrackID in this stream.
* When the MediaStreamGraph processes the added listener, it will traverse
* the graph and add it to the track's source stream (remapping the TrackID
* along the way).
* Note that the listener will be notified on the MediaStreamGraph thread
* with whether the installation of it at the source was successful or not.
*/
virtual void AddDirectTrackListener(DirectMediaStreamTrackListener* aListener,
TrackID aTrackID);
/**
* Removes aListener from the source stream of track aTrackID in this stream.
* Note that the listener has already been removed if the link between the
* source of track aTrackID and this stream has been broken (and made track
* aTrackID end). The caller doesn't have to care about this, removing when
* the source cannot be found, or when the listener had already been removed
* does nothing.
*/
virtual void RemoveDirectTrackListener(DirectMediaStreamTrackListener* aListener,
TrackID aTrackID);
// A disabled track has video replaced by black, and audio replaced by
// silence.
void SetTrackEnabled(TrackID aTrackID, DisabledTrackMode aMode);
// Finish event will be notified by calling methods of aListener. It is the
// responsibility of the caller to remove aListener before it is destroyed.
void AddMainThreadListener(MainThreadMediaStreamListener* aListener);
// It's safe to call this even if aListener is not currently a listener;
// the call will be ignored.
void RemoveMainThreadListener(MainThreadMediaStreamListener* aListener)
{
MOZ_ASSERT(NS_IsMainThread());
MOZ_ASSERT(aListener);
mMainThreadListeners.RemoveElement(aListener);
}
/**
* Ensure a runnable will run on the main thread after running all pending
* updates that were sent from the graph thread or will be sent before the
* graph thread receives the next graph update.
*
* If the graph has been shut down or destroyed, then the runnable will be
* dispatched to the event queue immediately. If the graph is non-realtime
* and has not started, then the runnable will be run
* synchronously/immediately. (There are no pending updates in these
* situations.)
*
* Main thread only.
*/
void RunAfterPendingUpdates(already_AddRefed<nsIRunnable> aRunnable);
// Signal that the client is done with this MediaStream. It will be deleted
// later. Do not mix usage of Destroy() with RegisterUser()/UnregisterUser().
// That will cause the MediaStream to be destroyed twice, which will cause
// some assertions to fail.
virtual void Destroy();
// Signal that a client is using this MediaStream. Useful to not have to
// explicitly manage ownership (responsibility to Destroy()) when there are
// multiple clients using a MediaStream.
void RegisterUser();
// Signal that a client no longer needs this MediaStream. When the number of
// clients using this MediaStream reaches 0, it will be destroyed.
void UnregisterUser();
// Returns the main-thread's view of how much data has been processed by
// this stream.
StreamTime GetCurrentTime()
{
NS_ASSERTION(NS_IsMainThread(), "Call only on main thread");
return mMainThreadCurrentTime;
}
// Return the main thread's view of whether this stream has finished.
bool IsFinished()
{
NS_ASSERTION(NS_IsMainThread(), "Call only on main thread");
return mMainThreadFinished;
}
bool IsDestroyed()
{
NS_ASSERTION(NS_IsMainThread(), "Call only on main thread");
return mMainThreadDestroyed;
}
friend class MediaStreamGraphImpl;
friend class MediaInputPort;
friend class AudioNodeExternalInputStream;
virtual SourceMediaStream* AsSourceStream() { return nullptr; }
virtual ProcessedMediaStream* AsProcessedStream() { return nullptr; }
virtual AudioNodeStream* AsAudioNodeStream() { return nullptr; }
virtual TrackUnionStream* AsTrackUnionStream() { return nullptr; }
// These Impl methods perform the core functionality of the control methods
// above, on the media graph thread.
/**
* Stop all stream activity and disconnect it from all inputs and outputs.
* This must be idempotent.
*/
virtual void DestroyImpl();
StreamTime GetTracksEnd() { return mTracks.GetEnd(); }
#ifdef DEBUG
void DumpTrackInfo() { return mTracks.DumpTrackInfo(); }
#endif
void SetAudioOutputVolumeImpl(void* aKey, float aVolume);
void AddAudioOutputImpl(void* aKey);
// Returns true if this stream has an audio output.
bool HasAudioOutput()
{
return !mAudioOutputs.IsEmpty();
}
void RemoveAudioOutputImpl(void* aKey);
void AddVideoOutputImpl(already_AddRefed<MediaStreamVideoSink> aSink,
TrackID aID);
void RemoveVideoOutputImpl(MediaStreamVideoSink* aSink, TrackID aID);
void AddListenerImpl(already_AddRefed<MediaStreamListener> aListener);
void RemoveListenerImpl(MediaStreamListener* aListener);
void RemoveAllListenersImpl();
virtual void AddTrackListenerImpl(already_AddRefed<MediaStreamTrackListener> aListener,
TrackID aTrackID);
virtual void RemoveTrackListenerImpl(MediaStreamTrackListener* aListener,
TrackID aTrackID);
virtual void AddDirectTrackListenerImpl(already_AddRefed<DirectMediaStreamTrackListener> aListener,
TrackID aTrackID);
virtual void RemoveDirectTrackListenerImpl(DirectMediaStreamTrackListener* aListener,
TrackID aTrackID);
virtual void SetTrackEnabledImpl(TrackID aTrackID, DisabledTrackMode aMode);
DisabledTrackMode GetDisabledTrackMode(TrackID aTrackID);
void AddConsumer(MediaInputPort* aPort)
{
mConsumers.AppendElement(aPort);
}
void RemoveConsumer(MediaInputPort* aPort)
{
mConsumers.RemoveElement(aPort);
}
uint32_t ConsumerCount()
{
return mConsumers.Length();
}
StreamTracks& GetStreamTracks() { return mTracks; }
GraphTime GetStreamTracksStartTime() { return mTracksStartTime; }
double StreamTimeToSeconds(StreamTime aTime)
{
NS_ASSERTION(0 <= aTime && aTime <= STREAM_TIME_MAX, "Bad time");
return static_cast<double>(aTime)/mTracks.GraphRate();
}
int64_t StreamTimeToMicroseconds(StreamTime aTime)
{
NS_ASSERTION(0 <= aTime && aTime <= STREAM_TIME_MAX, "Bad time");
return (aTime*1000000)/mTracks.GraphRate();
}
StreamTime SecondsToNearestStreamTime(double aSeconds)
{
NS_ASSERTION(0 <= aSeconds && aSeconds <= TRACK_TICKS_MAX/TRACK_RATE_MAX,
"Bad seconds");
return mTracks.GraphRate() * aSeconds + 0.5;
}
StreamTime MicrosecondsToStreamTimeRoundDown(int64_t aMicroseconds) {
return (aMicroseconds*mTracks.GraphRate())/1000000;
}
TrackTicks TimeToTicksRoundUp(TrackRate aRate, StreamTime aTime)
{
return RateConvertTicksRoundUp(aRate, mTracks.GraphRate(), aTime);
}
StreamTime TicksToTimeRoundDown(TrackRate aRate, TrackTicks aTicks)
{
return RateConvertTicksRoundDown(mTracks.GraphRate(), aRate, aTicks);
}
/**
* Convert graph time to stream time. aTime must be <= mStateComputedTime
* to ensure we know exactly how much time this stream will be blocked during
* the interval.
*/
StreamTime GraphTimeToStreamTimeWithBlocking(GraphTime aTime);
/**
* Convert graph time to stream time. This assumes there is no blocking time
* to take account of, which is always true except between a stream
* having its blocking time calculated in UpdateGraph and its blocking time
* taken account of in UpdateCurrentTimeForStreams.
*/
StreamTime GraphTimeToStreamTime(GraphTime aTime);
/**
* Convert stream time to graph time. This assumes there is no blocking time
* to take account of, which is always true except between a stream
* having its blocking time calculated in UpdateGraph and its blocking time
* taken account of in UpdateCurrentTimeForStreams.
*/
GraphTime StreamTimeToGraphTime(StreamTime aTime);
bool IsFinishedOnGraphThread() { return mFinished; }
void FinishOnGraphThread();
bool HasCurrentData() { return mHasCurrentData; }
/**
* Find track by track id.
*/
StreamTracks::Track* FindTrack(TrackID aID);
StreamTracks::Track* EnsureTrack(TrackID aTrack);
2014-12-09 13:37:01 +03:00
virtual void ApplyTrackDisabling(TrackID aTrackID, MediaSegment* aSegment, MediaSegment* aRawSegment = nullptr);
// Return true if the main thread needs to observe updates from this stream.
virtual bool MainThreadNeedsUpdates() const
{
return true;
}
virtual size_t SizeOfExcludingThis(MallocSizeOf aMallocSizeOf) const;
virtual size_t SizeOfIncludingThis(MallocSizeOf aMallocSizeOf) const;
bool IsSuspended() { return mSuspendedCount > 0; }
void IncrementSuspendCount();
void DecrementSuspendCount();
protected:
// |AdvanceTimeVaryingValuesToCurrentTime| will be override in SourceMediaStream.
virtual void AdvanceTimeVaryingValuesToCurrentTime(GraphTime aCurrentTime,
GraphTime aBlockedTime)
{
mTracksStartTime += aBlockedTime;
mTracks.ForgetUpTo(aCurrentTime - mTracksStartTime);
}
void NotifyMainThreadListeners()
{
NS_ASSERTION(NS_IsMainThread(), "Call only on main thread");
for (int32_t i = mMainThreadListeners.Length() - 1; i >= 0; --i) {
mMainThreadListeners[i]->NotifyMainThreadStreamFinished();
}
mMainThreadListeners.Clear();
}
bool ShouldNotifyStreamFinished()
{
NS_ASSERTION(NS_IsMainThread(), "Call only on main thread");
if (!mMainThreadFinished || mFinishedNotificationSent) {
return false;
}
mFinishedNotificationSent = true;
return true;
}
// This state is all initialized on the main thread but
// otherwise modified only on the media graph thread.
// Buffered data. The start of the buffer corresponds to mTracksStartTime.
// Conceptually the buffer contains everything this stream has ever played,
// but we forget some prefix of the buffered data to bound the space usage.
StreamTracks mTracks;
// The time when the buffered data could be considered to have started playing.
// This increases over time to account for time the stream was blocked before
// mCurrentTime.
GraphTime mTracksStartTime;
// Client-set volume of this stream
struct AudioOutput {
explicit AudioOutput(void* aKey) : mKey(aKey), mVolume(1.0f) {}
void* mKey;
float mVolume;
};
nsTArray<AudioOutput> mAudioOutputs;
nsTArray<TrackBound<MediaStreamVideoSink>> mVideoOutputs;
// We record the last played video frame to avoid playing the frame again
// with a different frame id.
VideoFrame mLastPlayedVideoFrame;
Bug 1207245 - part 6 - rename nsRefPtr<T> to RefPtr<T>; r=ehsan; a=Tomcat The bulk of this commit was generated with a script, executed at the top level of a typical source code checkout. The only non-machine-generated part was modifying MFBT's moz.build to reflect the new naming. CLOSED TREE makes big refactorings like this a piece of cake. # The main substitution. find . -name '*.cpp' -o -name '*.cc' -o -name '*.h' -o -name '*.mm' -o -name '*.idl'| \ xargs perl -p -i -e ' s/nsRefPtr\.h/RefPtr\.h/g; # handle includes s/nsRefPtr ?</RefPtr</g; # handle declarations and variables ' # Handle a special friend declaration in gfx/layers/AtomicRefCountedWithFinalize.h. perl -p -i -e 's/::nsRefPtr;/::RefPtr;/' gfx/layers/AtomicRefCountedWithFinalize.h # Handle nsRefPtr.h itself, a couple places that define constructors # from nsRefPtr, and code generators specially. We do this here, rather # than indiscriminantly s/nsRefPtr/RefPtr/, because that would rename # things like nsRefPtrHashtable. perl -p -i -e 's/nsRefPtr/RefPtr/g' \ mfbt/nsRefPtr.h \ xpcom/glue/nsCOMPtr.h \ xpcom/base/OwningNonNull.h \ ipc/ipdl/ipdl/lower.py \ ipc/ipdl/ipdl/builtin.py \ dom/bindings/Codegen.py \ python/lldbutils/lldbutils/utils.py # In our indiscriminate substitution above, we renamed # nsRefPtrGetterAddRefs, the class behind getter_AddRefs. Fix that up. find . -name '*.cpp' -o -name '*.h' -o -name '*.idl' | \ xargs perl -p -i -e 's/nsRefPtrGetterAddRefs/RefPtrGetterAddRefs/g' if [ -d .git ]; then git mv mfbt/nsRefPtr.h mfbt/RefPtr.h else hg mv mfbt/nsRefPtr.h mfbt/RefPtr.h fi --HG-- rename : mfbt/nsRefPtr.h => mfbt/RefPtr.h
2015-10-18 08:24:48 +03:00
nsTArray<RefPtr<MediaStreamListener> > mListeners;
nsTArray<TrackBound<MediaStreamTrackListener>> mTrackListeners;
nsTArray<MainThreadMediaStreamListener*> mMainThreadListeners;
// List of disabled TrackIDs and their associated disabled mode.
// They can either by disabled by frames being replaced by black, or by
// retaining the previous frame.
nsTArray<DisabledTrack> mDisabledTracks;
// GraphTime at which this stream starts blocking.
// This is only valid up to mStateComputedTime. The stream is considered to
// have not been blocked before mCurrentTime (its mTracksStartTime is increased
// as necessary to account for that time instead).
GraphTime mStartBlocking;
// MediaInputPorts to which this is connected
nsTArray<MediaInputPort*> mConsumers;
// Where audio output is going. There is one AudioOutputStream per
// audio track.
struct AudioOutputStream
{
// When we started audio playback for this track.
// Add mStream->GetPosition() to find the current audio playback position.
GraphTime mAudioPlaybackStartTime;
// Amount of time that we've wanted to play silence because of the stream
// blocking.
MediaTime mBlockedAudioTime;
// Last tick written to the audio output.
StreamTime mLastTickWritten;
TrackID mTrackID;
};
nsTArray<AudioOutputStream> mAudioOutputStreams;
/**
* Number of outstanding suspend operations on this stream. Stream is
* suspended when this is > 0.
*/
int32_t mSuspendedCount;
/**
* When true, this means the stream will be finished once all
* buffered data has been consumed.
*/
bool mFinished;
/**
* When true, mFinished is true and we've played all the data in this stream
* and fired NotifyFinished notifications.
*/
bool mNotifiedFinished;
/**
* When true, the last NotifyBlockingChanged delivered to the listeners
* indicated that the stream is blocked.
*/
bool mNotifiedBlocked;
/**
* True if some data can be present by this stream if/when it's unblocked.
* Set by the stream itself on the MediaStreamGraph thread. Only changes
* from false to true once a stream has data, since we won't
* unblock it until there's more data.
*/
bool mHasCurrentData;
/**
* True if mHasCurrentData is true and we've notified listeners.
*/
bool mNotifiedHasCurrentData;
// Main-thread views of state
StreamTime mMainThreadCurrentTime;
bool mMainThreadFinished;
bool mFinishedNotificationSent;
bool mMainThreadDestroyed;
int mNrOfMainThreadUsers;
// Our media stream graph. null if destroyed on the graph thread.
MediaStreamGraphImpl* mGraph;
};
/**
* This is a stream into which a decoder can write audio and video.
*
* Audio and video can be written on any thread, but you probably want to
* always write from the same thread to avoid unexpected interleavings.
*/
class SourceMediaStream : public MediaStream
{
public:
explicit SourceMediaStream();
SourceMediaStream* AsSourceStream() override { return this; }
// Media graph thread only
// Users of audio inputs go through the stream so it can track when the
// last stream referencing an input goes away, so it can close the cubeb
// input. Also note: callable on any thread (though it bounces through
// MainThread to set the command if needed).
nsresult OpenAudioInput(int aID,
AudioDataListener *aListener);
// Note: also implied when Destroy() happens
void CloseAudioInput();
void DestroyImpl() override;
// Call these on any thread.
/**
* Enable or disable pulling. When pulling is enabled, NotifyPull
* gets called on MediaStreamListeners for this stream during the
* MediaStreamGraph control loop. Pulling is initially disabled.
* Due to unavoidable race conditions, after a call to SetPullEnabled(false)
* it is still possible for a NotifyPull to occur.
*/
void SetPullEnabled(bool aEnabled);
/**
* These add/remove DirectListeners, which allow bypassing the graph and any
* synchronization delays for e.g. PeerConnection, which wants the data ASAP
* and lets the far-end handle sync and playout timing.
*/
void NotifyListenersEventImpl(MediaStreamGraphEvent aEvent);
void NotifyListenersEvent(MediaStreamGraphEvent aEvent);
enum {
ADDTRACK_QUEUED = 0x01 // Queue track add until FinishAddTracks()
};
/**
* Add a new track to the stream starting at the given base time (which
* must be greater than or equal to the last time passed to
* AdvanceKnownTracksTime). Takes ownership of aSegment. aSegment should
* contain data starting after aStart.
*/
void AddTrack(TrackID aID, StreamTime aStart, MediaSegment* aSegment,
uint32_t aFlags = 0)
{
AddTrackInternal(aID, GraphRate(), aStart, aSegment, aFlags);
}
/**
* Like AddTrack, but resamples audio from aRate to the graph rate.
*/
void AddAudioTrack(TrackID aID, TrackRate aRate, StreamTime aStart,
AudioSegment* aSegment, uint32_t aFlags = 0);
/**
* Call after a series of AddTrack or AddAudioTrack calls to implement
* any pending track adds.
*/
void FinishAddTracks();
/**
* Append media data to a track. Ownership of aSegment remains with the caller,
* but aSegment is emptied.
* Returns false if the data was not appended because no such track exists
* or the stream was already finished.
*/
virtual bool AppendToTrack(TrackID aID, MediaSegment* aSegment, MediaSegment *aRawSegment = nullptr);
/**
* Get the stream time of the end of the data that has been appended so far.
* Can be called from any thread but won't be useful if it can race with
* an AppendToTrack call, so should probably just be called from the thread
* that also calls AppendToTrack.
*/
StreamTime GetEndOfAppendedData(TrackID aID);
/**
* Indicate that a track has ended. Do not do any more API calls
* affecting this track.
* Ignored if the track does not exist.
*/
void EndTrack(TrackID aID);
/**
* Indicate that no tracks will be added starting before time aKnownTime.
* aKnownTime must be >= its value at the last call to AdvanceKnownTracksTime.
*/
void AdvanceKnownTracksTime(StreamTime aKnownTime);
/**
* Indicate that this stream should enter the "finished" state. All tracks
* must have been ended via EndTrack. The finish time of the stream is
* when all tracks have ended.
*/
void FinishWithLockHeld();
void Finish()
2014-12-09 13:37:01 +03:00
{
MutexAutoLock lock(mMutex);
FinishWithLockHeld();
}
// Overriding allows us to hold the mMutex lock while changing the track enable status
void SetTrackEnabledImpl(TrackID aTrackID, DisabledTrackMode aMode) override;
2014-12-09 13:37:01 +03:00
// Overriding allows us to ensure mMutex is locked while changing the track enable status
void
2014-12-09 13:37:01 +03:00
ApplyTrackDisabling(TrackID aTrackID, MediaSegment* aSegment,
MediaSegment* aRawSegment = nullptr) override {
2014-12-09 13:37:01 +03:00
mMutex.AssertCurrentThreadOwns();
MediaStream::ApplyTrackDisabling(aTrackID, aSegment, aRawSegment);
}
/**
* End all tracks and Finish() this stream. Used to voluntarily revoke access
* to a LocalMediaStream.
*/
void EndAllTrackAndFinish();
/**
* Removes all direct listeners and signals to them that they have been
* uninstalled.
*/
void RemoveAllDirectListeners();
void RegisterForAudioMixing();
/**
* Returns true if this SourceMediaStream contains at least one audio track
* that is in pending state.
* This is thread safe, and takes the SourceMediaStream mutex.
*/
bool HasPendingAudioTrack();
TimeStamp GetStreamTracksStrartTimeStamp() {
MutexAutoLock lock(mMutex);
return mStreamTracksStartTimeStamp;
}
bool OpenNewAudioCallbackDriver(AudioDataListener *aListener);
// XXX need a Reset API
friend class MediaStreamGraphImpl;
protected:
enum TrackCommands : uint32_t;
virtual ~SourceMediaStream();
/**
* Data for each track that hasn't ended.
*/
struct TrackData {
TrackID mID;
// Sample rate of the input data.
TrackRate mInputRate;
// Resampler if the rate of the input track does not match the
// MediaStreamGraph's.
nsAutoRef<SpeexResamplerState> mResampler;
int mResamplerChannelCount;
StreamTime mStart;
// End-time of data already flushed to the track (excluding mData)
StreamTime mEndOfFlushedData;
// Each time the track updates are flushed to the media graph thread,
// the segment buffer is emptied.
nsAutoPtr<MediaSegment> mData;
// Each time the track updates are flushed to the media graph thread,
// this is cleared.
uint32_t mCommands;
};
bool NeedsMixing();
void ResampleAudioToGraphSampleRate(TrackData* aTrackData, MediaSegment* aSegment);
void AddDirectTrackListenerImpl(already_AddRefed<DirectMediaStreamTrackListener> aListener,
TrackID aTrackID) override;
void RemoveDirectTrackListenerImpl(DirectMediaStreamTrackListener* aListener,
TrackID aTrackID) override;
void AddTrackInternal(TrackID aID, TrackRate aRate,
StreamTime aStart, MediaSegment* aSegment,
uint32_t aFlags);
TrackData* FindDataForTrack(TrackID aID)
{
mMutex.AssertCurrentThreadOwns();
for (uint32_t i = 0; i < mUpdateTracks.Length(); ++i) {
if (mUpdateTracks[i].mID == aID) {
return &mUpdateTracks[i];
}
}
return nullptr;
}
/**
* Notify direct consumers of new data to one of the stream tracks.
* The data doesn't have to be resampled (though it may be). This is called
* from AppendToTrack on the thread providing the data, and will call
* the Listeners on this thread.
*/
void NotifyDirectConsumers(TrackData *aTrack,
MediaSegment *aSegment);
virtual void
AdvanceTimeVaryingValuesToCurrentTime(GraphTime aCurrentTime,
GraphTime aBlockedTime) override;
void SetStreamTracksStartTimeStamp(const TimeStamp& aTimeStamp)
{
MutexAutoLock lock(mMutex);
mStreamTracksStartTimeStamp = aTimeStamp;
}
// Only accessed on the MSG thread. Used so to ask the MSGImpl to usecount
// users of a specific input.
// XXX Should really be a CubebUtils::AudioDeviceID, but they aren't
// copyable (opaque pointers)
RefPtr<AudioDataListener> mInputListener;
// This must be acquired *before* MediaStreamGraphImpl's lock, if they are
// held together.
Mutex mMutex;
// protected by mMutex
StreamTime mUpdateKnownTracksTime;
// This time stamp will be updated in adding and blocked SourceMediaStream,
// |AddStreamGraphThread| and |AdvanceTimeVaryingValuesToCurrentTime| in
// particularly.
TimeStamp mStreamTracksStartTimeStamp;
nsTArray<TrackData> mUpdateTracks;
nsTArray<TrackData> mPendingTracks;
nsTArray<TrackBound<DirectMediaStreamTrackListener>> mDirectTrackListeners;
bool mPullEnabled;
bool mUpdateFinished;
bool mNeedsMixing;
};
/**
* The blocking mode decides how a track should be blocked in a MediaInputPort.
*/
enum class BlockingMode
{
/**
* BlockingMode CREATION blocks the source track from being created
* in the destination. It'll end if it already exists.
*/
CREATION,
/**
* BlockingMode END_EXISTING allows a track to be created in the destination
* but will end it before any data has been passed through.
*/
END_EXISTING,
};
/**
* Represents a connection between a ProcessedMediaStream and one of its
* input streams.
* We make these refcounted so that stream-related messages with MediaInputPort*
* pointers can be sent to the main thread safely.
*
* A port can be locked to a specific track in the source stream, in which case
* only this track will be forwarded to the destination stream. TRACK_ANY
* can used to signal that all tracks shall be forwarded.
*
* When a port is locked to a specific track in the source stream, it may also
* indicate a TrackID to map this source track to in the destination stream
* by setting aDestTrack to an explicit ID. When we do this, we must know
* that this TrackID in the destination stream is available. We assert during
* processing that the ID is available and that there are no generic input
* ports already attached to the destination stream.
* Note that this is currently only handled by TrackUnionStreams.
*
* When a port's source or destination stream dies, the stream's DestroyImpl
* calls MediaInputPort::Disconnect to disconnect the port from
* the source and destination streams.
*
* The lifetimes of MediaInputPort are controlled from the main thread.
* The media graph adds a reference to the port. When a MediaInputPort is no
* longer needed, main-thread code sends a Destroy message for the port and
* clears its reference (the last main-thread reference to the object). When
* the Destroy message is processed on the graph manager thread we disconnect
* the port and drop the graph's reference, destroying the object.
*/
class MediaInputPort final
{
private:
// Do not call this constructor directly. Instead call aDest->AllocateInputPort.
MediaInputPort(MediaStream* aSource,
TrackID& aSourceTrack,
ProcessedMediaStream* aDest,
TrackID& aDestTrack,
uint16_t aInputNumber,
uint16_t aOutputNumber)
: mSource(aSource)
, mSourceTrack(aSourceTrack)
, mDest(aDest)
, mDestTrack(aDestTrack)
, mInputNumber(aInputNumber)
, mOutputNumber(aOutputNumber)
, mGraph(nullptr)
{
MOZ_COUNT_CTOR(MediaInputPort);
}
// Private destructor, to discourage deletion outside of Release():
~MediaInputPort()
{
MOZ_COUNT_DTOR(MediaInputPort);
}
public:
NS_INLINE_DECL_THREADSAFE_REFCOUNTING(MediaInputPort)
// Called on graph manager thread
// Do not call these from outside MediaStreamGraph.cpp!
void Init();
// Called during message processing to trigger removal of this stream.
void Disconnect();
// Control API
/**
* Disconnects and destroys the port. The caller must not reference this
* object again.
*/
void Destroy();
// Any thread
MediaStream* GetSource() { return mSource; }
TrackID GetSourceTrackId() { return mSourceTrack; }
ProcessedMediaStream* GetDestination() { return mDest; }
TrackID GetDestinationTrackId() { return mDestTrack; }
/**
* Block aTrackId in the source stream from being passed through the port.
* Consumers will interpret this track as ended.
* Returns a pledge that resolves on the main thread after the track block has
* been applied by the MSG.
*/
already_AddRefed<media::Pledge<bool, nsresult>> BlockSourceTrackId(TrackID aTrackId,
BlockingMode aBlockingMode);
private:
void BlockSourceTrackIdImpl(TrackID aTrackId, BlockingMode aBlockingMode);
public:
// Returns true if aTrackId has not been blocked for any reason and this port
// has not been locked to another track.
bool PassTrackThrough(TrackID aTrackId) {
bool blocked = false;
for (auto pair : mBlockedTracks) {
if (pair.first() == aTrackId &&
(pair.second() == BlockingMode::CREATION ||
pair.second() == BlockingMode::END_EXISTING)) {
blocked = true;
break;
}
}
return !blocked && (mSourceTrack == TRACK_ANY || mSourceTrack == aTrackId);
}
// Returns true if aTrackId has not been blocked for track creation and this
// port has not been locked to another track.
bool AllowCreationOf(TrackID aTrackId) {
bool blocked = false;
for (auto pair : mBlockedTracks) {
if (pair.first() == aTrackId &&
pair.second() == BlockingMode::CREATION) {
blocked = true;
break;
}
}
return !blocked && (mSourceTrack == TRACK_ANY || mSourceTrack == aTrackId);
}
uint16_t InputNumber() const { return mInputNumber; }
uint16_t OutputNumber() const { return mOutputNumber; }
// Call on graph manager thread
struct InputInterval {
GraphTime mStart;
GraphTime mEnd;
bool mInputIsBlocked;
};
// Find the next time interval starting at or after aTime during which
// mDest is not blocked and mSource's blocking status does not change.
InputInterval GetNextInputInterval(GraphTime aTime);
/**
* Returns the graph that owns this port.
*/
MediaStreamGraphImpl* GraphImpl();
MediaStreamGraph* Graph();
/**
* Sets the graph that owns this stream. Should only be called once.
*/
void SetGraphImpl(MediaStreamGraphImpl* aGraph);
/**
* Notify the port that the source MediaStream has been suspended.
*/
void Suspended();
/**
* Notify the port that the source MediaStream has been resumed.
*/
void Resumed();
size_t SizeOfExcludingThis(MallocSizeOf aMallocSizeOf) const
{
size_t amount = 0;
// Not owned:
// - mSource
// - mDest
// - mGraph
return amount;
}
size_t SizeOfIncludingThis(MallocSizeOf aMallocSizeOf) const
{
return aMallocSizeOf(this) + SizeOfExcludingThis(aMallocSizeOf);
}
private:
friend class MediaStreamGraphImpl;
friend class MediaStream;
friend class ProcessedMediaStream;
// Never modified after Init()
MediaStream* mSource;
TrackID mSourceTrack;
ProcessedMediaStream* mDest;
TrackID mDestTrack;
// The input and output numbers are optional, and are currently only used by
// Web Audio.
const uint16_t mInputNumber;
const uint16_t mOutputNumber;
typedef Pair<TrackID, BlockingMode> BlockedTrack;
nsTArray<BlockedTrack> mBlockedTracks;
// Our media stream graph
MediaStreamGraphImpl* mGraph;
};
/**
* This stream processes zero or more input streams in parallel to produce
* its output. The details of how the output is produced are handled by
* subclasses overriding the ProcessInput method.
*/
class ProcessedMediaStream : public MediaStream
{
public:
explicit ProcessedMediaStream()
: MediaStream()
, mAutofinish(false)
, mCycleMarker(0)
{}
// Control API.
/**
* Allocates a new input port attached to source aStream.
* This stream can be removed by calling MediaInputPort::Remove().
*
* The input port is tied to aTrackID in the source stream.
* aTrackID can be set to TRACK_ANY to automatically forward all tracks from
* aStream.
*
* If aTrackID is an explicit ID, aDestTrackID can also be made explicit
* to ensure that the track is assigned this ID in the destination stream.
* To avoid intermittent TrackID collisions the destination stream may not
* have any existing generic input ports (with TRACK_ANY source track) when
* you allocate an input port with a destination TrackID.
*
* To end a track in the destination stream forwarded with TRACK_ANY,
* it can be blocked in the input port through MediaInputPort::BlockTrackId().
*
* Tracks in aBlockedTracks will be blocked in the input port initially. This
* ensures that they don't get created by the MSG-thread before we can
* BlockTrackId() on the main thread.
*/
already_AddRefed<MediaInputPort>
AllocateInputPort(MediaStream* aStream,
TrackID aTrackID = TRACK_ANY,
TrackID aDestTrackID = TRACK_ANY,
uint16_t aInputNumber = 0,
uint16_t aOutputNumber = 0,
nsTArray<TrackID>* aBlockedTracks = nullptr);
/**
* Force this stream into the finished state.
*/
void Finish();
/**
* Set the autofinish flag on this stream (defaults to false). When this flag
* is set, and all input streams are in the finished state (including if there
* are no input streams), this stream automatically enters the finished state.
*/
void SetAutofinish(bool aAutofinish);
ProcessedMediaStream* AsProcessedStream() override { return this; }
friend class MediaStreamGraphImpl;
// Do not call these from outside MediaStreamGraph.cpp!
virtual void AddInput(MediaInputPort* aPort);
virtual void RemoveInput(MediaInputPort* aPort)
{
mInputs.RemoveElement(aPort) || mSuspendedInputs.RemoveElement(aPort);
}
bool HasInputPort(MediaInputPort* aPort)
{
return mInputs.Contains(aPort) || mSuspendedInputs.Contains(aPort);
}
uint32_t InputPortCount()
{
return mInputs.Length() + mSuspendedInputs.Length();
}
void InputSuspended(MediaInputPort* aPort);
void InputResumed(MediaInputPort* aPort);
virtual MediaStream* GetInputStreamFor(TrackID aTrackID) { return nullptr; }
virtual TrackID GetInputTrackIDFor(TrackID aTrackID) { return TRACK_NONE; }
void DestroyImpl() override;
/**
* This gets called after we've computed the blocking states for all
* streams (mBlocked is up to date up to mStateComputedTime).
* Also, we've produced output for all streams up to this one. If this stream
* is not in a cycle, then all its source streams have produced data.
* Generate output from aFrom to aTo.
* This will be called on streams that have finished. Most stream types should
* just return immediately if IsFinishedOnGraphThread(), but some may wish to
* update internal state (see AudioNodeStream).
* ProcessInput is allowed to call FinishOnGraphThread only if ALLOW_FINISH
* is in aFlags. (This flag will be set when aTo >= mStateComputedTime, i.e.
* when we've producing the last block of data we need to produce.) Otherwise
* we can get into a situation where we've determined the stream should not
* block before mStateComputedTime, but the stream finishes before
* mStateComputedTime, violating the invariant that finished streams are blocked.
*/
enum {
ALLOW_FINISH = 0x01
};
virtual void ProcessInput(GraphTime aFrom, GraphTime aTo, uint32_t aFlags) = 0;
void SetAutofinishImpl(bool aAutofinish) { mAutofinish = aAutofinish; }
// Only valid after MediaStreamGraphImpl::UpdateStreamOrder() has run.
// A DelayNode is considered to break a cycle and so this will not return
// true for echo loops, only for muted cycles.
bool InMutedCycle() const { return mCycleMarker; }
size_t SizeOfExcludingThis(MallocSizeOf aMallocSizeOf) const override
{
size_t amount = MediaStream::SizeOfExcludingThis(aMallocSizeOf);
// Not owned:
// - mInputs elements
// - mSuspendedInputs elements
amount += mInputs.ShallowSizeOfExcludingThis(aMallocSizeOf);
amount += mSuspendedInputs.ShallowSizeOfExcludingThis(aMallocSizeOf);
return amount;
}
size_t SizeOfIncludingThis(MallocSizeOf aMallocSizeOf) const override
{
return aMallocSizeOf(this) + SizeOfExcludingThis(aMallocSizeOf);
}
protected:
// This state is all accessed only on the media graph thread.
// The list of all inputs that are not currently suspended.
nsTArray<MediaInputPort*> mInputs;
// The list of all inputs that are currently suspended.
nsTArray<MediaInputPort*> mSuspendedInputs;
bool mAutofinish;
// After UpdateStreamOrder(), mCycleMarker is either 0 or 1 to indicate
// whether this stream is in a muted cycle. During ordering it can contain
// other marker values - see MediaStreamGraphImpl::UpdateStreamOrder().
uint32_t mCycleMarker;
};
/**
* There is a single MediaStreamGraph per window.
* Additionaly, each OfflineAudioContext object creates its own MediaStreamGraph
* object too..
*/
class MediaStreamGraph
{
public:
// We ensure that the graph current time advances in multiples of
// IdealAudioBlockSize()/AudioStream::PreferredSampleRate(). A stream that
// never blocks and has a track with the ideal audio rate will produce audio
// in multiples of the block size.
// Initializing an graph that outputs audio can be quite long on some
// platforms. Code that want to output audio at some point can express the
// fact that they will need an audio stream at some point by passing
// AUDIO_THREAD_DRIVER when getting an instance of MediaStreamGraph, so that
// the graph starts with the right driver.
enum GraphDriverType {
AUDIO_THREAD_DRIVER,
SYSTEM_THREAD_DRIVER,
OFFLINE_THREAD_DRIVER
};
static const uint32_t AUDIO_CALLBACK_DRIVER_SHUTDOWN_TIMEOUT = 20*1000;
// Main thread only
static MediaStreamGraph* GetInstanceIfExists(nsPIDOMWindowInner* aWindow);
static MediaStreamGraph* GetInstance(GraphDriverType aGraphDriverRequested,
nsPIDOMWindowInner* aWindow);
static MediaStreamGraph* CreateNonRealtimeInstance(
TrackRate aSampleRate,
nsPIDOMWindowInner* aWindowId);
// Return the correct main thread for this graph. This always returns
// something that is valid. Thread safe.
AbstractThread* AbstractMainThread();
// Idempotent
static void DestroyNonRealtimeInstance(MediaStreamGraph* aGraph);
virtual nsresult OpenAudioInput(int aID,
AudioDataListener *aListener) {
return NS_ERROR_FAILURE;
}
virtual void CloseAudioInput(AudioDataListener *aListener) {}
// Control API.
/**
* Create a stream that a media decoder (or some other source of
* media data, such as a camera) can write to.
*/
SourceMediaStream* CreateSourceStream();
/**
* Create a stream that will form the union of the tracks of its input
* streams.
* A TrackUnionStream contains all the tracks of all its input streams.
* Adding a new input stream makes that stream's tracks immediately appear as new
* tracks starting at the time the input stream was added.
* Removing an input stream makes the output tracks corresponding to the
* removed tracks immediately end.
* For each added track, the track ID of the output track is the track ID
* of the input track or one plus the maximum ID of all previously added
* tracks, whichever is greater.
* TODO at some point we will probably need to add API to select
* particular tracks of each input stream.
*/
ProcessedMediaStream* CreateTrackUnionStream();
/**
* Create a stream that will mix all its audio input.
*/
ProcessedMediaStream* CreateAudioCaptureStream(TrackID aTrackId);
/**
* Add a new stream to the graph. Main thread.
*/
void AddStream(MediaStream* aStream);
Bug 1094764 - Implement AudioContext.suspend and friends. r=roc,ehsan - Relevant spec text: - http://webaudio.github.io/web-audio-api/#widl-AudioContext-suspend-Promise - http://webaudio.github.io/web-audio-api/#widl-AudioContext-resume-Promise - http://webaudio.github.io/web-audio-api/#widl-AudioContext-close-Promise - http://webaudio.github.io/web-audio-api/#widl-AudioContext-state - http://webaudio.github.io/web-audio-api/#widl-AudioContext-onstatechange - In a couple words, the behavior we want: - Closed context cannot have new nodes created, but can do decodeAudioData, and create buffers, and such. - OfflineAudioContexts don't support those methods, transitions happen at startRendering and at the end of processing. onstatechange is used to make this observable. - (regular) AudioContexts support those methods. The promises and onstatechange should be resolved/called when the operation has actually completed on the rendering thread. Once a context has been closed, it cannot transition back to "running". An AudioContext switches to "running" when the audio callback start running, this allow authors to know how long the audio stack takes to start running. - MediaStreams that feed in/go out of a suspended graph should respectively not buffer at the graph input, and output silence - suspended context should not be doing much on the CPU, and we should try to pause audio streams if we can (this behaviour is the main reason we need this in the first place, for saving battery on mobile, and CPU on all platforms) - Now, the implementation: - AudioNodeStreams are now tagged with a context id, to be able to operate on all the streams of a given AudioContext on the Graph thread without having to go and lock everytime to touch the AudioContext. This happens in the AudioNodeStream ctor. IDs are of course constant for the lifetime of the node. - When an AudioContext goes into suspended mode, streams for this AudioContext are moved out of the mStreams array to a second array, mSuspendedStreams. Streams in mSuspendedStream are not ordered, and are not processed. - The MSG will automatically switch to a SystemClockDriver when it finds that there are no more AudioNodeStream/Stream with an audio track. This is how pausing the audio subsystem and saving battery works. Subsequently, when the MSG finds that there are only streams in mSuspendedStreams, it will go to sleep (block on a monitor), so we save CPU, but it does not shut itself down. This is mostly not a new behaviour (this is what the MSG does since the refactoring), but is important to note. - Promises are gripped (addref-ed) on the main thread, and then shepherd down other threads and to the GraphDriver, if needed (sometimes we can resolve them right away). They move between threads as void* to prevent calling methods on them, as they are not thread safe. Then, the driver executes the operation, and when it's done (initializing and closing audio streams can take some time), we send the promise back to the main thread, and resolve it, casting back to Promise* after asserting we're back on the main thread. This way, we can send them back on the main thread once an operation has complete (suspending an audio stream, starting it again on resume(), etc.), without having to do bookkeeping between suspend calls and their result. Promises are not thread safe, so we can't move them around AddRef-ed. - The stream destruction logic now takes into account that a stream can be destroyed while not being in mStreams. - A graph can now switch GraphDriver twice or more per iteration, for example if an author goes suspend()/resume()/suspend() in the same script. - Some operation have to be done on suspended stream, so we now use double for-loop around mSuspendedStreams and mStreams in some places in MediaStreamGraph.cpp. - A tricky part was making sure everything worked at AudioContext boundaries. TrackUnionStream that have one of their input stream suspended append null ticks instead. - The graph ordering algorithm had to be altered to not include suspended streams. - There are some edge cases (adding a stream on a suspended graph, calling suspend/resume when a graph has just been close()d).
2015-02-27 20:22:05 +03:00
/* From the main thread, ask the MSG to send back an event when the graph
* thread is running, and audio is being processed. */
void NotifyWhenGraphStarted(AudioNodeStream* aNodeStream);
/* From the main thread, suspend, resume or close an AudioContext.
* aStreams are the streams of all the AudioNodes of the AudioContext that
* need to be suspended or resumed. This can be empty if this is a second
* consecutive suspend call and all the nodes are already suspended.
Bug 1094764 - Implement AudioContext.suspend and friends. r=roc,ehsan - Relevant spec text: - http://webaudio.github.io/web-audio-api/#widl-AudioContext-suspend-Promise - http://webaudio.github.io/web-audio-api/#widl-AudioContext-resume-Promise - http://webaudio.github.io/web-audio-api/#widl-AudioContext-close-Promise - http://webaudio.github.io/web-audio-api/#widl-AudioContext-state - http://webaudio.github.io/web-audio-api/#widl-AudioContext-onstatechange - In a couple words, the behavior we want: - Closed context cannot have new nodes created, but can do decodeAudioData, and create buffers, and such. - OfflineAudioContexts don't support those methods, transitions happen at startRendering and at the end of processing. onstatechange is used to make this observable. - (regular) AudioContexts support those methods. The promises and onstatechange should be resolved/called when the operation has actually completed on the rendering thread. Once a context has been closed, it cannot transition back to "running". An AudioContext switches to "running" when the audio callback start running, this allow authors to know how long the audio stack takes to start running. - MediaStreams that feed in/go out of a suspended graph should respectively not buffer at the graph input, and output silence - suspended context should not be doing much on the CPU, and we should try to pause audio streams if we can (this behaviour is the main reason we need this in the first place, for saving battery on mobile, and CPU on all platforms) - Now, the implementation: - AudioNodeStreams are now tagged with a context id, to be able to operate on all the streams of a given AudioContext on the Graph thread without having to go and lock everytime to touch the AudioContext. This happens in the AudioNodeStream ctor. IDs are of course constant for the lifetime of the node. - When an AudioContext goes into suspended mode, streams for this AudioContext are moved out of the mStreams array to a second array, mSuspendedStreams. Streams in mSuspendedStream are not ordered, and are not processed. - The MSG will automatically switch to a SystemClockDriver when it finds that there are no more AudioNodeStream/Stream with an audio track. This is how pausing the audio subsystem and saving battery works. Subsequently, when the MSG finds that there are only streams in mSuspendedStreams, it will go to sleep (block on a monitor), so we save CPU, but it does not shut itself down. This is mostly not a new behaviour (this is what the MSG does since the refactoring), but is important to note. - Promises are gripped (addref-ed) on the main thread, and then shepherd down other threads and to the GraphDriver, if needed (sometimes we can resolve them right away). They move between threads as void* to prevent calling methods on them, as they are not thread safe. Then, the driver executes the operation, and when it's done (initializing and closing audio streams can take some time), we send the promise back to the main thread, and resolve it, casting back to Promise* after asserting we're back on the main thread. This way, we can send them back on the main thread once an operation has complete (suspending an audio stream, starting it again on resume(), etc.), without having to do bookkeeping between suspend calls and their result. Promises are not thread safe, so we can't move them around AddRef-ed. - The stream destruction logic now takes into account that a stream can be destroyed while not being in mStreams. - A graph can now switch GraphDriver twice or more per iteration, for example if an author goes suspend()/resume()/suspend() in the same script. - Some operation have to be done on suspended stream, so we now use double for-loop around mSuspendedStreams and mStreams in some places in MediaStreamGraph.cpp. - A tricky part was making sure everything worked at AudioContext boundaries. TrackUnionStream that have one of their input stream suspended append null ticks instead. - The graph ordering algorithm had to be altered to not include suspended streams. - There are some edge cases (adding a stream on a suspended graph, calling suspend/resume when a graph has just been close()d).
2015-02-27 20:22:05 +03:00
*
* This can possibly pause the graph thread, releasing system resources, if
* all streams have been suspended/closed.
*
* When the operation is complete, aPromise is resolved.
*/
void ApplyAudioContextOperation(MediaStream* aDestinationStream,
const nsTArray<MediaStream*>& aStreams,
Bug 1094764 - Implement AudioContext.suspend and friends. r=roc,ehsan - Relevant spec text: - http://webaudio.github.io/web-audio-api/#widl-AudioContext-suspend-Promise - http://webaudio.github.io/web-audio-api/#widl-AudioContext-resume-Promise - http://webaudio.github.io/web-audio-api/#widl-AudioContext-close-Promise - http://webaudio.github.io/web-audio-api/#widl-AudioContext-state - http://webaudio.github.io/web-audio-api/#widl-AudioContext-onstatechange - In a couple words, the behavior we want: - Closed context cannot have new nodes created, but can do decodeAudioData, and create buffers, and such. - OfflineAudioContexts don't support those methods, transitions happen at startRendering and at the end of processing. onstatechange is used to make this observable. - (regular) AudioContexts support those methods. The promises and onstatechange should be resolved/called when the operation has actually completed on the rendering thread. Once a context has been closed, it cannot transition back to "running". An AudioContext switches to "running" when the audio callback start running, this allow authors to know how long the audio stack takes to start running. - MediaStreams that feed in/go out of a suspended graph should respectively not buffer at the graph input, and output silence - suspended context should not be doing much on the CPU, and we should try to pause audio streams if we can (this behaviour is the main reason we need this in the first place, for saving battery on mobile, and CPU on all platforms) - Now, the implementation: - AudioNodeStreams are now tagged with a context id, to be able to operate on all the streams of a given AudioContext on the Graph thread without having to go and lock everytime to touch the AudioContext. This happens in the AudioNodeStream ctor. IDs are of course constant for the lifetime of the node. - When an AudioContext goes into suspended mode, streams for this AudioContext are moved out of the mStreams array to a second array, mSuspendedStreams. Streams in mSuspendedStream are not ordered, and are not processed. - The MSG will automatically switch to a SystemClockDriver when it finds that there are no more AudioNodeStream/Stream with an audio track. This is how pausing the audio subsystem and saving battery works. Subsequently, when the MSG finds that there are only streams in mSuspendedStreams, it will go to sleep (block on a monitor), so we save CPU, but it does not shut itself down. This is mostly not a new behaviour (this is what the MSG does since the refactoring), but is important to note. - Promises are gripped (addref-ed) on the main thread, and then shepherd down other threads and to the GraphDriver, if needed (sometimes we can resolve them right away). They move between threads as void* to prevent calling methods on them, as they are not thread safe. Then, the driver executes the operation, and when it's done (initializing and closing audio streams can take some time), we send the promise back to the main thread, and resolve it, casting back to Promise* after asserting we're back on the main thread. This way, we can send them back on the main thread once an operation has complete (suspending an audio stream, starting it again on resume(), etc.), without having to do bookkeeping between suspend calls and their result. Promises are not thread safe, so we can't move them around AddRef-ed. - The stream destruction logic now takes into account that a stream can be destroyed while not being in mStreams. - A graph can now switch GraphDriver twice or more per iteration, for example if an author goes suspend()/resume()/suspend() in the same script. - Some operation have to be done on suspended stream, so we now use double for-loop around mSuspendedStreams and mStreams in some places in MediaStreamGraph.cpp. - A tricky part was making sure everything worked at AudioContext boundaries. TrackUnionStream that have one of their input stream suspended append null ticks instead. - The graph ordering algorithm had to be altered to not include suspended streams. - There are some edge cases (adding a stream on a suspended graph, calling suspend/resume when a graph has just been close()d).
2015-02-27 20:22:05 +03:00
dom::AudioContextOperation aState,
void* aPromise);
Bug 1094764 - Implement AudioContext.suspend and friends. r=roc,ehsan - Relevant spec text: - http://webaudio.github.io/web-audio-api/#widl-AudioContext-suspend-Promise - http://webaudio.github.io/web-audio-api/#widl-AudioContext-resume-Promise - http://webaudio.github.io/web-audio-api/#widl-AudioContext-close-Promise - http://webaudio.github.io/web-audio-api/#widl-AudioContext-state - http://webaudio.github.io/web-audio-api/#widl-AudioContext-onstatechange - In a couple words, the behavior we want: - Closed context cannot have new nodes created, but can do decodeAudioData, and create buffers, and such. - OfflineAudioContexts don't support those methods, transitions happen at startRendering and at the end of processing. onstatechange is used to make this observable. - (regular) AudioContexts support those methods. The promises and onstatechange should be resolved/called when the operation has actually completed on the rendering thread. Once a context has been closed, it cannot transition back to "running". An AudioContext switches to "running" when the audio callback start running, this allow authors to know how long the audio stack takes to start running. - MediaStreams that feed in/go out of a suspended graph should respectively not buffer at the graph input, and output silence - suspended context should not be doing much on the CPU, and we should try to pause audio streams if we can (this behaviour is the main reason we need this in the first place, for saving battery on mobile, and CPU on all platforms) - Now, the implementation: - AudioNodeStreams are now tagged with a context id, to be able to operate on all the streams of a given AudioContext on the Graph thread without having to go and lock everytime to touch the AudioContext. This happens in the AudioNodeStream ctor. IDs are of course constant for the lifetime of the node. - When an AudioContext goes into suspended mode, streams for this AudioContext are moved out of the mStreams array to a second array, mSuspendedStreams. Streams in mSuspendedStream are not ordered, and are not processed. - The MSG will automatically switch to a SystemClockDriver when it finds that there are no more AudioNodeStream/Stream with an audio track. This is how pausing the audio subsystem and saving battery works. Subsequently, when the MSG finds that there are only streams in mSuspendedStreams, it will go to sleep (block on a monitor), so we save CPU, but it does not shut itself down. This is mostly not a new behaviour (this is what the MSG does since the refactoring), but is important to note. - Promises are gripped (addref-ed) on the main thread, and then shepherd down other threads and to the GraphDriver, if needed (sometimes we can resolve them right away). They move between threads as void* to prevent calling methods on them, as they are not thread safe. Then, the driver executes the operation, and when it's done (initializing and closing audio streams can take some time), we send the promise back to the main thread, and resolve it, casting back to Promise* after asserting we're back on the main thread. This way, we can send them back on the main thread once an operation has complete (suspending an audio stream, starting it again on resume(), etc.), without having to do bookkeeping between suspend calls and their result. Promises are not thread safe, so we can't move them around AddRef-ed. - The stream destruction logic now takes into account that a stream can be destroyed while not being in mStreams. - A graph can now switch GraphDriver twice or more per iteration, for example if an author goes suspend()/resume()/suspend() in the same script. - Some operation have to be done on suspended stream, so we now use double for-loop around mSuspendedStreams and mStreams in some places in MediaStreamGraph.cpp. - A tricky part was making sure everything worked at AudioContext boundaries. TrackUnionStream that have one of their input stream suspended append null ticks instead. - The graph ordering algorithm had to be altered to not include suspended streams. - There are some edge cases (adding a stream on a suspended graph, calling suspend/resume when a graph has just been close()d).
2015-02-27 20:22:05 +03:00
bool IsNonRealtime() const;
/**
* Start processing non-realtime for a specific number of ticks.
*/
void StartNonRealtimeProcessing(uint32_t aTicksToProcess);
/**
* Media graph thread only.
* Dispatches a runnable that will run on the main thread after all
* main-thread stream state has been next updated.
*
* Should only be called during MediaStreamListener callbacks or during
* ProcessedMediaStream::ProcessInput().
*/
virtual void DispatchToMainThreadAfterStreamStateUpdate(
already_AddRefed<nsIRunnable> aRunnable);
/**
* Returns graph sample rate in Hz.
*/
TrackRate GraphRate() const { return mSampleRate; }
void RegisterCaptureStreamForWindow(uint64_t aWindowId,
ProcessedMediaStream* aCaptureStream);
void UnregisterCaptureStreamForWindow(uint64_t aWindowId);
already_AddRefed<MediaInputPort> ConnectToCaptureStream(
uint64_t aWindowId, MediaStream* aMediaStream);
/**
* Data going to the speakers from the GraphDriver's DataCallback
* to notify any listeners (for echo cancellation).
*/
void NotifyOutputData(AudioDataValue* aBuffer, size_t aFrames,
TrackRate aRate, uint32_t aChannels);
void AssertOnGraphThreadOrNotRunning() const
{
MOZ_ASSERT(OnGraphThreadOrNotRunning());
}
protected:
explicit MediaStreamGraph(TrackRate aSampleRate)
: mSampleRate(aSampleRate)
{
MOZ_COUNT_CTOR(MediaStreamGraph);
}
virtual ~MediaStreamGraph()
{
MOZ_COUNT_DTOR(MediaStreamGraph);
}
// Intended only for assertions, either on graph thread, not running, or
// with monitor held.
bool OnGraphThreadOrNotRunning() const;
// Media graph thread only
nsTArray<nsCOMPtr<nsIRunnable> > mPendingUpdateRunnables;
/**
* Sample rate at which this graph runs. For real time graphs, this is
* the rate of the audio mixer. For offline graphs, this is the rate specified
* at construction.
*/
TrackRate mSampleRate;
/**
* CloseAudioInput is async, so hold a reference here.
*/
nsTArray<RefPtr<AudioDataListener>> mAudioInputs;
};
} // namespace mozilla
#endif /* MOZILLA_MEDIASTREAMGRAPH_H_ */