From ac369ba83b2b1d165b0d8d2c1be51429429416de Mon Sep 17 00:00:00 2001 From: Andreas Pehrson Date: Wed, 19 Sep 2018 15:51:46 +0000 Subject: [PATCH] Bug 1253499 - Break out VideoStreamFactory to separate file. r=dminor Differential Revision: https://phabricator.services.mozilla.com/D6263 --HG-- extra : moz-landing-system : lando --- .../src/media-conduit/VideoConduit.cpp | 230 +--------------- .../src/media-conduit/VideoConduit.h | 79 +----- .../src/media-conduit/VideoStreamFactory.cpp | 258 ++++++++++++++++++ .../src/media-conduit/VideoStreamFactory.h | 79 ++++++ .../signaling/src/media-conduit/moz.build | 1 + 5 files changed, 349 insertions(+), 298 deletions(-) create mode 100644 media/webrtc/signaling/src/media-conduit/VideoStreamFactory.cpp create mode 100644 media/webrtc/signaling/src/media-conduit/VideoStreamFactory.h diff --git a/media/webrtc/signaling/src/media-conduit/VideoConduit.cpp b/media/webrtc/signaling/src/media-conduit/VideoConduit.cpp index f066f7090f31..fc94218ecdca 100644 --- a/media/webrtc/signaling/src/media-conduit/VideoConduit.cpp +++ b/media/webrtc/signaling/src/media-conduit/VideoConduit.cpp @@ -8,6 +8,7 @@ #include "AudioConduit.h" #include "VideoConduit.h" +#include "VideoStreamFactory.h" #include "YuvStamper.h" #include "mozilla/TemplateLib.h" #include "mozilla/media/MediaUtils.h" @@ -88,12 +89,8 @@ static const char* kRedPayloadName = "red"; // is active and one or more layers are being scaled. #define SIMULCAST_RESOLUTION_ALIGNMENT 16 -// Convert (SI) kilobits/sec to (SI) bits/sec -#define KBPS(kbps) kbps * 1000 - // 32 bytes is what WebRTC CodecInst expects const unsigned int WebrtcVideoConduit::CODEC_PLNAME_SIZE = 32; -static const int kViEMinCodecBitrate_bps = KBPS(30); template T MinIgnoreZero(const T& a, const T& b) @@ -173,100 +170,6 @@ SelectSendFrameRate(const VideoCodecConfig* codecConfig, return new_framerate; } -#define MB_OF(w,h) ((unsigned int)((((w+15)>>4))*((unsigned int)((h+15)>>4)))) -// For now, try to set the max rates well above the knee in the curve. -// Chosen somewhat arbitrarily; it's hard to find good data oriented for -// realtime interactive/talking-head recording. These rates assume -// 30fps. - -// XXX Populate this based on a pref (which we should consider sorting because -// people won't assume they need to). -static WebrtcVideoConduit::ResolutionAndBitrateLimits kResolutionAndBitrateLimits[] = { - {MB_OF(1920, 1200), KBPS(1500), KBPS(2000), KBPS(10000)}, // >HD (3K, 4K, etc) - {MB_OF(1280, 720), KBPS(1200), KBPS(1500), KBPS(5000)}, // HD ~1080-1200 - {MB_OF(800, 480), KBPS(600), KBPS(800), KBPS(2500)}, // HD ~720 - {MB_OF(480, 270), KBPS(150), KBPS(500), KBPS(2000)}, // WVGA - {tl::Max::value, KBPS(125), KBPS(300), KBPS(1300)}, // VGA - {MB_OF(176, 144), KBPS(100), KBPS(150), KBPS(500)}, // WQVGA, CIF - {0 , KBPS(40), KBPS(80), KBPS(250)} // QCIF and below -}; - -static WebrtcVideoConduit::ResolutionAndBitrateLimits -GetLimitsFor(unsigned int aWidth, unsigned int aHeight, int aCapBps = 0) -{ - // max bandwidth should be proportional (not linearly!) to resolution, and - // proportional (perhaps linearly, or close) to current frame rate. - int fs = MB_OF(aWidth, aHeight); - - for (const auto& resAndLimits : kResolutionAndBitrateLimits) { - if (fs > resAndLimits.resolution_in_mb && - // pick the highest range where at least start rate is within cap - // (or if we're at the end of the array). - (aCapBps == 0 || - resAndLimits.start_bitrate_bps <= aCapBps || - resAndLimits.resolution_in_mb == 0)) { - return resAndLimits; - } - } - - MOZ_CRASH("Loop should have handled fallback"); -} - -/** - * Function to set the encoding bitrate limits based on incoming frame size and rate - * @param width, height: dimensions of the frame - * @param min: minimum bitrate in bps - * @param start: bitrate in bps that the encoder should start with - * @param cap: user-enforced max bitrate, or 0 - * @param pref_cap: cap enforced by prefs - * @param negotiated_cap: cap negotiated through SDP - * @param aVideoStream stream to apply bitrates to - */ -static void -SelectBitrates( - unsigned short width, unsigned short height, - int min, int start, - int cap, int pref_cap, int negotiated_cap, - webrtc::VideoStream& aVideoStream) -{ - int& out_min = aVideoStream.min_bitrate_bps; - int& out_start = aVideoStream.target_bitrate_bps; - int& out_max = aVideoStream.max_bitrate_bps; - - WebrtcVideoConduit::ResolutionAndBitrateLimits resAndLimits = - GetLimitsFor(width, height); - out_min = MinIgnoreZero(resAndLimits.min_bitrate_bps, cap); - out_start = MinIgnoreZero(resAndLimits.start_bitrate_bps, cap); - out_max = MinIgnoreZero(resAndLimits.max_bitrate_bps, cap); - - // Note: negotiated_cap is the max transport bitrate - it applies to - // a single codec encoding, but should also apply to the sum of all - // simulcast layers in this encoding! So sum(layers.maxBitrate) <= - // negotiated_cap - // Note that out_max already has had pref_cap applied to it - out_max = MinIgnoreZero(negotiated_cap, out_max); - out_min = std::min(out_min, out_max); - out_start = std::min(out_start, out_max); - - if (min && min > out_min) { - out_min = min; - } - // If we try to set a minimum bitrate that is too low, ViE will reject it. - out_min = std::max(kViEMinCodecBitrate_bps, out_min); - out_max = std::max(kViEMinCodecBitrate_bps, out_max); - if (start && start > out_start) { - out_start = start; - } - - // Ensure that min <= start <= max - if (out_min > out_max) { - out_min = out_max; - } - out_start = std::min(out_max, std::max(out_start, out_min)); - - MOZ_ASSERT(pref_cap == 0 || out_max <= pref_cap); -} - /** * Perform validation on the codecConfig to be applied */ @@ -898,135 +801,6 @@ CodecsDifferent(const nsTArray>& a, return false; } -std::vector -WebrtcVideoConduit::VideoStreamFactory::CreateEncoderStreams( - int width, int height, const webrtc::VideoEncoderConfig& config) -{ - size_t streamCount = config.number_of_streams; - - // We only allow one layer when screensharing - if (mCodecMode == webrtc::VideoCodecMode::kScreensharing) { - streamCount = 1; - } - - std::vector streams; - streams.reserve(streamCount); - - // We assume that the first stream is the full-resolution stream. - - // This ensures all simulcast layers will be of the same aspect ratio as the input. - mSimulcastAdapter->OnOutputFormatRequest( - cricket::VideoFormat(width, height, 0, 0)); - - for (size_t idx = streamCount - 1; streamCount > 0; idx--, streamCount--) { - webrtc::VideoStream video_stream; - auto& simulcastEncoding = mCodecConfig.mSimulcastEncodings[idx]; - MOZ_ASSERT(simulcastEncoding.constraints.scaleDownBy >= 1.0); - - // All streams' dimensions must retain the aspect ratio of the input stream. - // Note that the first stream might already have been scaled by us. - // Webrtc.org doesn't know this, so we have to adjust lower layers manually. - int unusedCropWidth, unusedCropHeight, outWidth, outHeight; - if (idx == 0) { - // This is the highest-resolution stream. We avoid calling - // AdaptFrameResolution on this because precision errors in VideoAdapter - // can cause the out-resolution to be an odd pixel smaller than the - // source (1920x1419 has caused this). We shortcut this instead. - outWidth = width; - outHeight = height; - } else { - float effectiveScaleDownBy = - simulcastEncoding.constraints.scaleDownBy / - mCodecConfig.mSimulcastEncodings[0].constraints.scaleDownBy; - MOZ_ASSERT(effectiveScaleDownBy >= 1.0); - mSimulcastAdapter->OnScaleResolutionBy( - effectiveScaleDownBy > 1.0 ? - rtc::Optional(effectiveScaleDownBy) : - rtc::Optional()); - bool rv = mSimulcastAdapter->AdaptFrameResolution( - width, - height, - 0, // Ok, since we don't request an output format with an interval - &unusedCropWidth, - &unusedCropHeight, - &outWidth, - &outHeight); - - if (!rv) { - // The only thing that can make AdaptFrameResolution fail in this case - // is if this layer is scaled so far down that it has less than one pixel. - outWidth = 0; - outHeight = 0; - } - } - - if (outWidth == 0 || outHeight == 0) { - CSFLogInfo(LOGTAG, - "%s Stream with RID %s ignored because of no resolution.", - __FUNCTION__, simulcastEncoding.rid.c_str()); - continue; - } - - MOZ_ASSERT(outWidth > 0); - MOZ_ASSERT(outHeight > 0); - video_stream.width = outWidth; - video_stream.height = outHeight; - - CSFLogInfo(LOGTAG, "%s Input frame %ux%u, RID %s scaling to %zux%zu", - __FUNCTION__, width, height, simulcastEncoding.rid.c_str(), - video_stream.width, video_stream.height); - - if (video_stream.width * height != width * video_stream.height) { - CSFLogInfo(LOGTAG, - "%s Stream with RID %s ignored because of bad aspect ratio.", - __FUNCTION__, simulcastEncoding.rid.c_str()); - continue; - } - - // We want to ensure this picks up the current framerate, so indirect - video_stream.max_framerate = mSendingFramerate; - - SelectBitrates( - video_stream.width, video_stream.height, - mMinBitrate, mStartBitrate, simulcastEncoding.constraints.maxBr, - mPrefMaxBitrate, mNegotiatedMaxBitrate, video_stream); - - video_stream.max_qp = kQpMax; - video_stream.SetRid(simulcastEncoding.rid); - - // leave vector temporal_layer_thresholds_bps empty for non-simulcast - video_stream.temporal_layer_thresholds_bps.clear(); - if (config.number_of_streams > 1) { - // XXX Note: in simulcast.cc in upstream code, the array value is - // 3(-1) for all streams, though it's in an array, except for screencasts, - // which use 1 (i.e 2 layers). - - // Oddly, though this is a 'bps' array, nothing really looks at the - // values for normal video, just the size of the array to know the - // number of temporal layers. - // For VideoEncoderConfig::ContentType::kScreen, though, in - // video_codec_initializer.cc it uses [0] to set the target bitrate - // for the screenshare. - if (mCodecMode == webrtc::VideoCodecMode::kScreensharing) { - video_stream.temporal_layer_thresholds_bps.push_back(video_stream.target_bitrate_bps); - } else { - video_stream.temporal_layer_thresholds_bps.resize(2); - } - // XXX Bug 1390215 investigate using more of - // simulcast.cc:GetSimulcastConfig() or our own algorithm to replace it - } - - if (mCodecConfig.mName == "H264") { - if (mCodecConfig.mEncodingConstraints.maxMbps > 0) { - // Not supported yet! - CSFLogError(LOGTAG, "%s H.264 max_mbps not supported yet", __FUNCTION__); - } - } - streams.push_back(video_stream); - } - return streams; -} - /** * Note: Setting the send-codec on the Video Engine will restart the encoder, * sets up new SSRC and reset RTP_RTCP module with the new codec setting. @@ -2654,7 +2428,7 @@ WebrtcVideoConduit::VideoEncoderConfigBuilder::SetEncoderSpecificSettings( } void -WebrtcVideoConduit::VideoEncoderConfigBuilder::SetVideoStreamFactory(rtc::scoped_refptr aFactory) +WebrtcVideoConduit::VideoEncoderConfigBuilder::SetVideoStreamFactory(rtc::scoped_refptr aFactory) { MOZ_ASSERT(NS_IsMainThread()); diff --git a/media/webrtc/signaling/src/media-conduit/VideoConduit.h b/media/webrtc/signaling/src/media-conduit/VideoConduit.h index 74e77bfe3e96..3ff0ed73a600 100644 --- a/media/webrtc/signaling/src/media-conduit/VideoConduit.h +++ b/media/webrtc/signaling/src/media-conduit/VideoConduit.h @@ -40,9 +40,17 @@ namespace mozilla { +// Convert (SI) kilobits/sec to (SI) bits/sec +#define KBPS(kbps) kbps * 1000 + +const int kViEMinCodecBitrate_bps = KBPS(30); const unsigned int kVideoMtu = 1200; const int kQpMax = 56; +template +T MinIgnoreZero(const T& a, const T& b); + +class VideoStreamFactory; class WebrtcAudioConduit; class nsThread; @@ -68,14 +76,6 @@ class WebrtcVideoConduit : public VideoSessionConduit , public rtc::VideoSourceInterface { public: - struct ResolutionAndBitrateLimits - { - int resolution_in_mb; - int min_bitrate_bps; - int start_bitrate_bps; - int max_bitrate_bps; - }; - //VoiceEngine defined constant for Payload Name Size. static const unsigned int CODEC_PLNAME_SIZE; @@ -421,8 +421,6 @@ private: * Stores encoder configuration information and produces * a VideoEncoderConfig from it. */ - class VideoStreamFactory; - class VideoEncoderConfigBuilder { public: /** @@ -434,7 +432,7 @@ private: double jsScaleDownBy=1.0; // user-controlled downscale }; void SetEncoderSpecificSettings(rtc::scoped_refptr aSettings); - void SetVideoStreamFactory(rtc::scoped_refptr aFactory); + void SetVideoStreamFactory(rtc::scoped_refptr aFactory); void SetMinTransmitBitrateBps(int aXmitMinBps); void SetContentType(webrtc::VideoEncoderConfig::ContentType aContentType); void SetResolutionDivisor(unsigned char aDivisor); @@ -456,65 +454,6 @@ private: // Utility function to dump recv codec database void DumpCodecDB() const; - // Factory class for VideoStreams... vie_encoder.cc will call this to reconfigure. - class VideoStreamFactory : public webrtc::VideoEncoderConfig::VideoStreamFactoryInterface - { - public: - VideoStreamFactory(VideoCodecConfig aConfig, - webrtc::VideoCodecMode aCodecMode, - int aMinBitrate, int aStartBitrate, - int aPrefMaxBitrate, int aNegotiatedMaxBitrate, - unsigned int aSendingFramerate) - : mCodecMode(aCodecMode) - , mSendingFramerate(aSendingFramerate) - , mCodecConfig(std::forward(aConfig)) - , mMinBitrate(aMinBitrate) - , mStartBitrate(aStartBitrate) - , mPrefMaxBitrate(aPrefMaxBitrate) - , mNegotiatedMaxBitrate(aNegotiatedMaxBitrate) - , mSimulcastAdapter(MakeUnique()) - {} - - void SetCodecMode(webrtc::VideoCodecMode aCodecMode) - { - MOZ_ASSERT(NS_IsMainThread()); - mCodecMode = aCodecMode; - } - - void SetSendingFramerate(unsigned int aSendingFramerate) - { - MOZ_ASSERT(NS_IsMainThread()); - mSendingFramerate = aSendingFramerate; - } - - private: - // This gets called off-main thread and may hold internal webrtc.org - // locks. May *NOT* lock the conduit's mutex, to avoid deadlocks. - std::vector - CreateEncoderStreams(int width, int height, - const webrtc::VideoEncoderConfig& config) override; - - // Used to limit number of streams for screensharing. - Atomic mCodecMode; - - // The framerate we're currently sending at. - Atomic mSendingFramerate; - - // The current send codec config, containing simulcast layer configs. - const VideoCodecConfig mCodecConfig; - - // Bitrate limits in bps. - const int mMinBitrate = 0; - const int mStartBitrate = 0; - const int mPrefMaxBitrate = 0; - const int mNegotiatedMaxBitrate = 0; - - // Adapter for simulcast layers. We use this to handle scaleResolutionDownBy - // for layers. It's separate from the conduit's mVideoAdapter to not affect - // scaling settings for incoming frames. - UniquePtr mSimulcastAdapter; - }; - // Video Latency Test averaging filter void VideoLatencyUpdate(uint64_t new_sample); diff --git a/media/webrtc/signaling/src/media-conduit/VideoStreamFactory.cpp b/media/webrtc/signaling/src/media-conduit/VideoStreamFactory.cpp new file mode 100644 index 000000000000..f16d5910aefd --- /dev/null +++ b/media/webrtc/signaling/src/media-conduit/VideoStreamFactory.cpp @@ -0,0 +1,258 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + +#include "VideoStreamFactory.h" + +#include "CSFLog.h" +#include "nsThreadUtils.h" +#include "VideoConduit.h" + +namespace mozilla { + +#ifdef LOGTAG +#undef LOGTAG +#endif +#define LOGTAG "WebrtcVideoSessionConduit" + +#define MB_OF(w,h) ((unsigned int)((((w+15)>>4))*((unsigned int)((h+15)>>4)))) +// For now, try to set the max rates well above the knee in the curve. +// Chosen somewhat arbitrarily; it's hard to find good data oriented for +// realtime interactive/talking-head recording. These rates assume +// 30fps. + +// XXX Populate this based on a pref (which we should consider sorting because +// people won't assume they need to). +static VideoStreamFactory::ResolutionAndBitrateLimits kResolutionAndBitrateLimits[] = { + {MB_OF(1920, 1200), KBPS(1500), KBPS(2000), KBPS(10000)}, // >HD (3K, 4K, etc) + {MB_OF(1280, 720), KBPS(1200), KBPS(1500), KBPS(5000)}, // HD ~1080-1200 + {MB_OF(800, 480), KBPS(600), KBPS(800), KBPS(2500)}, // HD ~720 + {MB_OF(480, 270), KBPS(150), KBPS(500), KBPS(2000)}, // WVGA + {tl::Max::value, KBPS(125), KBPS(300), KBPS(1300)}, // VGA + {MB_OF(176, 144), KBPS(100), KBPS(150), KBPS(500)}, // WQVGA, CIF + {0 , KBPS(40), KBPS(80), KBPS(250)} // QCIF and below +}; + +static VideoStreamFactory::ResolutionAndBitrateLimits +GetLimitsFor(unsigned int aWidth, unsigned int aHeight, int aCapBps = 0) +{ + // max bandwidth should be proportional (not linearly!) to resolution, and + // proportional (perhaps linearly, or close) to current frame rate. + int fs = MB_OF(aWidth, aHeight); + + for (const auto& resAndLimits : kResolutionAndBitrateLimits) { + if (fs > resAndLimits.resolution_in_mb && + // pick the highest range where at least start rate is within cap + // (or if we're at the end of the array). + (aCapBps == 0 || + resAndLimits.start_bitrate_bps <= aCapBps || + resAndLimits.resolution_in_mb == 0)) { + return resAndLimits; + } + } + + MOZ_CRASH("Loop should have handled fallback"); +} + +/** + * Function to set the encoding bitrate limits based on incoming frame size and rate + * @param width, height: dimensions of the frame + * @param min: minimum bitrate in bps + * @param start: bitrate in bps that the encoder should start with + * @param cap: user-enforced max bitrate, or 0 + * @param pref_cap: cap enforced by prefs + * @param negotiated_cap: cap negotiated through SDP + * @param aVideoStream stream to apply bitrates to + */ +static void +SelectBitrates( + unsigned short width, unsigned short height, + int min, int start, + int cap, int pref_cap, int negotiated_cap, + webrtc::VideoStream& aVideoStream) +{ + int& out_min = aVideoStream.min_bitrate_bps; + int& out_start = aVideoStream.target_bitrate_bps; + int& out_max = aVideoStream.max_bitrate_bps; + + VideoStreamFactory::ResolutionAndBitrateLimits resAndLimits = + GetLimitsFor(width, height); + out_min = MinIgnoreZero(resAndLimits.min_bitrate_bps, cap); + out_start = MinIgnoreZero(resAndLimits.start_bitrate_bps, cap); + out_max = MinIgnoreZero(resAndLimits.max_bitrate_bps, cap); + + // Note: negotiated_cap is the max transport bitrate - it applies to + // a single codec encoding, but should also apply to the sum of all + // simulcast layers in this encoding! So sum(layers.maxBitrate) <= + // negotiated_cap + // Note that out_max already has had pref_cap applied to it + out_max = MinIgnoreZero(negotiated_cap, out_max); + out_min = std::min(out_min, out_max); + out_start = std::min(out_start, out_max); + + if (min && min > out_min) { + out_min = min; + } + // If we try to set a minimum bitrate that is too low, ViE will reject it. + out_min = std::max(kViEMinCodecBitrate_bps, out_min); + out_max = std::max(kViEMinCodecBitrate_bps, out_max); + if (start && start > out_start) { + out_start = start; + } + + // Ensure that min <= start <= max + if (out_min > out_max) { + out_min = out_max; + } + out_start = std::min(out_max, std::max(out_start, out_min)); + + MOZ_ASSERT(pref_cap == 0 || out_max <= pref_cap); +} + +void +VideoStreamFactory::SetCodecMode(webrtc::VideoCodecMode aCodecMode) +{ + MOZ_ASSERT(NS_IsMainThread()); + mCodecMode = aCodecMode; +} + +void +VideoStreamFactory::SetSendingFramerate(unsigned int aSendingFramerate) +{ + MOZ_ASSERT(NS_IsMainThread()); + mSendingFramerate = aSendingFramerate; +} + +std::vector +VideoStreamFactory::CreateEncoderStreams( + int width, int height, const webrtc::VideoEncoderConfig& config) +{ + size_t streamCount = config.number_of_streams; + + // We only allow one layer when screensharing + if (mCodecMode == webrtc::VideoCodecMode::kScreensharing) { + streamCount = 1; + } + + std::vector streams; + streams.reserve(streamCount); + + // We assume that the first stream is the full-resolution stream. + + // This ensures all simulcast layers will be of the same aspect ratio as the input. + mSimulcastAdapter->OnOutputFormatRequest( + cricket::VideoFormat(width, height, 0, 0)); + + for (size_t idx = streamCount - 1; streamCount > 0; idx--, streamCount--) { + webrtc::VideoStream video_stream; + auto& simulcastEncoding = mCodecConfig.mSimulcastEncodings[idx]; + MOZ_ASSERT(simulcastEncoding.constraints.scaleDownBy >= 1.0); + + // All streams' dimensions must retain the aspect ratio of the input stream. + // Note that the first stream might already have been scaled by us. + // Webrtc.org doesn't know this, so we have to adjust lower layers manually. + int unusedCropWidth, unusedCropHeight, outWidth, outHeight; + if (idx == 0) { + // This is the highest-resolution stream. We avoid calling + // AdaptFrameResolution on this because precision errors in VideoAdapter + // can cause the out-resolution to be an odd pixel smaller than the + // source (1920x1419 has caused this). We shortcut this instead. + outWidth = width; + outHeight = height; + } else { + float effectiveScaleDownBy = + simulcastEncoding.constraints.scaleDownBy / + mCodecConfig.mSimulcastEncodings[0].constraints.scaleDownBy; + MOZ_ASSERT(effectiveScaleDownBy >= 1.0); + mSimulcastAdapter->OnScaleResolutionBy( + effectiveScaleDownBy > 1.0 ? + rtc::Optional(effectiveScaleDownBy) : + rtc::Optional()); + bool rv = mSimulcastAdapter->AdaptFrameResolution( + width, + height, + 0, // Ok, since we don't request an output format with an interval + &unusedCropWidth, + &unusedCropHeight, + &outWidth, + &outHeight); + + if (!rv) { + // The only thing that can make AdaptFrameResolution fail in this case + // is if this layer is scaled so far down that it has less than one pixel. + outWidth = 0; + outHeight = 0; + } + } + + if (outWidth == 0 || outHeight == 0) { + CSFLogInfo(LOGTAG, + "%s Stream with RID %s ignored because of no resolution.", + __FUNCTION__, simulcastEncoding.rid.c_str()); + continue; + } + + MOZ_ASSERT(outWidth > 0); + MOZ_ASSERT(outHeight > 0); + video_stream.width = outWidth; + video_stream.height = outHeight; + + CSFLogInfo(LOGTAG, "%s Input frame %ux%u, RID %s scaling to %zux%zu", + __FUNCTION__, width, height, simulcastEncoding.rid.c_str(), + video_stream.width, video_stream.height); + + if (video_stream.width * height != width * video_stream.height) { + CSFLogInfo(LOGTAG, + "%s Stream with RID %s ignored because of bad aspect ratio.", + __FUNCTION__, simulcastEncoding.rid.c_str()); + continue; + } + + // We want to ensure this picks up the current framerate, so indirect + video_stream.max_framerate = mSendingFramerate; + + SelectBitrates( + video_stream.width, video_stream.height, + mMinBitrate, mStartBitrate, simulcastEncoding.constraints.maxBr, + mPrefMaxBitrate, mNegotiatedMaxBitrate, video_stream); + + video_stream.max_qp = kQpMax; + video_stream.SetRid(simulcastEncoding.rid); + + // leave vector temporal_layer_thresholds_bps empty for non-simulcast + video_stream.temporal_layer_thresholds_bps.clear(); + if (config.number_of_streams > 1) { + // XXX Note: in simulcast.cc in upstream code, the array value is + // 3(-1) for all streams, though it's in an array, except for screencasts, + // which use 1 (i.e 2 layers). + + // Oddly, though this is a 'bps' array, nothing really looks at the + // values for normal video, just the size of the array to know the + // number of temporal layers. + // For VideoEncoderConfig::ContentType::kScreen, though, in + // video_codec_initializer.cc it uses [0] to set the target bitrate + // for the screenshare. + if (mCodecMode == webrtc::VideoCodecMode::kScreensharing) { + video_stream.temporal_layer_thresholds_bps.push_back(video_stream.target_bitrate_bps); + } else { + video_stream.temporal_layer_thresholds_bps.resize(2); + } + // XXX Bug 1390215 investigate using more of + // simulcast.cc:GetSimulcastConfig() or our own algorithm to replace it + } + + if (mCodecConfig.mName == "H264") { + if (mCodecConfig.mEncodingConstraints.maxMbps > 0) { + // Not supported yet! + CSFLogError(LOGTAG, "%s H.264 max_mbps not supported yet", __FUNCTION__); + } + } + streams.push_back(video_stream); + } + return streams; +} + +} // namespace mozilla + diff --git a/media/webrtc/signaling/src/media-conduit/VideoStreamFactory.h b/media/webrtc/signaling/src/media-conduit/VideoStreamFactory.h new file mode 100644 index 000000000000..ca24007c0a7b --- /dev/null +++ b/media/webrtc/signaling/src/media-conduit/VideoStreamFactory.h @@ -0,0 +1,79 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + +#ifndef VideoStreamFactory_h +#define VideoStreamFactory_h + +#include "CodecConfig.h" +#include "mozilla/Atomics.h" +#include "mozilla/UniquePtr.h" +#include "webrtc/config.h" +#include "webrtc/media/base/videoadapter.h" + +namespace mozilla { + +// Factory class for VideoStreams... vie_encoder.cc will call this to reconfigure. +class VideoStreamFactory : public webrtc::VideoEncoderConfig::VideoStreamFactoryInterface +{ +public: + struct ResolutionAndBitrateLimits + { + int resolution_in_mb; + int min_bitrate_bps; + int start_bitrate_bps; + int max_bitrate_bps; + }; + + VideoStreamFactory(VideoCodecConfig aConfig, + webrtc::VideoCodecMode aCodecMode, + int aMinBitrate, int aStartBitrate, + int aPrefMaxBitrate, int aNegotiatedMaxBitrate, + unsigned int aSendingFramerate) + : mCodecMode(aCodecMode) + , mSendingFramerate(aSendingFramerate) + , mCodecConfig(std::forward(aConfig)) + , mMinBitrate(aMinBitrate) + , mStartBitrate(aStartBitrate) + , mPrefMaxBitrate(aPrefMaxBitrate) + , mNegotiatedMaxBitrate(aNegotiatedMaxBitrate) + , mSimulcastAdapter(MakeUnique()) + {} + + void SetCodecMode(webrtc::VideoCodecMode aCodecMode); + void SetSendingFramerate(unsigned int aSendingFramerate); + + // This gets called off-main thread and may hold internal webrtc.org + // locks. May *NOT* lock the conduit's mutex, to avoid deadlocks. + std::vector + CreateEncoderStreams(int width, int height, + const webrtc::VideoEncoderConfig& config) override; + +private: + // Used to limit number of streams for screensharing. + Atomic mCodecMode; + + // The framerate we're currently sending at. + Atomic mSendingFramerate; + + // The current send codec config, containing simulcast layer configs. + const VideoCodecConfig mCodecConfig; + + // Bitrate limits in bps. + const int mMinBitrate = 0; + const int mStartBitrate = 0; + const int mPrefMaxBitrate = 0; + const int mNegotiatedMaxBitrate = 0; + + // Adapter for simulcast layers. We use this to handle scaleResolutionDownBy + // for layers. It's separate from the conduit's mVideoAdapter to not affect + // scaling settings for incoming frames. + UniquePtr mSimulcastAdapter; +}; + +} // namespace mozilla + +#endif + diff --git a/media/webrtc/signaling/src/media-conduit/moz.build b/media/webrtc/signaling/src/media-conduit/moz.build index ef663638b2d9..f4dfbca4c283 100644 --- a/media/webrtc/signaling/src/media-conduit/moz.build +++ b/media/webrtc/signaling/src/media-conduit/moz.build @@ -24,6 +24,7 @@ UNIFIED_SOURCES += [ 'MediaDataDecoderCodec.cpp', 'RtpSourceObserver.cpp', 'VideoConduit.cpp', + 'VideoStreamFactory.cpp', 'WebrtcGmpVideoCodec.cpp', 'WebrtcMediaDataDecoderCodec.cpp', ]