From eb74247a9a9a62774c3f6f3df389b3d6924872c7 Mon Sep 17 00:00:00 2001 From: "julian.reschke@gmx.de" Date: Sun, 29 Apr 2012 13:40:00 -0700 Subject: [PATCH 001/162] Bug 693806: RFC2231/5987 encoding: charset information should be treated as authoritative. r=jduell --- netwerk/mime/nsMIMEHeaderParamImpl.cpp | 46 ++++++++++++++++++++++++-- netwerk/test/unit/test_MIME_params.js | 32 ++++++++++++++++++ 2 files changed, 76 insertions(+), 2 deletions(-) diff --git a/netwerk/mime/nsMIMEHeaderParamImpl.cpp b/netwerk/mime/nsMIMEHeaderParamImpl.cpp index 1d6ac5de7ae..6a2654c5226 100644 --- a/netwerk/mime/nsMIMEHeaderParamImpl.cpp +++ b/netwerk/mime/nsMIMEHeaderParamImpl.cpp @@ -298,6 +298,35 @@ PRInt32 parseSegmentNumber(const char *aValue, PRInt32 aLen) return segmentNumber; } +// validate a given octet sequence for compliance with the specified +// encoding +bool IsValidOctetSequenceForCharset(nsACString& aCharset, const char *aOctets) +{ + nsCOMPtr cvtUTF8(do_GetService + (NS_UTF8CONVERTERSERVICE_CONTRACTID)); + if (!cvtUTF8) { + NS_WARNING("Can't get UTF8ConverterService\n"); + return false; + } + + nsCAutoString tmpRaw; + tmpRaw.Assign(aOctets); + nsCAutoString tmpDecoded; + + nsresult rv = cvtUTF8->ConvertStringToUTF8(tmpRaw, + PromiseFlatCString(aCharset).get(), + true, tmpDecoded); + + if (rv != NS_OK) { + // we can't decode; charset may be unsupported, or the octet sequence + // is broken (illegal or incomplete octet sequence contained) + NS_WARNING("RFC2231/5987 parameter value does not decode according to specified charset\n"); + return false; + } + + return true; +} + // moved almost verbatim from mimehdrs.cpp // char * // MimeHeaders_get_parameter (const char *header_value, const char *parm_name, @@ -617,6 +646,20 @@ increment_str: caseCDResult = combineContinuations(segments); + if (caseBResult && !charsetB.IsEmpty()) { + // check that the 2231/5987 result decodes properly given the + // specified character set + if (!IsValidOctetSequenceForCharset(charsetB, caseBResult)) + caseBResult = NULL; + } + + if (caseCDResult && !charsetCD.IsEmpty()) { + // check that the 2231/5987 result decodes properly given the + // specified character set + if (!IsValidOctetSequenceForCharset(charsetCD, caseCDResult)) + caseCDResult = NULL; + } + if (caseBResult) { // prefer simple 5987 format over 2231 with continuations *aResult = caseBResult; @@ -716,9 +759,8 @@ nsMIMEHeaderParamImpl::DecodeParameter(const nsACString& aParamValue, { nsCOMPtr cvtUTF8(do_GetService(NS_UTF8CONVERTERSERVICE_CONTRACTID)); if (cvtUTF8) - // skip ASCIIness/UTF8ness test if aCharset is 7bit non-ascii charset. return cvtUTF8->ConvertStringToUTF8(aParamValue, aCharset, - IS_7BIT_NON_ASCII_CHARSET(aCharset), aResult); + true, aResult); } const nsAFlatCString& param = PromiseFlatCString(aParamValue); diff --git a/netwerk/test/unit/test_MIME_params.js b/netwerk/test/unit/test_MIME_params.js index 165aa5dd239..29873d1576f 100644 --- a/netwerk/test/unit/test_MIME_params.js +++ b/netwerk/test/unit/test_MIME_params.js @@ -330,6 +330,38 @@ var tests = [ // previously with the fix for 692574: // "attachment", "bar.html"], + // Bug 693806: RFC2231/5987 encoding: charset information should be treated + // as authoritative + + // UTF-8 labeled ISO-8859-1 + ["attachment; filename*=ISO-8859-1''%c3%a4", + "attachment", "\u00c3\u00a4"], + + // UTF-8 labeled ISO-8859-1, but with octets not allowed in ISO-8859-1 + // accepts x82, understands it as Win1252, maps it to Unicode \u20a1 + ["attachment; filename*=ISO-8859-1''%e2%82%ac", + "attachment", "\u00e2\u201a\u00ac"], + + // defective UTF-8 + ["attachment; filename*=UTF-8''A%e4B", + "attachment", Cr.NS_ERROR_INVALID_ARG], + + // defective UTF-8, with fallback + ["attachment; filename*=UTF-8''A%e4B; filename=fallback", + "attachment", "fallback"], + + // defective UTF-8 (continuations), with fallback + ["attachment; filename*0*=UTF-8''A%e4B; filename=fallback", + "attachment", "fallback"], + + // check that charsets aren't mixed up + ["attachment; filename*0*=ISO-8859-15''euro-sign%3d%a4; filename*=ISO-8859-1''currency-sign%3d%a4", + "attachment", "currency-sign=\u00a4"], + + // same as above, except reversed + ["attachment; filename*=ISO-8859-1''currency-sign%3d%a4; filename*0*=ISO-8859-15''euro-sign%3d%a4", + "attachment", "currency-sign=\u00a4"], + // Bug 704989: add workaround for broken Outlook Web App (OWA) // attachment handling From fd7631eab2241c26a1b40b66e2945f857f0c4dba Mon Sep 17 00:00:00 2001 From: Mark Capella Date: Fri, 27 Apr 2012 12:47:00 +0200 Subject: [PATCH 002/162] Bug 723530 - double error reporting in ctype JS api implementation, r=bholley --- js/src/ctypes/CTypes.cpp | 79 +++++++++++++++++++-------- js/src/ctypes/Library.cpp | 12 +++- js/xpconnect/wrappers/XrayWrapper.cpp | 4 +- 3 files changed, 69 insertions(+), 26 deletions(-) diff --git a/js/src/ctypes/CTypes.cpp b/js/src/ctypes/CTypes.cpp index 1f9e337f024..1c479a929b0 100644 --- a/js/src/ctypes/CTypes.cpp +++ b/js/src/ctypes/CTypes.cpp @@ -3404,7 +3404,9 @@ JSBool CType::CreateArray(JSContext* cx, unsigned argc, jsval* vp) { JSObject* baseType = JS_THIS_OBJECT(cx, vp); - if (!baseType || !CType::IsCType(baseType)) { + if (!baseType) + return JS_FALSE; + if (!CType::IsCType(baseType)) { JS_ReportError(cx, "not a CType"); return JS_FALSE; } @@ -3435,7 +3437,9 @@ JSBool CType::ToString(JSContext* cx, unsigned argc, jsval* vp) { JSObject* obj = JS_THIS_OBJECT(cx, vp); - if (!obj || !(CType::IsCType(obj) || CType::IsCTypeProto(obj))) { + if (!obj) + return JS_FALSE; + if (!CType::IsCType(obj) && !CType::IsCTypeProto(obj)) { JS_ReportError(cx, "not a CType"); return JS_FALSE; } @@ -3463,8 +3467,9 @@ JSBool CType::ToSource(JSContext* cx, unsigned argc, jsval* vp) { JSObject* obj = JS_THIS_OBJECT(cx, vp); - if (!obj || - !(CType::IsCType(obj) || CType::IsCTypeProto(obj))) + if (!obj) + return JS_FALSE; + if (!CType::IsCType(obj) && !CType::IsCTypeProto(obj)) { JS_ReportError(cx, "not a CType"); return JS_FALSE; @@ -3699,7 +3704,9 @@ JSBool PointerType::IsNull(JSContext* cx, unsigned argc, jsval* vp) { JSObject* obj = JS_THIS_OBJECT(cx, vp); - if (!obj || !CData::IsCData(obj)) { + if (!obj) + return JS_FALSE; + if (!CData::IsCData(obj)) { JS_ReportError(cx, "not a CData"); return JS_FALSE; } @@ -3721,7 +3728,9 @@ JSBool PointerType::OffsetBy(JSContext* cx, int offset, jsval* vp) { JSObject* obj = JS_THIS_OBJECT(cx, vp); - if (!obj || !CData::IsCData(obj)) { + if (!obj) + return JS_FALSE; + if (!CData::IsCData(obj)) { JS_ReportError(cx, "not a CData"); return JS_FALSE; } @@ -4226,7 +4235,9 @@ JSBool ArrayType::AddressOfElement(JSContext* cx, unsigned argc, jsval* vp) { JSObject* obj = JS_THIS_OBJECT(cx, vp); - if (!obj || !CData::IsCData(obj)) { + if (!obj) + return JS_FALSE; + if (!CData::IsCData(obj)) { JS_ReportError(cx, "not a CData"); return JS_FALSE; } @@ -4602,8 +4613,9 @@ JSBool StructType::Define(JSContext* cx, unsigned argc, jsval* vp) { JSObject* obj = JS_THIS_OBJECT(cx, vp); - if (!obj || - !CType::IsCType(obj) || + if (!obj) + return JS_FALSE; + if (!CType::IsCType(obj) || CType::GetTypeCode(obj) != TYPE_struct) { JS_ReportError(cx, "not a StructType"); return JS_FALSE; @@ -4849,7 +4861,9 @@ JSBool StructType::AddressOfField(JSContext* cx, unsigned argc, jsval* vp) { JSObject* obj = JS_THIS_OBJECT(cx, vp); - if (!obj || !CData::IsCData(obj)) { + if (!obj) + return JS_FALSE; + if (!CData::IsCData(obj)) { JS_ReportError(cx, "not a CData"); return JS_FALSE; } @@ -6075,7 +6089,9 @@ CData::Address(JSContext* cx, unsigned argc, jsval* vp) } JSObject* obj = JS_THIS_OBJECT(cx, vp); - if (!obj || !IsCData(obj)) { + if (!obj) + return JS_FALSE; + if (!IsCData(obj)) { JS_ReportError(cx, "not a CData"); return JS_FALSE; } @@ -6183,7 +6199,9 @@ CData::ReadString(JSContext* cx, unsigned argc, jsval* vp) } JSObject* obj = JS_THIS_OBJECT(cx, vp); - if (!obj || !IsCData(obj)) { + if (!obj) + return JS_FALSE; + if (!IsCData(obj)) { JS_ReportError(cx, "not a CData"); return JS_FALSE; } @@ -6294,8 +6312,9 @@ CData::ToSource(JSContext* cx, unsigned argc, jsval* vp) } JSObject* obj = JS_THIS_OBJECT(cx, vp); - if (!obj || - !(CData::IsCData(obj) || CData::IsCDataProto(obj))) { + if (!obj) + return JS_FALSE; + if (!CData::IsCData(obj) && !CData::IsCDataProto(obj)) { JS_ReportError(cx, "not a CData"); return JS_FALSE; } @@ -6348,7 +6367,9 @@ JSBool CDataFinalizer::Methods::ToSource(JSContext *cx, unsigned argc, jsval *vp) { JSObject* objThis = JS_THIS_OBJECT(cx, vp); - if (!objThis || !CDataFinalizer::IsCDataFinalizer(objThis)) { + if (!objThis) + return JS_FALSE; + if (!CDataFinalizer::IsCDataFinalizer(objThis)) { JS_ReportError(cx, "not a CDataFinalizer"); return JS_FALSE; } @@ -6405,7 +6426,9 @@ JSBool CDataFinalizer::Methods::ToString(JSContext *cx, unsigned argc, jsval *vp) { JSObject* objThis = JS_THIS_OBJECT(cx, vp); - if (!objThis || !CDataFinalizer::IsCDataFinalizer(objThis)) { + if (!objThis) + return JS_FALSE; + if (!CDataFinalizer::IsCDataFinalizer(objThis)) { JS_ReportError(cx, "not a CDataFinalizer"); return JS_FALSE; } @@ -6697,7 +6720,9 @@ CDataFinalizer::Methods::Forget(JSContext* cx, unsigned argc, jsval *vp) } JSObject *obj = JS_THIS_OBJECT(cx, vp); - if (!obj || !CDataFinalizer::IsCDataFinalizer(obj)) { + if (!obj) + return JS_FALSE; + if (!CDataFinalizer::IsCDataFinalizer(obj)) { return TypeError(cx, "a CDataFinalizer", OBJECT_TO_JSVAL(obj)); } @@ -6740,7 +6765,9 @@ CDataFinalizer::Methods::Dispose(JSContext* cx, unsigned argc, jsval *vp) } JSObject *obj = JS_THIS_OBJECT(cx, vp); - if (!obj || !CDataFinalizer::IsCDataFinalizer(obj)) { + if (!obj) + return JS_FALSE; + if (!CDataFinalizer::IsCDataFinalizer(obj)) { return TypeError(cx, "a CDataFinalizer", OBJECT_TO_JSVAL(obj)); } @@ -7008,7 +7035,9 @@ JSBool Int64::ToString(JSContext* cx, unsigned argc, jsval* vp) { JSObject* obj = JS_THIS_OBJECT(cx, vp); - if (!obj || !Int64::IsInt64(obj)) { + if (!obj) + return JS_FALSE; + if (!Int64::IsInt64(obj)) { JS_ReportError(cx, "not an Int64"); return JS_FALSE; } @@ -7020,7 +7049,9 @@ JSBool Int64::ToSource(JSContext* cx, unsigned argc, jsval* vp) { JSObject* obj = JS_THIS_OBJECT(cx, vp); - if (!obj || !Int64::IsInt64(obj)) { + if (!obj) + return JS_FALSE; + if (!Int64::IsInt64(obj)) { JS_ReportError(cx, "not an Int64"); return JS_FALSE; } @@ -7179,7 +7210,9 @@ JSBool UInt64::ToString(JSContext* cx, unsigned argc, jsval* vp) { JSObject* obj = JS_THIS_OBJECT(cx, vp); - if (!obj || !UInt64::IsUInt64(obj)) { + if (!obj) + return JS_FALSE; + if (!UInt64::IsUInt64(obj)) { JS_ReportError(cx, "not a UInt64"); return JS_FALSE; } @@ -7191,7 +7224,9 @@ JSBool UInt64::ToSource(JSContext* cx, unsigned argc, jsval* vp) { JSObject* obj = JS_THIS_OBJECT(cx, vp); - if (!obj || !UInt64::IsUInt64(obj)) { + if (!obj) + return JS_FALSE; + if (!UInt64::IsUInt64(obj)) { JS_ReportError(cx, "not a UInt64"); return JS_FALSE; } diff --git a/js/src/ctypes/Library.cpp b/js/src/ctypes/Library.cpp index 8aaccb47d12..6e946afb068 100644 --- a/js/src/ctypes/Library.cpp +++ b/js/src/ctypes/Library.cpp @@ -223,7 +223,9 @@ JSBool Library::Open(JSContext* cx, unsigned argc, jsval *vp) { JSObject* ctypesObj = JS_THIS_OBJECT(cx, vp); - if (!ctypesObj || !IsCTypesGlobal(ctypesObj)) { + if (!ctypesObj) + return JS_FALSE; + if (!IsCTypesGlobal(ctypesObj)) { JS_ReportError(cx, "not a ctypes object"); return JS_FALSE; } @@ -245,7 +247,9 @@ JSBool Library::Close(JSContext* cx, unsigned argc, jsval* vp) { JSObject* obj = JS_THIS_OBJECT(cx, vp); - if (!obj || !IsLibrary(obj)) { + if (!obj) + return JS_FALSE; + if (!IsLibrary(obj)) { JS_ReportError(cx, "not a library"); return JS_FALSE; } @@ -267,7 +271,9 @@ JSBool Library::Declare(JSContext* cx, unsigned argc, jsval* vp) { JSObject* obj = JS_THIS_OBJECT(cx, vp); - if (!obj || !IsLibrary(obj)) { + if (!obj) + return JS_FALSE; + if (!IsLibrary(obj)) { JS_ReportError(cx, "not a library"); return JS_FALSE; } diff --git a/js/xpconnect/wrappers/XrayWrapper.cpp b/js/xpconnect/wrappers/XrayWrapper.cpp index 69ed63f6b09..1ee6ea4c599 100644 --- a/js/xpconnect/wrappers/XrayWrapper.cpp +++ b/js/xpconnect/wrappers/XrayWrapper.cpp @@ -1012,7 +1012,9 @@ static JSBool XrayToString(JSContext *cx, unsigned argc, jsval *vp) { JSObject *wrapper = JS_THIS_OBJECT(cx, vp); - if (!wrapper || !IsWrapper(wrapper) || !WrapperFactory::IsXrayWrapper(wrapper)) { + if (!wrapper) + return false; + if (!IsWrapper(wrapper) || !WrapperFactory::IsXrayWrapper(wrapper)) { JS_ReportError(cx, "XrayToString called on an incompatible object"); return false; } From 29f191e9a6ee5b6c7b8855ae0a8b5d27a0e2024e Mon Sep 17 00:00:00 2001 From: Andreas Gal Date: Sun, 29 Apr 2012 14:41:13 -0700 Subject: [PATCH 003/162] Render active layers at full resolution and transform them on the GPU (bug 750006, r=roc). --- gfx/thebes/gfxUtils.cpp | 4 +++- layout/base/FrameLayerBuilder.cpp | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/gfx/thebes/gfxUtils.cpp b/gfx/thebes/gfxUtils.cpp index c59c20b78d4..5247b399d76 100644 --- a/gfx/thebes/gfxUtils.cpp +++ b/gfx/thebes/gfxUtils.cpp @@ -547,7 +547,9 @@ gfxUtils::ClampToScaleFactor(gfxFloat aVal) power = ceil(power); } - return pow(kScaleResolution, power); + gfxFloat scale = pow(kScaleResolution, power); + + return NS_MAX(scale, 1.0); } diff --git a/layout/base/FrameLayerBuilder.cpp b/layout/base/FrameLayerBuilder.cpp index e0bf5424062..e6daff4aa6f 100644 --- a/layout/base/FrameLayerBuilder.cpp +++ b/layout/base/FrameLayerBuilder.cpp @@ -1810,7 +1810,7 @@ ChooseScaleAndSetTransform(FrameLayerBuilder* aLayerBuilder, // If the scale factors are too small, just use 1.0. The content is being // scaled out of sight anyway. if (fabs(scale.width) < 1e-8 || fabs(scale.height) < 1e-8) { - scale.width = scale.height = 1.0; + scale = gfxSize(1.0, 1.0); } } else { scale = gfxSize(1.0, 1.0); From 39eedd06d716a7c3451b6007904e90b880e15047 Mon Sep 17 00:00:00 2001 From: Masayuki Nakano Date: Mon, 30 Apr 2012 08:43:21 +0900 Subject: [PATCH 004/162] Bug 749500 Define new API for 10.5 SDK r=smichaud --- widget/cocoa/nsChildView.h | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/widget/cocoa/nsChildView.h b/widget/cocoa/nsChildView.h index 1837eca90f9..6905b2d73d6 100644 --- a/widget/cocoa/nsChildView.h +++ b/widget/cocoa/nsChildView.h @@ -170,6 +170,13 @@ extern "C" long TSMProcessRawKeyEvent(EventRef carbonEvent); - (long long)_scrollPhase; @end +#if !defined(MAC_OS_X_VERSION_10_6) || \ +MAC_OS_X_VERSION_MAX_ALLOWED < MAC_OS_X_VERSION_10_6 +@interface NSEvent (SnowLeopardEventFeatures) ++ (NSUInteger)pressedMouseButtons; +@end +#endif + // The following section, required to support fluid swipe tracking on OS X 10.7 // and up, contains defines/declarations that are only available on 10.7 and up. // [NSEvent trackSwipeEventWithOptions:...] also requires that the compiler From 6a48592e95615c7dbde7be255507ba0c354f3fcb Mon Sep 17 00:00:00 2001 From: Thomas Powell Date: Sun, 29 Apr 2012 22:37:04 -0400 Subject: [PATCH 005/162] Bug 742736 - Change box shadows to match rounded or sharp corners of shadowed object when round/sharp alternate corners; r=dbaron --- layout/base/nsCSSRendering.cpp | 28 ++++------------ layout/base/nsCSSRenderingBorders.cpp | 32 +++++++++++++++++++ layout/base/nsCSSRenderingBorders.h | 8 +++++ .../boxshadow-threecorners-ref.html | 3 ++ .../box-shadow/boxshadow-threecorners.html | 2 ++ .../box-shadow/boxshadow-twocorners-ref.html | 3 ++ .../box-shadow/boxshadow-twocorners.html | 2 ++ layout/reftests/box-shadow/reftest.list | 2 ++ 8 files changed, 59 insertions(+), 21 deletions(-) create mode 100644 layout/reftests/box-shadow/boxshadow-threecorners-ref.html create mode 100644 layout/reftests/box-shadow/boxshadow-threecorners.html create mode 100644 layout/reftests/box-shadow/boxshadow-twocorners-ref.html create mode 100644 layout/reftests/box-shadow/boxshadow-twocorners.html diff --git a/layout/base/nsCSSRendering.cpp b/layout/base/nsCSSRendering.cpp index 95ccce192a8..9555564c4fe 100644 --- a/layout/base/nsCSSRendering.cpp +++ b/layout/base/nsCSSRendering.cpp @@ -1247,30 +1247,16 @@ nsCSSRendering::PaintBoxShadowOuter(nsPresContext* aPresContext, shadowContext->NewPath(); if (hasBorderRadius) { gfxCornerSizes clipRectRadii; - gfxFloat spreadDistance = -shadowItem->mSpread / twipsPerPixel; - gfxFloat borderSizes[4] = { 0, 0, 0, 0 }; + gfxFloat spreadDistance = shadowItem->mSpread / twipsPerPixel; - // We only give the spread radius to corners with a radius on them, otherwise we'll - // give a rounded shadow corner to a frame corner with 0 border radius, should - // the author use non-uniform border radii sizes (border-top-left-radius etc) - // (bug 514670) - if (borderRadii[C_TL].width > 0 || borderRadii[C_BL].width > 0) { - borderSizes[NS_SIDE_LEFT] = spreadDistance; - } + gfxFloat borderSizes[4]; - if (borderRadii[C_TL].height > 0 || borderRadii[C_TR].height > 0) { - borderSizes[NS_SIDE_TOP] = spreadDistance; - } + borderSizes[NS_SIDE_LEFT] = spreadDistance; + borderSizes[NS_SIDE_TOP] = spreadDistance; + borderSizes[NS_SIDE_RIGHT] = spreadDistance; + borderSizes[NS_SIDE_BOTTOM] = spreadDistance; - if (borderRadii[C_TR].width > 0 || borderRadii[C_BR].width > 0) { - borderSizes[NS_SIDE_RIGHT] = spreadDistance; - } - - if (borderRadii[C_BL].height > 0 || borderRadii[C_BR].height > 0) { - borderSizes[NS_SIDE_BOTTOM] = spreadDistance; - } - - nsCSSBorderRenderer::ComputeInnerRadii(borderRadii, borderSizes, + nsCSSBorderRenderer::ComputeOuterRadii(borderRadii, borderSizes, &clipRectRadii); shadowContext->RoundedRectangle(shadowGfxRect, clipRectRadii); } else { diff --git a/layout/base/nsCSSRenderingBorders.cpp b/layout/base/nsCSSRenderingBorders.cpp index 1470ef73c7b..b266b2b52dd 100644 --- a/layout/base/nsCSSRenderingBorders.cpp +++ b/layout/base/nsCSSRenderingBorders.cpp @@ -221,6 +221,38 @@ nsCSSBorderRenderer::ComputeInnerRadii(const gfxCornerSizes& aRadii, iRadii[C_BL].height = NS_MAX(0.0, aRadii[C_BL].height - aBorderSizes[NS_SIDE_BOTTOM]); } +/* static */ void +nsCSSBorderRenderer::ComputeOuterRadii(const gfxCornerSizes& aRadii, + const gfxFloat *aBorderSizes, + gfxCornerSizes *aOuterRadiiRet) +{ + gfxCornerSizes& oRadii = *aOuterRadiiRet; + + // default all corners to sharp corners + oRadii = gfxCornerSizes(0.0); + + // round the edges that have radii > 0.0 to start with + if (aRadii[C_TL].width > 0.0 && aRadii[C_TL].height > 0.0) { + oRadii[C_TL].width = NS_MAX(0.0, aRadii[C_TL].width + aBorderSizes[NS_SIDE_LEFT]); + oRadii[C_TL].height = NS_MAX(0.0, aRadii[C_TL].height + aBorderSizes[NS_SIDE_TOP]); + } + + if (aRadii[C_TR].width > 0.0 && aRadii[C_TR].height > 0.0) { + oRadii[C_TR].width = NS_MAX(0.0, aRadii[C_TR].width + aBorderSizes[NS_SIDE_RIGHT]); + oRadii[C_TR].height = NS_MAX(0.0, aRadii[C_TR].height + aBorderSizes[NS_SIDE_TOP]); + } + + if (aRadii[C_BR].width > 0.0 && aRadii[C_BR].height > 0.0) { + oRadii[C_BR].width = NS_MAX(0.0, aRadii[C_BR].width + aBorderSizes[NS_SIDE_RIGHT]); + oRadii[C_BR].height = NS_MAX(0.0, aRadii[C_BR].height + aBorderSizes[NS_SIDE_BOTTOM]); + } + + if (aRadii[C_BL].width > 0.0 && aRadii[C_BL].height > 0.0) { + oRadii[C_BL].width = NS_MAX(0.0, aRadii[C_BL].width + aBorderSizes[NS_SIDE_LEFT]); + oRadii[C_BL].height = NS_MAX(0.0, aRadii[C_BL].height + aBorderSizes[NS_SIDE_BOTTOM]); + } +} + /*static*/ void ComputeBorderCornerDimensions(const gfxRect& aOuterRect, const gfxRect& aInnerRect, diff --git a/layout/base/nsCSSRenderingBorders.h b/layout/base/nsCSSRenderingBorders.h index 648a660dfcf..f35a3a7820e 100644 --- a/layout/base/nsCSSRenderingBorders.h +++ b/layout/base/nsCSSRenderingBorders.h @@ -236,6 +236,14 @@ struct nsCSSBorderRenderer { static void ComputeInnerRadii(const gfxCornerSizes& aRadii, const gfxFloat *aBorderSizes, gfxCornerSizes *aInnerRadiiRet); + + // Given aRadii as the border radii for a rectangle, compute the + // appropriate radii for another rectangle *outside* that rectangle + // by increasing the radii, except keeping sharp corners sharp. + // Used for spread box-shadows + static void ComputeOuterRadii(const gfxCornerSizes& aRadii, + const gfxFloat *aBorderSizes, + gfxCornerSizes *aOuterRadiiRet); }; #ifdef DEBUG_NEW_BORDERS diff --git a/layout/reftests/box-shadow/boxshadow-threecorners-ref.html b/layout/reftests/box-shadow/boxshadow-threecorners-ref.html new file mode 100644 index 00000000000..cd9f383e1f8 --- /dev/null +++ b/layout/reftests/box-shadow/boxshadow-threecorners-ref.html @@ -0,0 +1,3 @@ + +
 
+
Test
diff --git a/layout/reftests/box-shadow/boxshadow-threecorners.html b/layout/reftests/box-shadow/boxshadow-threecorners.html new file mode 100644 index 00000000000..f00135f9597 --- /dev/null +++ b/layout/reftests/box-shadow/boxshadow-threecorners.html @@ -0,0 +1,2 @@ + +
Test
diff --git a/layout/reftests/box-shadow/boxshadow-twocorners-ref.html b/layout/reftests/box-shadow/boxshadow-twocorners-ref.html new file mode 100644 index 00000000000..bc3263cdb17 --- /dev/null +++ b/layout/reftests/box-shadow/boxshadow-twocorners-ref.html @@ -0,0 +1,3 @@ + +
 
+
Test
diff --git a/layout/reftests/box-shadow/boxshadow-twocorners.html b/layout/reftests/box-shadow/boxshadow-twocorners.html new file mode 100644 index 00000000000..e62c316f2b6 --- /dev/null +++ b/layout/reftests/box-shadow/boxshadow-twocorners.html @@ -0,0 +1,2 @@ + +
Test
diff --git a/layout/reftests/box-shadow/reftest.list b/layout/reftests/box-shadow/reftest.list index abe0be23547..5289bf29cd5 100644 --- a/layout/reftests/box-shadow/reftest.list +++ b/layout/reftests/box-shadow/reftest.list @@ -16,6 +16,8 @@ random-if(layersGPUAccelerated) == boxshadow-mixed.html boxshadow-mixed-ref.html random-if(d2d) == boxshadow-rounded-spread.html boxshadow-rounded-spread-ref.html HTTP(..) == boxshadow-dynamic.xul boxshadow-dynamic-ref.xul random-if(d2d) == boxshadow-onecorner.html boxshadow-onecorner-ref.html +random-if(d2d) == boxshadow-twocorners.html boxshadow-twocorners-ref.html +random-if(d2d) == boxshadow-threecorners.html boxshadow-threecorners-ref.html == boxshadow-skiprect.html boxshadow-skiprect-ref.html == overflow-not-scrollable-1.html overflow-not-scrollable-1-ref.html From 79f3c0fd8a0bbd8bc08dd7cbd40cd23ad5b061be Mon Sep 17 00:00:00 2001 From: Robert O'Callahan Date: Mon, 30 Apr 2012 15:11:00 +1200 Subject: [PATCH 006/162] Bug 664918. Part 0: Refactor nsMediaCache's handling of principals into a helper method in nsContentUtils. r=bzbarsky --- content/base/public/nsContentUtils.h | 17 +++++++++++++++ content/base/src/nsContentUtils.cpp | 26 ++++++++++++++++++++++- content/media/nsMediaCache.cpp | 31 ++-------------------------- content/media/nsMediaCache.h | 7 ++----- 4 files changed, 46 insertions(+), 35 deletions(-) diff --git a/content/base/public/nsContentUtils.h b/content/base/public/nsContentUtils.h index 36710c369e0..d6d8c41019b 100644 --- a/content/base/public/nsContentUtils.h +++ b/content/base/public/nsContentUtils.h @@ -1372,6 +1372,23 @@ public: */ static bool IsSystemPrincipal(nsIPrincipal* aPrincipal); + /** + * *aResourcePrincipal is a principal describing who may access the contents + * of a resource. The resource can only be consumed by a principal that + * subsumes *aResourcePrincipal. MAKE SURE THAT NOTHING EVER ACTS WITH THE + * AUTHORITY OF *aResourcePrincipal. + * It may be null to indicate that the resource has no data from any origin + * in it yet and anything may access the resource. + * Additional data is being mixed into the resource from aExtraPrincipal + * (which may be null; if null, no data is being mixed in and this function + * will do nothing). Update *aResourcePrincipal to reflect the new data. + * If *aResourcePrincipal subsumes aExtraPrincipal, nothing needs to change, + * otherwise *aResourcePrincipal is replaced with the system principal. + * Returns true if *aResourcePrincipal changed. + */ + static bool CombineResourcePrincipals(nsCOMPtr* aResourcePrincipal, + nsIPrincipal* aExtraPrincipal); + /** * Trigger a link with uri aLinkURI. If aClick is false, this triggers a * mouseover on the link, otherwise it triggers a load after doing a diff --git a/content/base/src/nsContentUtils.cpp b/content/base/src/nsContentUtils.cpp index a926cf7b4da..c6180205b0c 100644 --- a/content/base/src/nsContentUtils.cpp +++ b/content/base/src/nsContentUtils.cpp @@ -147,6 +147,7 @@ static NS_DEFINE_CID(kXTFServiceCID, NS_XTFSERVICE_CID); #include "nsIPermissionManager.h" #include "nsIContentPrefService.h" #include "nsIScriptObjectPrincipal.h" +#include "nsNullPrincipal.h" #include "nsIRunnable.h" #include "nsDOMJSUtils.h" #include "nsGenericHTMLElement.h" @@ -4120,7 +4121,7 @@ nsContentUtils::ConvertToPlainText(const nsAString& aSourceBuffer, nsCOMPtr uri; NS_NewURI(getter_AddRefs(uri), "about:blank"); nsCOMPtr principal = - do_CreateInstance("@mozilla.org/nullprincipal;1"); + do_CreateInstance(NS_NULLPRINCIPAL_CONTRACTID); nsCOMPtr domDocument; nsresult rv = nsContentUtils::CreateDocument(EmptyString(), EmptyString(), @@ -4466,6 +4467,29 @@ nsContentUtils::IsSystemPrincipal(nsIPrincipal* aPrincipal) return NS_SUCCEEDED(rv) && isSystem; } +bool +nsContentUtils::CombineResourcePrincipals(nsCOMPtr* aResourcePrincipal, + nsIPrincipal* aExtraPrincipal) +{ + if (!aExtraPrincipal) { + return false; + } + if (!*aResourcePrincipal) { + *aResourcePrincipal = aExtraPrincipal; + return true; + } + if (*aResourcePrincipal == aExtraPrincipal) { + return false; + } + bool subsumes; + if (NS_SUCCEEDED((*aResourcePrincipal)->Subsumes(aExtraPrincipal, &subsumes)) && + subsumes) { + return false; + } + sSecurityManager->GetSystemPrincipal(getter_AddRefs(*aResourcePrincipal)); + return true; +} + /* static */ void nsContentUtils::TriggerLink(nsIContent *aContent, nsPresContext *aPresContext, diff --git a/content/media/nsMediaCache.cpp b/content/media/nsMediaCache.cpp index f4a263d624e..0ee5f98dca8 100644 --- a/content/media/nsMediaCache.cpp +++ b/content/media/nsMediaCache.cpp @@ -45,6 +45,7 @@ #include "nsXULAppAPI.h" #include "nsNetUtil.h" #include "prio.h" +#include "nsContentUtils.h" #include "nsThreadUtils.h" #include "MediaResource.h" #include "nsMathUtils.h" @@ -1705,35 +1706,7 @@ nsMediaCacheStream::NotifyDataStarted(PRInt64 aOffset) void nsMediaCacheStream::UpdatePrincipal(nsIPrincipal* aPrincipal) { - if (!mPrincipal) { - NS_ASSERTION(!mUsingNullPrincipal, "Are we using a null principal or not?"); - if (mUsingNullPrincipal) { - // Don't let mPrincipal be set to anything - return; - } - mPrincipal = aPrincipal; - return; - } - - if (mPrincipal == aPrincipal) { - // Common case - NS_ASSERTION(!mUsingNullPrincipal, "We can't receive data from a null principal"); - return; - } - if (mUsingNullPrincipal) { - // We've already fallen back to a null principal, so nothing more - // to do. - return; - } - - bool equal; - nsresult rv = mPrincipal->Equals(aPrincipal, &equal); - if (NS_SUCCEEDED(rv) && equal) - return; - - // Principals are not equal, so set mPrincipal to a null principal. - mPrincipal = do_CreateInstance("@mozilla.org/nullprincipal;1"); - mUsingNullPrincipal = true; + nsContentUtils::CombineResourcePrincipals(&mPrincipal, aPrincipal); } void diff --git a/content/media/nsMediaCache.h b/content/media/nsMediaCache.h index 045b5a7820d..fe0a17b700f 100644 --- a/content/media/nsMediaCache.h +++ b/content/media/nsMediaCache.h @@ -236,7 +236,6 @@ public: mDidNotifyDataEnded(false), mIsSeekable(false), mCacheSuspended(false), mChannelEnded(false), - mUsingNullPrincipal(false), mChannelOffset(0), mStreamLength(-1), mStreamOffset(0), mPlaybackBytesPerSecond(10000), mPinCount(0), mCurrentMode(MODE_PLAYBACK), @@ -273,7 +272,8 @@ public: return !mClosed && (!mDidNotifyDataEnded || NS_SUCCEEDED(mNotifyDataEndedStatus)); } - // Get the principal for this stream. + // Get the principal for this stream. Anything accessing the contents of + // this stream must have a principal that subsumes this principal. nsIPrincipal* GetCurrentPrincipal() { return mPrincipal; } // Ensure a global media cache update has run with this stream present. // This ensures the cache has had a chance to suspend or unsuspend this stream. @@ -490,9 +490,6 @@ private: bool mCacheSuspended; // True if the channel ended and we haven't seeked it again. bool mChannelEnded; - // True if mPrincipal is a null principal because we saw data from - // multiple origins - bool mUsingNullPrincipal; // The offset where the next data from the channel will arrive PRInt64 mChannelOffset; // The reported or discovered length of the data, or -1 if nothing is From c0f9814cc011ef723022bc865572f244dcc9f116 Mon Sep 17 00:00:00 2001 From: Robert O'Callahan Date: Mon, 30 Apr 2012 15:11:08 +1200 Subject: [PATCH 007/162] Bug 664918. Part 1: Create TimeVarying class to represent values that change over time. r=jesup --- content/media/Makefile.in | 13 ++- content/media/TimeVarying.h | 223 ++++++++++++++++++++++++++++++++++++ 2 files changed, 230 insertions(+), 6 deletions(-) create mode 100644 content/media/TimeVarying.h diff --git a/content/media/Makefile.in b/content/media/Makefile.in index e670698df2d..c01f3736484 100644 --- a/content/media/Makefile.in +++ b/content/media/Makefile.in @@ -47,26 +47,27 @@ LIBXUL_LIBRARY = 1 EXPORTS = \ FileBlockCache.h \ + MediaResource.h \ nsAudioAvailableEventManager.h \ - nsMediaDecoder.h \ - nsMediaCache.h \ nsBuiltinDecoder.h \ nsBuiltinDecoderStateMachine.h \ nsBuiltinDecoderReader.h \ - MediaResource.h \ + nsMediaCache.h \ + nsMediaDecoder.h \ + TimeVarying.h \ VideoFrameContainer.h \ VideoUtils.h \ $(NULL) CPPSRCS = \ FileBlockCache.cpp \ + MediaResource.cpp \ nsAudioAvailableEventManager.cpp \ - nsMediaDecoder.cpp \ - nsMediaCache.cpp \ nsBuiltinDecoder.cpp \ nsBuiltinDecoderStateMachine.cpp \ nsBuiltinDecoderReader.cpp \ - MediaResource.cpp \ + nsMediaCache.cpp \ + nsMediaDecoder.cpp \ VideoFrameContainer.cpp \ VideoUtils.cpp \ $(NULL) diff --git a/content/media/TimeVarying.h b/content/media/TimeVarying.h new file mode 100644 index 00000000000..3cce63a023f --- /dev/null +++ b/content/media/TimeVarying.h @@ -0,0 +1,223 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*-*/ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef MOZILLA_TIMEVARYING_H_ +#define MOZILLA_TIMEVARYING_H_ + +#include "nsTArray.h" + +namespace mozilla { + +/** + * This is just for CTOR/DTOR tracking. We can't put this in + * TimeVarying directly because the different template instances have + * different sizes and that confuses things. + */ +class TimeVaryingBase { +protected: + TimeVaryingBase() + { + MOZ_COUNT_CTOR(TimeVaryingBase); + } + ~TimeVaryingBase() + { + MOZ_COUNT_DTOR(TimeVaryingBase); + } +}; + +/** + * Objects of this class represent values that can change over time --- + * a mathematical function of time. + * Time is the type of time values, T is the value that changes over time. + * There are a finite set of "change times"; at each change time, the function + * instantly changes to a new value. + * There is also a "current time" which must always advance (not go backward). + * The function is constant for all times less than the current time. + * When the current time is advanced, the value of the function at the new + * current time replaces the values for all previous times. + * + * The implementation records a mCurrent (the value at the current time) + * and an array of "change times" (greater than the current time) and the + * new value for each change time. This is a simple but dumb implementation. + */ +template +class TimeVarying : public TimeVaryingBase { +public: + TimeVarying(const T& aInitial) : mCurrent(aInitial) {} + /** + * This constructor can only be called if mCurrent has a no-argument + * constructor. + */ + TimeVarying() : mCurrent() {} + + /** + * Sets the value for all times >= aTime to aValue. + */ + void SetAtAndAfter(Time aTime, const T& aValue) + { + for (PRInt32 i = mChanges.Length() - 1; i >= 0; --i) { + NS_ASSERTION(i == PRInt32(mChanges.Length() - 1), + "Always considering last element of array"); + if (aTime > mChanges[i].mTime) { + if (mChanges[i].mValue != aValue) { + mChanges.AppendElement(Entry(aTime, aValue)); + } + return; + } + if (aTime == mChanges[i].mTime) { + if ((i > 0 ? mChanges[i - 1].mValue : mCurrent) == aValue) { + mChanges.RemoveElementAt(i); + return; + } + mChanges[i].mValue = aValue; + return; + } + mChanges.RemoveElementAt(i); + } + mChanges.InsertElementAt(0, Entry(aTime, aValue)); + } + /** + * Returns the final value of the function. If aTime is non-null, + * sets aTime to the time at which the function changes to that final value. + * If there are no changes after the current time, returns PR_INT64_MIN in aTime. + */ + const T& GetLast(Time* aTime = nsnull) const + { + if (mChanges.IsEmpty()) { + if (aTime) { + *aTime = PR_INT64_MIN; + } + return mCurrent; + } + if (aTime) { + *aTime = mChanges[mChanges.Length() - 1].mTime; + } + return mChanges[mChanges.Length() - 1].mValue; + } + /** + * Returns the value of the function just before time aTime. + */ + const T& GetBefore(Time aTime) const + { + if (mChanges.IsEmpty() || aTime <= mChanges[0].mTime) { + return mCurrent; + } + PRInt32 changesLength = mChanges.Length(); + if (mChanges[changesLength - 1].mTime < aTime) { + return mChanges[changesLength - 1].mValue; + } + for (PRUint32 i = 1; ; ++i) { + if (aTime <= mChanges[i].mTime) { + NS_ASSERTION(mChanges[i].mValue != mChanges[i - 1].mValue, + "Only changed values appear in array"); + return mChanges[i - 1].mValue; + } + } + } + /** + * Returns the value of the function at time aTime. + * If aEnd is non-null, sets *aEnd to the time at which the function will + * change from the returned value to a new value, or PR_INT64_MAX if that + * never happens. + * If aStart is non-null, sets *aStart to the time at which the function + * changed to the returned value, or PR_INT64_MIN if that happened at or + * before the current time. + * + * Currently uses a linear search, but could use a binary search. + */ + const T& GetAt(Time aTime, Time* aEnd = nsnull, Time* aStart = nsnull) const + { + if (mChanges.IsEmpty() || aTime < mChanges[0].mTime) { + if (aStart) { + *aStart = PR_INT64_MIN; + } + if (aEnd) { + *aEnd = mChanges.IsEmpty() ? PR_INT64_MAX : mChanges[0].mTime; + } + return mCurrent; + } + PRInt32 changesLength = mChanges.Length(); + if (mChanges[changesLength - 1].mTime <= aTime) { + if (aEnd) { + *aEnd = PR_INT64_MAX; + } + if (aStart) { + *aStart = mChanges[changesLength - 1].mTime; + } + return mChanges[changesLength - 1].mValue; + } + + for (PRUint32 i = 1; ; ++i) { + if (aTime < mChanges[i].mTime) { + if (aEnd) { + *aEnd = mChanges[i].mTime; + } + if (aStart) { + *aStart = mChanges[i - 1].mTime; + } + NS_ASSERTION(mChanges[i].mValue != mChanges[i - 1].mValue, + "Only changed values appear in array"); + return mChanges[i - 1].mValue; + } + } + } + /** + * Advance the current time to aTime. + */ + void AdvanceCurrentTime(Time aTime) + { + for (PRUint32 i = 0; i < mChanges.Length(); ++i) { + if (aTime < mChanges[i].mTime) { + mChanges.RemoveElementsAt(0, i); + return; + } + mCurrent = mChanges[i].mValue; + } + mChanges.Clear(); + } + /** + * Make all currently pending changes happen aDelta later than their + * current change times. + */ + void InsertTimeAtStart(Time aDelta) + { + for (PRUint32 i = 0; i < mChanges.Length(); ++i) { + mChanges[i].mTime += aDelta; + } + } + + /** + * Replace the values of this function at aTimeOffset and later with the + * values of aOther taken from zero, so if aOther is V at time T >= 0 + * then this function will be V at time T + aTimeOffset. aOther's current + * time must be >= 0. + */ + void Append(const TimeVarying& aOther, Time aTimeOffset) + { + NS_ASSERTION(aOther.mChanges.IsEmpty() || aOther.mChanges[0].mTime >= 0, + "Negative time not allowed here"); + NS_ASSERTION(&aOther != this, "Can't self-append"); + SetAtAndAfter(aTimeOffset, aOther.mCurrent); + for (PRUint32 i = 0; i < aOther.mChanges.Length(); ++i) { + const Entry& e = aOther.mChanges[i]; + SetAtAndAfter(aTimeOffset + e.mTime, e.mValue); + } + } + +private: + struct Entry { + Entry(Time aTime, const T& aValue) : mTime(aTime), mValue(aValue) {} + + // The time at which the value changes to mValue + Time mTime; + T mValue; + }; + nsTArray mChanges; + T mCurrent; +}; + +} + +#endif /* MOZILLA_TIMEVARYING_H_ */ From cbe41ae9066faa2c122d27aaf9568657868cd6c7 Mon Sep 17 00:00:00 2001 From: Robert O'Callahan Date: Mon, 30 Apr 2012 15:11:19 +1200 Subject: [PATCH 008/162] Bug 664918. Part 2: Create MediaSegment, AudioSegment and VideoSegment classes to manage intervals of media data. r=jesup Also introduces a SharedBuffer class, representing a blob of binary data with threadsafe refcounting. --- content/media/AudioSegment.cpp | 193 ++++++++++++++++++++++ content/media/AudioSegment.h | 151 +++++++++++++++++ content/media/Makefile.in | 7 + content/media/MediaSegment.h | 270 +++++++++++++++++++++++++++++++ content/media/SharedBuffer.h | 45 ++++++ content/media/StreamBuffer.cpp | 60 +++++++ content/media/StreamBuffer.h | 286 +++++++++++++++++++++++++++++++++ content/media/VideoSegment.h | 117 ++++++++++++++ 8 files changed, 1129 insertions(+) create mode 100644 content/media/AudioSegment.cpp create mode 100644 content/media/AudioSegment.h create mode 100644 content/media/MediaSegment.h create mode 100644 content/media/SharedBuffer.h create mode 100644 content/media/StreamBuffer.cpp create mode 100644 content/media/StreamBuffer.h create mode 100644 content/media/VideoSegment.h diff --git a/content/media/AudioSegment.cpp b/content/media/AudioSegment.cpp new file mode 100644 index 00000000000..242c2d83822 --- /dev/null +++ b/content/media/AudioSegment.cpp @@ -0,0 +1,193 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "AudioSegment.h" + +namespace mozilla { + +static PRUint16 +FlipByteOrderIfBigEndian(PRUint16 aValue) +{ + PRUint16 s = aValue; +#if defined(IS_BIG_ENDIAN) + s = (s << 8) | (s >> 8); +#endif + return s; +} + +/* + * Use "2^N" conversion since it's simple, fast, "bit transparent", used by + * many other libraries and apparently behaves reasonably. + * http://blog.bjornroche.com/2009/12/int-float-int-its-jungle-out-there.html + * http://blog.bjornroche.com/2009/12/linearity-and-dynamic-range-in-int.html + */ +static float +SampleToFloat(float aValue) +{ + return aValue; +} +static float +SampleToFloat(PRUint8 aValue) +{ + return (aValue - 128)/128.0f; +} +static float +SampleToFloat(PRInt16 aValue) +{ + return PRInt16(FlipByteOrderIfBigEndian(aValue))/32768.0f; +} + +static void +FloatToSample(float aValue, float* aOut) +{ + *aOut = aValue; +} +static void +FloatToSample(float aValue, PRUint8* aOut) +{ + float v = aValue*128 + 128; + float clamped = NS_MAX(0.0f, NS_MIN(255.0f, v)); + *aOut = PRUint8(clamped); +} +static void +FloatToSample(float aValue, PRInt16* aOut) +{ + float v = aValue*32768.0f; + float clamped = NS_MAX(-32768.0f, NS_MIN(32767.0f, v)); + *aOut = PRInt16(FlipByteOrderIfBigEndian(PRInt16(clamped))); +} + +template +static void +InterleaveAndConvertBuffer(const SrcT* aSource, PRInt32 aSourceLength, + PRInt32 aLength, + float aVolume, + PRInt32 aChannels, + DestT* aOutput) +{ + DestT* output = aOutput; + for (PRInt32 i = 0; i < aLength; ++i) { + for (PRInt32 channel = 0; channel < aChannels; ++channel) { + float v = SampleToFloat(aSource[channel*aSourceLength + i])*aVolume; + FloatToSample(v, output); + ++output; + } + } +} + +static void +InterleaveAndConvertBuffer(const PRInt16* aSource, PRInt32 aSourceLength, + PRInt32 aLength, + float aVolume, + PRInt32 aChannels, + PRInt16* aOutput) +{ + PRInt16* output = aOutput; + float v = NS_MAX(NS_MIN(aVolume, 1.0f), -1.0f); + PRInt32 volume = PRInt32((1 << 16) * v); + for (PRInt32 i = 0; i < aLength; ++i) { + for (PRInt32 channel = 0; channel < aChannels; ++channel) { + PRInt16 s = FlipByteOrderIfBigEndian(aSource[channel*aSourceLength + i]); + *output = FlipByteOrderIfBigEndian(PRInt16((PRInt32(s) * volume) >> 16)); + ++output; + } + } +} + +template +static void +InterleaveAndConvertBuffer(const SrcT* aSource, PRInt32 aSourceLength, + PRInt32 aLength, + float aVolume, + PRInt32 aChannels, + void* aOutput, nsAudioStream::SampleFormat aOutputFormat) +{ + switch (aOutputFormat) { + case nsAudioStream::FORMAT_FLOAT32: + InterleaveAndConvertBuffer(aSource, aSourceLength, aLength, aVolume, + aChannels, static_cast(aOutput)); + break; + case nsAudioStream::FORMAT_S16_LE: + InterleaveAndConvertBuffer(aSource, aSourceLength, aLength, aVolume, + aChannels, static_cast(aOutput)); + break; + case nsAudioStream::FORMAT_U8: + InterleaveAndConvertBuffer(aSource, aSourceLength, aLength, aVolume, + aChannels, static_cast(aOutput)); + break; + } +} + +static void +InterleaveAndConvertBuffer(const void* aSource, nsAudioStream::SampleFormat aSourceFormat, + PRInt32 aSourceLength, + PRInt32 aOffset, PRInt32 aLength, + float aVolume, + PRInt32 aChannels, + void* aOutput, nsAudioStream::SampleFormat aOutputFormat) +{ + switch (aSourceFormat) { + case nsAudioStream::FORMAT_FLOAT32: + InterleaveAndConvertBuffer(static_cast(aSource) + aOffset, aSourceLength, + aLength, + aVolume, + aChannels, + aOutput, aOutputFormat); + break; + case nsAudioStream::FORMAT_S16_LE: + InterleaveAndConvertBuffer(static_cast(aSource) + aOffset, aSourceLength, + aLength, + aVolume, + aChannels, + aOutput, aOutputFormat); + break; + case nsAudioStream::FORMAT_U8: + InterleaveAndConvertBuffer(static_cast(aSource) + aOffset, aSourceLength, + aLength, + aVolume, + aChannels, + aOutput, aOutputFormat); + break; + } +} + +void +AudioSegment::ApplyVolume(float aVolume) +{ + for (ChunkIterator ci(*this); !ci.IsEnded(); ci.Next()) { + ci->mVolume *= aVolume; + } +} + +static const int STATIC_AUDIO_BUFFER_BYTES = 50000; + +void +AudioSegment::WriteTo(nsAudioStream* aOutput) +{ + NS_ASSERTION(mChannels == aOutput->GetChannels(), "Wrong number of channels"); + nsAutoTArray buf; + PRUint32 frameSize = GetSampleSize(aOutput->GetFormat())*mChannels; + for (ChunkIterator ci(*this); !ci.IsEnded(); ci.Next()) { + AudioChunk& c = *ci; + if (frameSize*c.mDuration > PR_UINT32_MAX) { + NS_ERROR("Buffer overflow"); + return; + } + buf.SetLength(PRInt32(frameSize*c.mDuration)); + if (c.mBuffer) { + InterleaveAndConvertBuffer(c.mBuffer->Data(), c.mBufferFormat, c.mBufferLength, + c.mOffset, PRInt32(c.mDuration), + c.mVolume, + aOutput->GetChannels(), + buf.Elements(), aOutput->GetFormat()); + } else { + // Assumes that a bit pattern of zeroes == 0.0f + memset(buf.Elements(), 0, buf.Length()); + } + aOutput->Write(buf.Elements(), PRInt32(c.mDuration)); + } +} + +} diff --git a/content/media/AudioSegment.h b/content/media/AudioSegment.h new file mode 100644 index 00000000000..396fabb452f --- /dev/null +++ b/content/media/AudioSegment.h @@ -0,0 +1,151 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef MOZILLA_AUDIOSEGMENT_H_ +#define MOZILLA_AUDIOSEGMENT_H_ + +#include "MediaSegment.h" +#include "nsISupportsImpl.h" +#include "nsAudioStream.h" +#include "SharedBuffer.h" + +namespace mozilla { + +struct AudioChunk { + typedef nsAudioStream::SampleFormat SampleFormat; + + // Generic methods + void SliceTo(TrackTicks aStart, TrackTicks aEnd) + { + NS_ASSERTION(aStart >= 0 && aStart < aEnd && aEnd <= mDuration, + "Slice out of bounds"); + if (mBuffer) { + mOffset += PRInt32(aStart); + } + mDuration = aEnd - aStart; + } + TrackTicks GetDuration() const { return mDuration; } + bool CanCombineWithFollowing(const AudioChunk& aOther) const + { + if (aOther.mBuffer != mBuffer) { + return false; + } + if (mBuffer) { + NS_ASSERTION(aOther.mBufferFormat == mBufferFormat && aOther.mBufferLength == mBufferLength, + "Wrong metadata about buffer"); + return aOther.mOffset == mOffset + mDuration && aOther.mVolume == mVolume; + } + return true; + } + bool IsNull() const { return mBuffer == nsnull; } + void SetNull(TrackTicks aDuration) + { + mBuffer = nsnull; + mDuration = aDuration; + mOffset = 0; + mVolume = 1.0f; + } + + TrackTicks mDuration; // in frames within the buffer + nsRefPtr mBuffer; // null means data is all zeroes + PRInt32 mBufferLength; // number of frames in mBuffer (only meaningful if mBuffer is nonnull) + SampleFormat mBufferFormat; // format of frames in mBuffer (only meaningful if mBuffer is nonnull) + PRInt32 mOffset; // in frames within the buffer (zero if mBuffer is null) + float mVolume; // volume multiplier to apply (1.0f if mBuffer is nonnull) +}; + +/** + * A list of audio samples consisting of a sequence of slices of SharedBuffers. + * The audio rate is determined by the track, not stored in this class. + */ +class AudioSegment : public MediaSegmentBase { +public: + typedef nsAudioStream::SampleFormat SampleFormat; + + static int GetSampleSize(SampleFormat aFormat) + { + switch (aFormat) { + case nsAudioStream::FORMAT_U8: return 1; + case nsAudioStream::FORMAT_S16_LE: return 2; + case nsAudioStream::FORMAT_FLOAT32: return 4; + } + NS_ERROR("Bad format"); + return 0; + } + + AudioSegment() : MediaSegmentBase(AUDIO), mChannels(0) {} + + bool IsInitialized() + { + return mChannels > 0; + } + void Init(PRInt32 aChannels) + { + NS_ASSERTION(aChannels > 0, "Bad number of channels"); + NS_ASSERTION(!IsInitialized(), "Already initialized"); + mChannels = aChannels; + } + PRInt32 GetChannels() + { + NS_ASSERTION(IsInitialized(), "Not initialized"); + return mChannels; + } + /** + * Returns the format of the first audio frame that has data, or + * FORMAT_FLOAT32 if there is none. + */ + SampleFormat GetFirstFrameFormat() + { + for (ChunkIterator ci(*this); !ci.IsEnded(); ci.Next()) { + if (ci->mBuffer) { + return ci->mBufferFormat; + } + } + return nsAudioStream::FORMAT_FLOAT32; + } + void AppendFrames(already_AddRefed aBuffer, PRInt32 aBufferLength, + PRInt32 aStart, PRInt32 aEnd, SampleFormat aFormat) + { + NS_ASSERTION(mChannels > 0, "Not initialized"); + AudioChunk* chunk = AppendChunk(aEnd - aStart); + chunk->mBuffer = aBuffer; + chunk->mBufferFormat = aFormat; + chunk->mBufferLength = aBufferLength; + chunk->mOffset = aStart; + chunk->mVolume = 1.0f; + } + void ApplyVolume(float aVolume); + /** + * aOutput must have a matching number of channels, but we will automatically + * convert sample formats. + */ + void WriteTo(nsAudioStream* aOutput); + + void AppendFrom(AudioSegment* aSource) + { + NS_ASSERTION(aSource->mChannels == mChannels, "Non-matching channels"); + MediaSegmentBase::AppendFrom(aSource); + } + + // Segment-generic methods not in MediaSegmentBase + void InitFrom(const AudioSegment& aOther) + { + NS_ASSERTION(mChannels == 0, "Channels already set"); + mChannels = aOther.mChannels; + } + void SliceFrom(const AudioSegment& aOther, TrackTicks aStart, TrackTicks aEnd) + { + InitFrom(aOther); + BaseSliceFrom(aOther, aStart, aEnd); + } + static Type StaticType() { return AUDIO; } + +protected: + PRInt32 mChannels; +}; + +} + +#endif /* MOZILLA_AUDIOSEGMENT_H_ */ diff --git a/content/media/Makefile.in b/content/media/Makefile.in index c01f3736484..a794d080400 100644 --- a/content/media/Makefile.in +++ b/content/media/Makefile.in @@ -46,20 +46,26 @@ LIBRARY_NAME = gkconmedia_s LIBXUL_LIBRARY = 1 EXPORTS = \ + AudioSegment.h \ FileBlockCache.h \ MediaResource.h \ + MediaSegment.h \ nsAudioAvailableEventManager.h \ nsBuiltinDecoder.h \ nsBuiltinDecoderStateMachine.h \ nsBuiltinDecoderReader.h \ nsMediaCache.h \ nsMediaDecoder.h \ + SharedBuffer.h \ + StreamBuffer.h \ TimeVarying.h \ VideoFrameContainer.h \ VideoUtils.h \ + VideoSegment.h \ $(NULL) CPPSRCS = \ + AudioSegment.cpp \ FileBlockCache.cpp \ MediaResource.cpp \ nsAudioAvailableEventManager.cpp \ @@ -68,6 +74,7 @@ CPPSRCS = \ nsBuiltinDecoderReader.cpp \ nsMediaCache.cpp \ nsMediaDecoder.cpp \ + StreamBuffer.cpp \ VideoFrameContainer.cpp \ VideoUtils.cpp \ $(NULL) diff --git a/content/media/MediaSegment.h b/content/media/MediaSegment.h new file mode 100644 index 00000000000..6ad098a789d --- /dev/null +++ b/content/media/MediaSegment.h @@ -0,0 +1,270 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef MOZILLA_MEDIASEGMENT_H_ +#define MOZILLA_MEDIASEGMENT_H_ + +#include "nsTArray.h" + +namespace mozilla { + +/** + * We represent media times in 64-bit fixed point. So 1 MediaTime is + * 1/(2^MEDIA_TIME_FRAC_BITS) seconds. + */ +typedef PRInt64 MediaTime; +const PRInt64 MEDIA_TIME_FRAC_BITS = 20; +const PRInt64 MEDIA_TIME_MAX = PR_INT64_MAX; + +inline MediaTime MillisecondsToMediaTime(PRInt32 aMS) +{ + return (MediaTime(aMS) << MEDIA_TIME_FRAC_BITS)/1000; +} + +inline MediaTime SecondsToMediaTime(double aS) +{ + NS_ASSERTION(aS <= (MEDIA_TIME_MAX >> MEDIA_TIME_FRAC_BITS), + "Out of range"); + return MediaTime(aS * (1 << MEDIA_TIME_FRAC_BITS)); +} + +inline double MediaTimeToSeconds(MediaTime aTime) +{ + return aTime*(1.0/(1 << MEDIA_TIME_FRAC_BITS)); +} + +/** + * A number of ticks at a rate determined by some underlying track (e.g. + * audio sample rate). We want to make sure that multiplying TrackTicks by + * 2^MEDIA_TIME_FRAC_BITS doesn't overflow, so we set its max accordingly. + */ +typedef PRInt64 TrackTicks; +const PRInt64 TRACK_TICKS_MAX = PR_INT64_MAX >> MEDIA_TIME_FRAC_BITS; + +/** + * A MediaSegment is a chunk of media data sequential in time. Different + * types of data have different subclasses of MediaSegment, all inheriting + * from MediaSegmentBase. + * All MediaSegment data is timed using TrackTicks. The actual tick rate + * is defined on a per-track basis. For some track types, this can be + * a fixed constant for all tracks of that type (e.g. 1MHz for video). + * + * Each media segment defines a concept of "null media data" (e.g. silence + * for audio or "no video frame" for video), which can be efficiently + * represented. This is used for padding. + */ +class MediaSegment { +public: + virtual ~MediaSegment() + { + MOZ_COUNT_DTOR(MediaSegment); + } + + enum Type { + AUDIO, + VIDEO, + TYPE_COUNT + }; + + /** + * Gets the total duration of the segment. + */ + TrackTicks GetDuration() { return mDuration; } + Type GetType() { return mType; } + + /** + * Create a MediaSegment of the same type. + */ + virtual MediaSegment* CreateEmptyClone() = 0; + /** + * Moves contents of aSource to the end of this segment. + */ + virtual void AppendFrom(MediaSegment* aSource) = 0; + /** + * Replace all contents up to aDuration with null data. + */ + virtual void ForgetUpTo(TrackTicks aDuration) = 0; + /** + * Insert aDuration of null data at the start of the segment. + */ + virtual void InsertNullDataAtStart(TrackTicks aDuration) = 0; + +protected: + MediaSegment(Type aType) : mDuration(0), mType(aType) + { + MOZ_COUNT_CTOR(MediaSegment); + } + + TrackTicks mDuration; // total of mDurations of all chunks + Type mType; +}; + +/** + * C is the implementation class subclassed from MediaSegmentBase. + * C must contain a Chunk class. + */ +template class MediaSegmentBase : public MediaSegment { +public: + virtual MediaSegment* CreateEmptyClone() + { + C* s = new C(); + s->InitFrom(*static_cast(this)); + return s; + } + + /** + * Appends the contents of aSource to this segment, clearing aSource. + */ + virtual void AppendFrom(MediaSegmentBase* aSource) + { + mDuration += aSource->mDuration; + aSource->mDuration = 0; + if (!mChunks.IsEmpty() && !aSource->mChunks.IsEmpty() && + mChunks[mChunks.Length() - 1].CanCombineWithFollowing(aSource->mChunks[0])) { + mChunks[mChunks.Length() - 1].mDuration += aSource->mChunks[0].mDuration; + aSource->mChunks.RemoveElementAt(0); + } + mChunks.MoveElementsFrom(aSource->mChunks); + } + void RemoveLeading(TrackTicks aDuration) + { + RemoveLeadingInternal(aDuration, 0); + } + virtual void AppendFrom(MediaSegment* aSource) + { + NS_ASSERTION(aSource->GetType() == C::StaticType(), "Wrong type"); + AppendFrom(static_cast(aSource)); + } + /** + * Replace the first aDuration ticks with null media data, because the data + * will not be required again. + */ + virtual void ForgetUpTo(TrackTicks aDuration) + { + if (mChunks.IsEmpty() || aDuration <= 0) { + return; + } + if (mChunks[0].IsNull()) { + TrackTicks extraToForget = NS_MIN(aDuration, mDuration) - mChunks[0].GetDuration(); + if (extraToForget > 0) { + RemoveLeadingInternal(extraToForget, 1); + mChunks[0].mDuration += extraToForget; + mDuration += extraToForget; + } + return; + } + RemoveLeading(aDuration); + mChunks.InsertElementAt(0)->SetNull(aDuration); + mDuration += aDuration; + } + virtual void InsertNullDataAtStart(TrackTicks aDuration) + { + if (aDuration <= 0) { + return; + } + if (!mChunks.IsEmpty() && mChunks[0].IsNull()) { + mChunks[0].mDuration += aDuration; + } else { + mChunks.InsertElementAt(0)->SetNull(aDuration); + } + mDuration += aDuration; + } + +protected: + MediaSegmentBase(Type aType) : MediaSegment(aType) {} + + void BaseSliceFrom(const MediaSegmentBase& aOther, + TrackTicks aStart, TrackTicks aEnd) + { + NS_ASSERTION(aStart >= 0 && aEnd <= aOther.mDuration, + "Slice out of range"); + TrackTicks offset = 0; + for (PRUint32 i = 0; i < aOther.mChunks.Length() && offset < aEnd; ++i) { + const Chunk& c = aOther.mChunks[i]; + TrackTicks start = NS_MAX(aStart, offset); + TrackTicks nextOffset = offset + c.GetDuration(); + TrackTicks end = NS_MIN(aEnd, nextOffset); + if (start < end) { + mChunks.AppendElement(c)->SliceTo(start - offset, end - offset); + } + offset = nextOffset; + } + } + + Chunk* AppendChunk(TrackTicks aDuration) + { + Chunk* c = mChunks.AppendElement(); + c->mDuration = aDuration; + mDuration += aDuration; + return c; + } + + Chunk* FindChunkContaining(TrackTicks aOffset, TrackTicks* aStart = nsnull) + { + if (aOffset < 0) { + return nsnull; + } + TrackTicks offset = 0; + for (PRUint32 i = 0; i < mChunks.Length(); ++i) { + Chunk& c = mChunks[i]; + TrackTicks nextOffset = offset + c.GetDuration(); + if (aOffset < nextOffset) { + if (aStart) { + *aStart = offset; + } + return &c; + } + offset = nextOffset; + } + return nsnull; + } + + Chunk* GetLastChunk() + { + if (mChunks.IsEmpty()) { + return nsnull; + } + return &mChunks[mChunks.Length() - 1]; + } + + class ChunkIterator { + public: + ChunkIterator(MediaSegmentBase& aSegment) + : mSegment(aSegment), mIndex(0) {} + bool IsEnded() { return mIndex >= mSegment.mChunks.Length(); } + void Next() { ++mIndex; } + Chunk& operator*() { return mSegment.mChunks[mIndex]; } + Chunk* operator->() { return &mSegment.mChunks[mIndex]; } + private: + MediaSegmentBase& mSegment; + PRUint32 mIndex; + }; + +protected: + void RemoveLeadingInternal(TrackTicks aDuration, PRUint32 aStartIndex) + { + NS_ASSERTION(aDuration >= 0, "Can't remove negative duration"); + TrackTicks t = aDuration; + PRUint32 chunksToRemove = 0; + for (PRUint32 i = aStartIndex; i < mChunks.Length() && t > 0; ++i) { + Chunk* c = &mChunks[i]; + if (c->GetDuration() > t) { + c->SliceTo(t, c->GetDuration()); + t = 0; + break; + } + t -= c->GetDuration(); + chunksToRemove = i + 1 - aStartIndex; + } + mChunks.RemoveElementsAt(aStartIndex, chunksToRemove); + mDuration -= aDuration - t; + } + + nsTArray mChunks; +}; + +} + +#endif /* MOZILLA_MEDIASEGMENT_H_ */ diff --git a/content/media/SharedBuffer.h b/content/media/SharedBuffer.h new file mode 100644 index 00000000000..49ad6b59fe4 --- /dev/null +++ b/content/media/SharedBuffer.h @@ -0,0 +1,45 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef MOZILLA_SHAREDBUFFER_H_ +#define MOZILLA_SHAREDBUFFER_H_ + +#include "mozilla/mozalloc.h" + +namespace mozilla { + +/** + * Heap-allocated chunk of arbitrary data with threadsafe refcounting. + * Typically you would allocate one of these, fill it in, and then treat it as + * immutable while it's shared. + * This only guarantees 4-byte alignment of the data. For alignment we + * simply assume that the refcount is at least 4-byte aligned and its size + * is divisible by 4. + */ +class SharedBuffer { +public: + NS_INLINE_DECL_THREADSAFE_REFCOUNTING(SharedBuffer) + ~SharedBuffer() {} + + void* Data() { return this + 1; } + + // Takes ownership of aData (which will be freed via moz_free()). + // aData consists of aChannels consecutive buffers, each of aLength samples. + static already_AddRefed Create(size_t aSize) + { + void* m = moz_xmalloc(sizeof(SharedBuffer) + aSize); + nsRefPtr p = new (m) SharedBuffer(); + NS_ASSERTION((reinterpret_cast(p.get() + 1) - reinterpret_cast(p.get())) % 4 == 0, + "SharedBuffers should be at least 4-byte aligned"); + return p.forget(); + } + +private: + SharedBuffer() {} +}; + +} + +#endif /* MOZILLA_SHAREDBUFFER_H_ */ diff --git a/content/media/StreamBuffer.cpp b/content/media/StreamBuffer.cpp new file mode 100644 index 00000000000..bac42d3878f --- /dev/null +++ b/content/media/StreamBuffer.cpp @@ -0,0 +1,60 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "StreamBuffer.h" + +namespace mozilla { + +StreamTime +StreamBuffer::GetEnd() const +{ + StreamTime t = mTracksKnownTime; + for (PRUint32 i = 0; i < mTracks.Length(); ++i) { + Track* track = mTracks[i]; + if (!track->IsEnded()) { + t = NS_MIN(t, track->GetEndTimeRoundDown()); + } + } + return t; +} + +StreamBuffer::Track* +StreamBuffer::FindTrack(TrackID aID) +{ + if (aID == TRACK_NONE) + return nsnull; + for (PRUint32 i = 0; i < mTracks.Length(); ++i) { + Track* track = mTracks[i]; + if (track->GetID() == aID) { + return track; + } + } + return nsnull; +} + +void +StreamBuffer::ForgetUpTo(StreamTime aTime) +{ + // Round to nearest 50ms so we don't spend too much time pruning segments. + const int roundTo = MillisecondsToMediaTime(50); + StreamTime forget = (aTime/roundTo)*roundTo; + if (forget <= mForgottenTime) { + return; + } + mForgottenTime = forget; + + for (PRUint32 i = 0; i < mTracks.Length(); ++i) { + Track* track = mTracks[i]; + if (track->IsEnded() && track->GetEndTimeRoundDown() <= forget) { + mTracks.RemoveElementAt(i); + --i; + continue; + } + TrackTicks forgetTo = NS_MIN(track->GetEnd() - 1, track->TimeToTicksRoundDown(forget)); + track->ForgetUpTo(forgetTo); + } +} + +} diff --git a/content/media/StreamBuffer.h b/content/media/StreamBuffer.h new file mode 100644 index 00000000000..1cda4870231 --- /dev/null +++ b/content/media/StreamBuffer.h @@ -0,0 +1,286 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef MOZILLA_STREAMBUFFER_H_ +#define MOZILLA_STREAMBUFFER_H_ + +#include "mozilla/Util.h" +#include "MediaSegment.h" +#include "nsAutoPtr.h" + +namespace mozilla { + +/** + * Media time relative to the start of a StreamBuffer. + */ +typedef MediaTime StreamTime; +const StreamTime STREAM_TIME_MAX = MEDIA_TIME_MAX; + +/** + * Track rate in Hz. Maximum 1 << MEDIA_TIME_FRAC_BITS Hz. This ensures + * calculations below don't overflow. + */ +typedef PRInt32 TrackRate; +const TrackRate TRACK_RATE_MAX = 1 << MEDIA_TIME_FRAC_BITS; + +/** + * Unique ID for track within a StreamBuffer. Tracks from different + * StreamBuffers may have the same ID; this matters when appending StreamBuffers, + * since tracks with the same ID are matched. Only IDs greater than 0 are allowed. + */ +typedef PRInt32 TrackID; +const TrackID TRACK_NONE = 0; + +inline TrackTicks TimeToTicksRoundUp(TrackRate aRate, StreamTime aMicroseconds) +{ + NS_ASSERTION(0 < aRate && aRate <= TRACK_RATE_MAX, "Bad rate"); + NS_ASSERTION(0 <= aMicroseconds && aMicroseconds <= STREAM_TIME_MAX, "Bad microseconds"); + return (aMicroseconds*aRate + (1 << MEDIA_TIME_FRAC_BITS) - 1) >> MEDIA_TIME_FRAC_BITS; +} + +inline TrackTicks TimeToTicksRoundDown(TrackRate aRate, StreamTime aMicroseconds) +{ + NS_ASSERTION(0 < aRate && aRate <= TRACK_RATE_MAX, "Bad rate"); + NS_ASSERTION(0 <= aMicroseconds && aMicroseconds <= STREAM_TIME_MAX, "Bad microseconds"); + return (aMicroseconds*aRate) >> MEDIA_TIME_FRAC_BITS; +} + +inline StreamTime TicksToTimeRoundUp(TrackRate aRate, TrackTicks aTicks) +{ + NS_ASSERTION(0 < aRate && aRate <= TRACK_RATE_MAX, "Bad rate"); + NS_ASSERTION(0 <= aTicks && aTicks <= TRACK_TICKS_MAX, "Bad samples"); + return ((aTicks << MEDIA_TIME_FRAC_BITS) + aRate - 1)/aRate; +} + +inline StreamTime TicksToTimeRound(TrackRate aRate, TrackTicks aTicks) +{ + NS_ASSERTION(0 < aRate && aRate <= TRACK_RATE_MAX, "Bad rate"); + NS_ASSERTION(0 <= aTicks && aTicks <= TRACK_TICKS_MAX, "Bad samples"); + return ((aTicks << MEDIA_TIME_FRAC_BITS) + aRate/2)/aRate; +} + +inline StreamTime TicksToTimeRoundDown(TrackRate aRate, TrackTicks aTicks) +{ + NS_ASSERTION(0 < aRate && aRate <= TRACK_RATE_MAX, "Bad rate"); + NS_ASSERTION(0 <= aTicks && aTicks <= TRACK_TICKS_MAX, "Bad samples"); + return (aTicks << MEDIA_TIME_FRAC_BITS)/aRate; +} + +/** + * This object contains the decoded data for a stream's tracks. + * A StreamBuffer can be appended to. Logically a StreamBuffer only gets longer, + * but we also have the ability to "forget" data before a certain time that + * we know won't be used again. (We prune a whole number of seconds internally.) + * + * StreamBuffers should only be used from one thread at a time. + * + * A StreamBuffer has a set of tracks that can be of arbitrary types --- + * the data for each track is a MediaSegment. The set of tracks can vary + * over the timeline of the StreamBuffer. + */ +class StreamBuffer { +public: + /** + * Every track has a start time --- when it started in the StreamBuffer. + * It has an end flag; when false, no end point is known; when true, + * the track ends when the data we have for the track runs out. + * Tracks have a unique ID assigned at creation. This allows us to identify + * the same track across StreamBuffers. A StreamBuffer should never have + * two tracks with the same ID (even if they don't overlap in time). + * TODO Tracks can also be enabled and disabled over time. + * TODO Add TimeVarying mEnabled. + */ + class Track { + public: + Track(TrackID aID, TrackRate aRate, TrackTicks aStart, MediaSegment* aSegment) + : mStart(aStart), + mSegment(aSegment), + mRate(aRate), + mID(aID), + mEnded(false) + { + MOZ_COUNT_CTOR(Track); + + NS_ASSERTION(aID > TRACK_NONE, "Bad track ID"); + NS_ASSERTION(0 < aRate && aRate <= TRACK_RATE_MAX, "Invalid rate"); + NS_ASSERTION(0 <= aStart && aStart <= aSegment->GetDuration(), "Bad start position"); + } + ~Track() + { + MOZ_COUNT_DTOR(Track); + } + template T* Get() const + { + if (mSegment->GetType() == T::StaticType()) { + return static_cast(mSegment.get()); + } + return nsnull; + } + MediaSegment* GetSegment() const { return mSegment; } + TrackRate GetRate() const { return mRate; } + TrackID GetID() const { return mID; } + bool IsEnded() const { return mEnded; } + TrackTicks GetStart() const { return mStart; } + TrackTicks GetEnd() const { return mSegment->GetDuration(); } + StreamTime GetEndTimeRoundDown() const + { + return mozilla::TicksToTimeRoundDown(mRate, mSegment->GetDuration()); + } + StreamTime GetStartTimeRoundDown() const + { + return mozilla::TicksToTimeRoundDown(mRate, mStart); + } + TrackTicks TimeToTicksRoundDown(StreamTime aTime) const + { + return mozilla::TimeToTicksRoundDown(mRate, aTime); + } + StreamTime TicksToTimeRoundDown(TrackTicks aTicks) const + { + return mozilla::TicksToTimeRoundDown(mRate, aTicks); + } + MediaSegment::Type GetType() const { return mSegment->GetType(); } + + void SetEnded() { mEnded = true; } + void AppendFrom(Track* aTrack) + { + NS_ASSERTION(!mEnded, "Can't append to ended track"); + NS_ASSERTION(aTrack->mID == mID, "IDs must match"); + NS_ASSERTION(aTrack->mStart == 0, "Source track must start at zero"); + NS_ASSERTION(aTrack->mSegment->GetType() == GetType(), "Track types must match"); + NS_ASSERTION(aTrack->mRate == mRate, "Track rates must match"); + + mSegment->AppendFrom(aTrack->mSegment); + mEnded = aTrack->mEnded; + } + MediaSegment* RemoveSegment() + { + return mSegment.forget(); + } + void ForgetUpTo(TrackTicks aTime) + { + mSegment->ForgetUpTo(aTime); + } + + protected: + friend class StreamBuffer; + + // Start offset is in ticks at rate mRate + TrackTicks mStart; + // The segment data starts at the start of the owning StreamBuffer, i.e., + // there's mStart silence/no video at the beginning. + nsAutoPtr mSegment; + TrackRate mRate; // rate in ticks per second + // Unique ID + TrackID mID; + // True when the track ends with the data in mSegment + bool mEnded; + }; + + class CompareTracksByID { + public: + bool Equals(Track* aA, Track* aB) const { + return aA->GetID() == aB->GetID(); + } + bool LessThan(Track* aA, Track* aB) const { + return aA->GetID() < aB->GetID(); + } + }; + + StreamBuffer() + : mTracksKnownTime(0), mForgottenTime(0) + { + MOZ_COUNT_CTOR(StreamBuffer); + } + ~StreamBuffer() + { + MOZ_COUNT_DTOR(StreamBuffer); + } + + /** + * Takes ownership of aSegment. Don't do this while iterating, or while + * holding a Track reference. + * aSegment must have aStart worth of null data. + */ + Track& AddTrack(TrackID aID, TrackRate aRate, TrackTicks aStart, MediaSegment* aSegment) + { + NS_ASSERTION(TimeToTicksRoundDown(aRate, mTracksKnownTime) <= aStart, + "Start time too early"); + NS_ASSERTION(!FindTrack(aID), "Track with this ID already exists"); + + return **mTracks.InsertElementSorted(new Track(aID, aRate, aStart, aSegment), + CompareTracksByID()); + } + void AdvanceKnownTracksTime(StreamTime aKnownTime) + { + NS_ASSERTION(aKnownTime >= mTracksKnownTime, "Can't move tracks-known time earlier"); + mTracksKnownTime = aKnownTime; + } + + /** + * The end time for the StreamBuffer is the latest time for which we have + * data for all tracks that haven't ended by that time. + */ + StreamTime GetEnd() const; + + Track* FindTrack(TrackID aID); + + class TrackIter { + public: + /** + * Iterate through the tracks of aBuffer in order of ID. + */ + TrackIter(const StreamBuffer& aBuffer) : + mBuffer(&aBuffer.mTracks), mIndex(0), mMatchType(false) {} + /** + * Iterate through the tracks of aBuffer with type aType, in order of ID. + */ + TrackIter(const StreamBuffer& aBuffer, MediaSegment::Type aType) : + mBuffer(&aBuffer.mTracks), mIndex(0), mType(aType), mMatchType(true) { FindMatch(); } + bool IsEnded() { return mIndex >= mBuffer->Length(); } + void Next() + { + ++mIndex; + FindMatch(); + } + Track& operator*() { return *mBuffer->ElementAt(mIndex); } + Track* operator->() { return mBuffer->ElementAt(mIndex); } + private: + void FindMatch() + { + if (!mMatchType) + return; + while (mIndex < mBuffer->Length() && + mBuffer->ElementAt(mIndex)->GetType() != mType) { + ++mIndex; + } + } + + const nsTArray >* mBuffer; + PRUint32 mIndex; + MediaSegment::Type mType; + bool mMatchType; + }; + friend class TrackIter; + + /** + * Forget stream data before aTime; they will no longer be needed. + * Also can forget entire tracks that have ended at or before aTime. + * Can't be used to forget beyond GetEnd(). + */ + void ForgetUpTo(StreamTime aTime); + +protected: + // Any new tracks added will start at or after this time. In other words, the track + // list is complete and correct for all times less than this time. + StreamTime mTracksKnownTime; + StreamTime mForgottenTime; + // All known tracks for this StreamBuffer + nsTArray > mTracks; +}; + +} + +#endif /* MOZILLA_STREAMBUFFER_H_ */ + diff --git a/content/media/VideoSegment.h b/content/media/VideoSegment.h new file mode 100644 index 00000000000..99973cb9115 --- /dev/null +++ b/content/media/VideoSegment.h @@ -0,0 +1,117 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef MOZILLA_VIDEOSEGMENT_H_ +#define MOZILLA_VIDEOSEGMENT_H_ + +#include "MediaSegment.h" +#include "ImageLayers.h" + +namespace mozilla { + +class VideoFrame { +public: + typedef mozilla::layers::Image Image; + + VideoFrame(already_AddRefed aImage, const gfxIntSize& aIntrinsicSize) + : mImage(aImage), mIntrinsicSize(aIntrinsicSize) {} + VideoFrame() : mIntrinsicSize(0, 0) {} + + bool operator==(const VideoFrame& aFrame) const + { + return mImage == aFrame.mImage && mIntrinsicSize == aFrame.mIntrinsicSize; + } + bool operator!=(const VideoFrame& aFrame) const + { + return !operator==(aFrame); + } + + Image* GetImage() const { return mImage; } + const gfxIntSize& GetIntrinsicSize() const { return mIntrinsicSize; } + void SetNull() { mImage = nsnull; mIntrinsicSize = gfxIntSize(0, 0); } + void TakeFrom(VideoFrame* aFrame) + { + mImage = aFrame->mImage.forget(); + mIntrinsicSize = aFrame->mIntrinsicSize; + } + +protected: + // mImage can be null to indicate "no video" (aka "empty frame"). It can + // still have an intrinsic size in this case. + nsRefPtr mImage; + // The desired size to render the video frame at. + gfxIntSize mIntrinsicSize; +}; + + +struct VideoChunk { + void SliceTo(TrackTicks aStart, TrackTicks aEnd) + { + NS_ASSERTION(aStart >= 0 && aStart < aEnd && aEnd <= mDuration, + "Slice out of bounds"); + mDuration = aEnd - aStart; + } + TrackTicks GetDuration() const { return mDuration; } + bool CanCombineWithFollowing(const VideoChunk& aOther) const + { + return aOther.mFrame == mFrame; + } + bool IsNull() const { return !mFrame.GetImage(); } + void SetNull(TrackTicks aDuration) + { + mDuration = aDuration; + mFrame.SetNull(); + } + + TrackTicks mDuration; + VideoFrame mFrame; +}; + +class VideoSegment : public MediaSegmentBase { +public: + typedef mozilla::layers::Image Image; + + VideoSegment() : MediaSegmentBase(VIDEO) {} + + void AppendFrame(already_AddRefed aImage, TrackTicks aDuration, + const gfxIntSize& aIntrinsicSize) + { + VideoChunk* chunk = AppendChunk(aDuration); + VideoFrame frame(aImage, aIntrinsicSize); + chunk->mFrame.TakeFrom(&frame); + } + const VideoFrame* GetFrameAt(TrackTicks aOffset, TrackTicks* aStart = nsnull) + { + VideoChunk* c = FindChunkContaining(aOffset, aStart); + if (!c) { + return nsnull; + } + return &c->mFrame; + } + const VideoFrame* GetLastFrame(TrackTicks* aStart = nsnull) + { + VideoChunk* c = GetLastChunk(); + if (!c) { + return nsnull; + } + if (aStart) { + *aStart = mDuration - c->mDuration; + } + return &c->mFrame; + } + + // Segment-generic methods not in MediaSegmentBase + void InitFrom(const VideoSegment& aOther) + { + } + void SliceFrom(const VideoSegment& aOther, TrackTicks aStart, TrackTicks aEnd) { + BaseSliceFrom(aOther, aStart, aEnd); + } + static Type StaticType() { return VIDEO; } +}; + +} + +#endif /* MOZILLA_VIDEOSEGMENT_H_ */ From a619144e05886657aabd20bb4bfd98e1acbcb1b9 Mon Sep 17 00:00:00 2001 From: Robert O'Callahan Date: Mon, 30 Apr 2012 15:11:26 +1200 Subject: [PATCH 009/162] Bug 664918. Part 3: Create MediaStream and MediaGraphManager for internal management of real-time media processing. r=jesup --- content/media/Makefile.in | 2 + content/media/MediaStreamGraph.cpp | 1772 ++++++++++++++++++++++++++++ content/media/MediaStreamGraph.h | 394 +++++++ 3 files changed, 2168 insertions(+) create mode 100644 content/media/MediaStreamGraph.cpp create mode 100644 content/media/MediaStreamGraph.h diff --git a/content/media/Makefile.in b/content/media/Makefile.in index a794d080400..e1b217e4e39 100644 --- a/content/media/Makefile.in +++ b/content/media/Makefile.in @@ -50,6 +50,7 @@ EXPORTS = \ FileBlockCache.h \ MediaResource.h \ MediaSegment.h \ + MediaStreamGraph.h \ nsAudioAvailableEventManager.h \ nsBuiltinDecoder.h \ nsBuiltinDecoderStateMachine.h \ @@ -68,6 +69,7 @@ CPPSRCS = \ AudioSegment.cpp \ FileBlockCache.cpp \ MediaResource.cpp \ + MediaStreamGraph.cpp \ nsAudioAvailableEventManager.cpp \ nsBuiltinDecoder.cpp \ nsBuiltinDecoderStateMachine.cpp \ diff --git a/content/media/MediaStreamGraph.cpp b/content/media/MediaStreamGraph.cpp new file mode 100644 index 00000000000..96e09babc27 --- /dev/null +++ b/content/media/MediaStreamGraph.cpp @@ -0,0 +1,1772 @@ +/*-*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "MediaStreamGraph.h" + +#include "mozilla/Monitor.h" +#include "mozilla/TimeStamp.h" +#include "AudioSegment.h" +#include "VideoSegment.h" +#include "nsContentUtils.h" +#include "nsIAppShell.h" +#include "nsIObserver.h" +#include "nsServiceManagerUtils.h" +#include "nsWidgetsCID.h" +#include "nsXPCOMCIDInternal.h" +#include "prlog.h" +#include "VideoUtils.h" + +using namespace mozilla::layers; + +namespace mozilla { + +namespace { + +#ifdef PR_LOGGING +PRLogModuleInfo* gMediaStreamGraphLog; +#define LOG(type, msg) PR_LOG(gMediaStreamGraphLog, type, msg) +#else +#define LOG(type, msg) +#endif + +/** + * Assume we can run an iteration of the MediaStreamGraph loop in this much time + * or less. + * We try to run the control loop at this rate. + */ +const int MEDIA_GRAPH_TARGET_PERIOD_MS = 10; + +/** + * Assume that we might miss our scheduled wakeup of the MediaStreamGraph by + * this much. + */ +const int SCHEDULE_SAFETY_MARGIN_MS = 10; + +/** + * Try have this much audio buffered in streams and queued to the hardware. + * The maximum delay to the end of the next control loop + * is 2*MEDIA_GRAPH_TARGET_PERIOD_MS + SCHEDULE_SAFETY_MARGIN_MS. + * There is no point in buffering more audio than this in a stream at any + * given time (until we add processing). + * This is not optimal yet. + */ +const int AUDIO_TARGET_MS = 2*MEDIA_GRAPH_TARGET_PERIOD_MS + + SCHEDULE_SAFETY_MARGIN_MS; + +/** + * Try have this much video buffered. Video frames are set + * near the end of the iteration of the control loop. The maximum delay + * to the setting of the next video frame is 2*MEDIA_GRAPH_TARGET_PERIOD_MS + + * SCHEDULE_SAFETY_MARGIN_MS. This is not optimal yet. + */ +const int VIDEO_TARGET_MS = 2*MEDIA_GRAPH_TARGET_PERIOD_MS + + SCHEDULE_SAFETY_MARGIN_MS; + +/** + * A per-stream update message passed from the media graph thread to the + * main thread. + */ +struct StreamUpdate { + PRInt64 mGraphUpdateIndex; + nsRefPtr mStream; + StreamTime mNextMainThreadCurrentTime; + bool mNextMainThreadFinished; +}; + +/** + * This represents a message passed from the main thread to the graph thread. + * A ControlMessage always references a particular affected stream. + */ +class ControlMessage { +public: + ControlMessage(MediaStream* aStream) : mStream(aStream) + { + MOZ_COUNT_CTOR(ControlMessage); + } + // All these run on the graph thread + virtual ~ControlMessage() + { + MOZ_COUNT_DTOR(ControlMessage); + } + // Executed before we know what the action time for this message will be. + // Call NoteStreamAffected on the stream whose output will be + // modified by this message. Default implementation calls + // NoteStreamAffected(mStream). + virtual void UpdateAffectedStream(); + // Executed after we know what the action time for this message will be. + virtual void Process() {} + // When we're shutting down the application, most messages are ignored but + // some cleanup messages should still be processed (on the main thread). + virtual void ProcessDuringShutdown() {} + +protected: + // We do not hold a reference to mStream. The main thread will be holding + // a reference to the stream while this message is in flight. The last message + // referencing a stream is the Destroy message for that stream. + MediaStream* mStream; +}; + +} + +/** + * The implementation of a media stream graph. This class is private to this + * file. It's not in the anonymous namespace because MediaStream needs to + * be able to friend it. + * + * Currently we only have one per process. + */ +class MediaStreamGraphImpl : public MediaStreamGraph { +public: + MediaStreamGraphImpl(); + ~MediaStreamGraphImpl() + { + NS_ASSERTION(IsEmpty(), + "All streams should have been destroyed by messages from the main thread"); + LOG(PR_LOG_DEBUG, ("MediaStreamGraph %p destroyed", this)); + } + + // Main thread only. + /** + * This runs every time we need to sync state from the media graph thread + * to the main thread while the main thread is not in the middle + * of a script. It runs during a "stable state" (per HTML5) or during + * an event posted to the main thread. + */ + void RunInStableState(); + /** + * Ensure a runnable to run RunInStableState is posted to the appshell to + * run at the next stable state (per HTML5). + * See EnsureStableStateEventPosted. + */ + void EnsureRunInStableState(); + /** + * Called to apply a StreamUpdate to its stream. + */ + void ApplyStreamUpdate(StreamUpdate* aUpdate); + /** + * Append a ControlMessage to the message queue. This queue is drained + * during RunInStableState; the messages will run on the graph thread. + */ + void AppendMessage(ControlMessage* aMessage); + /** + * Make this MediaStreamGraph enter forced-shutdown state. This state + * will be noticed by the media graph thread, which will shut down all streams + * and other state controlled by the media graph thread. + * This is called during application shutdown. + */ + void ForceShutDown(); + /** + * Shutdown() this MediaStreamGraph's threads and return when they've shut down. + */ + void ShutdownThreads(); + + // The following methods run on the graph thread (or possibly the main thread if + // mLifecycleState > LIFECYCLE_RUNNING) + /** + * Runs main control loop on the graph thread. Normally a single invocation + * of this runs for the entire lifetime of the graph thread. + */ + void RunThread(); + /** + * Call this to indicate that another iteration of the control loop is + * required on its regular schedule. The monitor must not be held. + */ + void EnsureNextIteration(); + /** + * As above, but with the monitor already held. + */ + void EnsureNextIterationLocked(MonitorAutoLock& aLock); + /** + * Call this to indicate that another iteration of the control loop is + * required immediately. The monitor must already be held. + */ + void EnsureImmediateWakeUpLocked(MonitorAutoLock& aLock); + /** + * Ensure there is an event posted to the main thread to run RunInStableState. + * mMonitor must be held. + * See EnsureRunInStableState + */ + void EnsureStableStateEventPosted(); + /** + * Generate messages to the main thread to update it for all state changes. + * mMonitor must be held. + */ + void PrepareUpdatesToMainThreadState(); + // The following methods are the various stages of RunThread processing. + /** + * Compute a new current time for the graph and advance all on-graph-thread + * state to the new current time. + */ + void UpdateCurrentTime(); + /** + * Update mLastActionTime to the time at which the current set of messages + * will take effect. + */ + void ChooseActionTime(); + /** + * Compute the blocking states of streams from mBlockingDecisionsMadeUntilTime + * until the desired future time (determined by heuristic). + * Updates mBlockingDecisionsMadeUntilTime and sets MediaStream::mBlocked + * for all streams. + */ + void RecomputeBlocking(); + // The following methods are used to help RecomputeBlocking. + /** + * Mark a stream blocked at time aTime. If this results in decisions that need + * to be revisited at some point in the future, *aEnd will be reduced to the + * first time in the future to recompute those decisions. + */ + void MarkStreamBlocked(MediaStream* aStream, GraphTime aTime, GraphTime* aEnd); + /** + * Recompute blocking for all streams for the interval starting at aTime. + * If this results in decisions that need to be revisited at some point + * in the future, *aEnd will be reduced to the first time in the future to + * recompute those decisions. + */ + void RecomputeBlockingAt(GraphTime aTime, GraphTime aEndBlockingDecisions, + GraphTime* aEnd); + /** + * Returns true if aStream will underrun at aTime for its own playback. + * aEndBlockingDecisions is when we plan to stop making blocking decisions. + * *aEnd will be reduced to the first time in the future to recompute these + * decisions. + */ + bool WillUnderrun(MediaStream* aStream, GraphTime aTime, + GraphTime aEndBlockingDecisions, GraphTime* aEnd); + /** + * Return true if there is an explicit blocker set from the current time + * indefinitely far into the future. + */ + bool IsAlwaysExplicitlyBlocked(MediaStream* aStream); + /** + * Given a graph time aTime, convert it to a stream time taking into + * account the time during which aStream is scheduled to be blocked. + */ + StreamTime GraphTimeToStreamTime(MediaStream* aStream, StreamTime aTime); + enum { + INCLUDE_TRAILING_BLOCKED_INTERVAL = 0x01 + }; + /** + * Given a stream time aTime, convert it to a graph time taking into + * account the time during which aStream is scheduled to be blocked. + * aTime must be <= mBlockingDecisionsMadeUntilTime since blocking decisions + * are only known up to that point. + * If aTime is exactly at the start of a blocked interval, then the blocked + * interval is included in the time returned if and only if + * aFlags includes INCLUDE_TRAILING_BLOCKED_INTERVAL. + */ + GraphTime StreamTimeToGraphTime(MediaStream* aStream, StreamTime aTime, + PRUint32 aFlags = 0); + /** + * Get the current audio position of the stream's audio output. + */ + GraphTime GetAudioPosition(MediaStream* aStream); + /** + * If aStream needs an audio stream but doesn't have one, create it. + * If aStream doesn't need an audio stream but has one, destroy it. + */ + void CreateOrDestroyAudioStream(GraphTime aAudioOutputStartTime, + MediaStream* aStream); + /** + * Update aStream->mFirstActiveTracks. + */ + void UpdateFirstActiveTracks(MediaStream* aStream); + /** + * Queue audio (mix of stream audio and silence for blocked intervals) + * to the audio output stream. + */ + void PlayAudio(MediaStream* aStream, GraphTime aFrom, GraphTime aTo); + /** + * Set the correct current video frame for stream aStream. + */ + void PlayVideo(MediaStream* aStream); + /** + * No more data will be forthcoming for aStream. The stream will end + * at the current buffer end point. The StreamBuffer's tracks must be + * explicitly set to finished by the caller. + */ + void FinishStream(MediaStream* aStream); + /** + * Compute how much stream data we would like to buffer for aStream. + */ + StreamTime GetDesiredBufferEnd(MediaStream* aStream); + /** + * Returns true when there are no active streams. + */ + bool IsEmpty() { return mStreams.IsEmpty(); } + + // For use by control messages + /** + * Identify which graph update index we are currently processing. + */ + PRInt64 GetProcessingGraphUpdateIndex() { return mProcessingGraphUpdateIndex; } + /** + * Marks aStream as affected by a change in its output at desired time aTime + * (in the timeline of aStream). The change may not actually happen at this time, + * it may be delayed until later if there is buffered data we can't change. + */ + void NoteStreamAffected(MediaStream* aStream, double aTime); + /** + * Marks aStream as affected by a change in its output at the earliest + * possible time. + */ + void NoteStreamAffected(MediaStream* aStream); + /** + * Add aStream to the graph and initializes its graph-specific state. + */ + void AddStream(MediaStream* aStream); + /** + * Remove aStream from the graph. Ensures that pending messages about the + * stream back to the main thread are flushed. + */ + void RemoveStream(MediaStream* aStream); + + /** + * Compute the earliest time at which an action be allowed to occur on any + * stream. Actions cannot be earlier than the previous action time, and + * cannot affect already-committed blocking decisions (and associated + * buffered audio). + */ + GraphTime GetEarliestActionTime() + { + return NS_MAX(mCurrentTime, NS_MAX(mLastActionTime, mBlockingDecisionsMadeUntilTime)); + } + + // Data members + + /** + * Media graph thread. + * Readonly after initialization on the main thread. + */ + nsCOMPtr mThread; + + // The following state is managed on the graph thread only, unless + // mLifecycleState > LIFECYCLE_RUNNING in which case the graph thread + // is not running and this state can be used from the main thread. + + nsTArray > mStreams; + /** + * The time the last action was deemed to have occurred. This could be + * later than mCurrentTime if actions have to be delayed during data + * buffering, or before mCurrentTime if mCurrentTime has advanced since + * the last action happened. In ControlMessage::Process calls, + * mLastActionTime has always been updated to be >= mCurrentTime. + */ + GraphTime mLastActionTime; + /** + * The current graph time for the current iteration of the RunThread control + * loop. + */ + GraphTime mCurrentTime; + /** + * Blocking decisions have been made up to this time. We also buffer audio + * up to this time. + */ + GraphTime mBlockingDecisionsMadeUntilTime; + /** + * This is only used for logging. + */ + TimeStamp mInitialTimeStamp; + /** + * The real timestamp of the latest run of UpdateCurrentTime. + */ + TimeStamp mCurrentTimeStamp; + /** + * Which update batch we are currently processing. + */ + PRInt64 mProcessingGraphUpdateIndex; + + // mMonitor guards the data below. + // MediaStreamGraph normally does its work without holding mMonitor, so it is + // not safe to just grab mMonitor from some thread and start monkeying with + // the graph. Instead, communicate with the graph thread using provided + // mechanisms such as the ControlMessage queue. + Monitor mMonitor; + + // Data guarded by mMonitor (must always be accessed with mMonitor held, + // regardless of the value of mLifecycleState. + + /** + * State to copy to main thread + */ + nsTArray mStreamUpdates; + /** + * Runnables to run after the next update to main thread state. + */ + nsTArray > mUpdateRunnables; + struct MessageBlock { + PRInt64 mGraphUpdateIndex; + nsTArray > mMessages; + }; + /** + * A list of batches of messages to process. Each batch is processed + * as an atomic unit. + */ + nsTArray mMessageQueue; + /** + * This enum specifies where this graph is in its lifecycle. This is used + * to control shutdown. + * Shutdown is tricky because it can happen in two different ways: + * 1) Shutdown due to inactivity. RunThread() detects that it has no + * pending messages and no streams, and exits. The next RunInStableState() + * checks if there are new pending messages from the main thread (true only + * if new stream creation raced with shutdown); if there are, it revives + * RunThread(), otherwise it commits to shutting down the graph. New stream + * creation after this point will create a new graph. An async event is + * dispatched to Shutdown() the graph's threads and then delete the graph + * object. + * 2) Forced shutdown at application shutdown. A flag is set, RunThread() + * detects the flag and exits, the next RunInStableState() detects the flag, + * and dispatches the async event to Shutdown() the graph's threads. However + * the graph object is not deleted. New messages for the graph are processed + * synchronously on the main thread if necessary. When the last stream is + * destroyed, the graph object is deleted. + */ + enum LifecycleState { + // The graph thread hasn't started yet. + LIFECYCLE_THREAD_NOT_STARTED, + // RunThread() is running normally. + LIFECYCLE_RUNNING, + // In the following states, the graph thread is not running so + // all "graph thread only" state in this class can be used safely + // on the main thread. + // RunThread() has exited and we're waiting for the next + // RunInStableState(), at which point we can clean up the main-thread + // side of the graph. + LIFECYCLE_WAITING_FOR_MAIN_THREAD_CLEANUP, + // RunInStableState() posted a ShutdownRunnable, and we're waiting for it + // to shut down the graph thread(s). + LIFECYCLE_WAITING_FOR_THREAD_SHUTDOWN, + // Graph threads have shut down but we're waiting for remaining streams + // to be destroyed. Only happens during application shutdown since normally + // we'd only shut down a graph when it has no streams. + LIFECYCLE_WAITING_FOR_STREAM_DESTRUCTION + }; + LifecycleState mLifecycleState; + /** + * This enum specifies the wait state of the graph thread. + */ + enum WaitState { + // RunThread() is running normally + WAITSTATE_RUNNING, + // RunThread() is paused waiting for its next iteration, which will + // happen soon + WAITSTATE_WAITING_FOR_NEXT_ITERATION, + // RunThread() is paused indefinitely waiting for something to change + WAITSTATE_WAITING_INDEFINITELY, + // Something has signaled RunThread() to wake up immediately, + // but it hasn't done so yet + WAITSTATE_WAKING_UP + }; + WaitState mWaitState; + /** + * True when another iteration of the control loop is required. + */ + bool mNeedAnotherIteration; + /** + * True when we need to do a forced shutdown during application shutdown. + */ + bool mForceShutDown; + /** + * True when we have posted an event to the main thread to run + * RunInStableState() and the event hasn't run yet. + */ + bool mPostedRunInStableStateEvent; + + // Main thread only + + /** + * Messages posted by the current event loop task. These are forwarded to + * the media graph thread during RunInStableState. We can't forward them + * immediately because we want all messages between stable states to be + * processed as an atomic batch. + */ + nsTArray > mCurrentTaskMessageQueue; + /** + * True when RunInStableState has determined that mLifecycleState is > + * LIFECYCLE_RUNNING. Since only the main thread can reset mLifecycleState to + * LIFECYCLE_RUNNING, this can be relied on to not change unexpectedly. + */ + bool mDetectedNotRunning; + /** + * True when a stable state runner has been posted to the appshell to run + * RunInStableState at the next stable state. + */ + bool mPostedRunInStableState; +}; + +/** + * The singleton graph instance. + */ +static MediaStreamGraphImpl* gGraph; + +StreamTime +MediaStreamGraphImpl::GetDesiredBufferEnd(MediaStream* aStream) +{ + StreamTime current = mCurrentTime - aStream->mBufferStartTime; + StreamTime desiredEnd = current; + if (!aStream->mAudioOutputs.IsEmpty()) { + desiredEnd = NS_MAX(desiredEnd, current + MillisecondsToMediaTime(AUDIO_TARGET_MS)); + } + if (!aStream->mVideoOutputs.IsEmpty()) { + desiredEnd = NS_MAX(desiredEnd, current + MillisecondsToMediaTime(VIDEO_TARGET_MS)); + } + return desiredEnd; +} + +bool +MediaStreamGraphImpl::IsAlwaysExplicitlyBlocked(MediaStream* aStream) +{ + GraphTime t = mCurrentTime; + while (true) { + GraphTime end; + if (aStream->mExplicitBlockerCount.GetAt(t, &end) == 0) + return false; + if (end >= GRAPH_TIME_MAX) + return true; + t = end; + } +} + +void +MediaStreamGraphImpl::FinishStream(MediaStream* aStream) +{ + if (aStream->mFinished) + return; + LOG(PR_LOG_DEBUG, ("MediaStream %p will finish", aStream)); + aStream->mFinished = true; + // Force at least one more iteration of the control loop, since we rely + // on UpdateCurrentTime to notify our listeners once the stream end + // has been reached. + EnsureNextIteration(); +} + +void +MediaStreamGraphImpl::NoteStreamAffected(MediaStream* aStream, double aTime) +{ + NS_ASSERTION(aTime >= 0, "Bad time"); + GraphTime t = + NS_MAX(GetEarliestActionTime(), + StreamTimeToGraphTime(aStream, SecondsToMediaTime(aTime), + INCLUDE_TRAILING_BLOCKED_INTERVAL)); + aStream->mMessageAffectedTime = NS_MIN(aStream->mMessageAffectedTime, t); +} + +void +MediaStreamGraphImpl::NoteStreamAffected(MediaStream* aStream) +{ + GraphTime t = GetEarliestActionTime(); + aStream->mMessageAffectedTime = NS_MIN(aStream->mMessageAffectedTime, t); +} + +void +ControlMessage::UpdateAffectedStream() +{ + NS_ASSERTION(mStream, "Must have stream for default UpdateAffectedStream"); + mStream->GraphImpl()->NoteStreamAffected(mStream); +} + +void +MediaStreamGraphImpl::AddStream(MediaStream* aStream) +{ + aStream->mBufferStartTime = mCurrentTime; + aStream->mMessageAffectedTime = GetEarliestActionTime(); + *mStreams.AppendElement() = already_AddRefed(aStream); + LOG(PR_LOG_DEBUG, ("Adding media stream %p to the graph", aStream)); +} + +void +MediaStreamGraphImpl::RemoveStream(MediaStream* aStream) +{ + // Remove references in mStreamUpdates before we allow aStream to die. + // Pending updates are not needed (since the main thread has already given + // up the stream) so we will just drop them. + { + MonitorAutoLock lock(mMonitor); + for (PRUint32 i = 0; i < mStreamUpdates.Length(); ++i) { + if (mStreamUpdates[i].mStream == aStream) { + mStreamUpdates[i].mStream = nsnull; + } + } + } + + // This unrefs the stream, probably destroying it + mStreams.RemoveElement(aStream); + + LOG(PR_LOG_DEBUG, ("Removing media stream %p from the graph", aStream)); +} + +void +MediaStreamGraphImpl::ChooseActionTime() +{ + mLastActionTime = GetEarliestActionTime(); +} + +StreamTime +MediaStreamGraphImpl::GraphTimeToStreamTime(MediaStream* aStream, + GraphTime aTime) +{ + NS_ASSERTION(aTime <= mBlockingDecisionsMadeUntilTime, + "Don't ask about times where we haven't made blocking decisions yet"); + if (aTime <= mCurrentTime) { + return NS_MAX(0, aTime - aStream->mBufferStartTime); + } + GraphTime t = mCurrentTime; + StreamTime s = t - aStream->mBufferStartTime; + while (t < aTime) { + GraphTime end; + if (!aStream->mBlocked.GetAt(t, &end)) { + s += NS_MIN(aTime, end) - t; + } + t = end; + } + return NS_MAX(0, s); +} + +GraphTime +MediaStreamGraphImpl::StreamTimeToGraphTime(MediaStream* aStream, + StreamTime aTime, PRUint32 aFlags) +{ + if (aTime >= STREAM_TIME_MAX) { + return GRAPH_TIME_MAX; + } + MediaTime bufferElapsedToCurrentTime = mCurrentTime - aStream->mBufferStartTime; + if (aTime < bufferElapsedToCurrentTime || + (aTime == bufferElapsedToCurrentTime && !(aFlags & INCLUDE_TRAILING_BLOCKED_INTERVAL))) { + return aTime + aStream->mBufferStartTime; + } + + MediaTime streamAmount = aTime - bufferElapsedToCurrentTime; + NS_ASSERTION(streamAmount >= 0, "Can't answer queries before current time"); + + GraphTime t = mCurrentTime; + while (t < GRAPH_TIME_MAX) { + bool blocked; + GraphTime end; + if (t < mBlockingDecisionsMadeUntilTime) { + blocked = aStream->mBlocked.GetAt(t, &end); + end = NS_MIN(end, mBlockingDecisionsMadeUntilTime); + } else { + blocked = false; + end = GRAPH_TIME_MAX; + } + if (blocked) { + t = end; + } else { + if (streamAmount == 0) { + // No more stream time to consume at time t, so we're done. + break; + } + MediaTime consume = NS_MIN(end - t, streamAmount); + streamAmount -= consume; + t += consume; + } + } + return t; +} + +GraphTime +MediaStreamGraphImpl::GetAudioPosition(MediaStream* aStream) +{ + if (!aStream->mAudioOutput) { + return mCurrentTime; + } + return aStream->mAudioPlaybackStartTime + + TicksToTimeRoundDown(aStream->mAudioOutput->GetRate(), + aStream->mAudioOutput->GetPositionInFrames()); +} + +void +MediaStreamGraphImpl::UpdateCurrentTime() +{ + GraphTime prevCurrentTime = mCurrentTime; + + TimeStamp now = TimeStamp::Now(); + // The earliest buffer end time for streams that haven't finished. We can't + // advance the current time past this point. + GraphTime minBufferEndTime = GRAPH_TIME_MAX; + for (PRUint32 i = 0; i < mStreams.Length(); ++i) { + MediaStream* stream = mStreams[i]; + GraphTime blockedBufferEndTime = + StreamTimeToGraphTime(stream, stream->GetBufferEnd(), INCLUDE_TRAILING_BLOCKED_INTERVAL); + if (stream->mAudioOutput && + (!stream->mFinished || mBlockingDecisionsMadeUntilTime <= blockedBufferEndTime)) { + // XXX We should take audio positions into account when determining how + // far to advance the current time. Basically the current time should + // track the average or minimum of the audio positions. We don't do this + // currently since the audio positions aren't accurate enough. This + // logging code is helpful to track the accuracy of audio positions. + GraphTime audioPosition = GetAudioPosition(stream); + LOG(PR_LOG_DEBUG, ("Audio position for stream %p is %f", stream, + MediaTimeToSeconds(audioPosition))); + } + if (!stream->mFinished) { + minBufferEndTime = NS_MIN(minBufferEndTime, blockedBufferEndTime); + } + } + + NS_ASSERTION(mCurrentTime <= minBufferEndTime, + "We shouldn't have already advanced beyond buffer end!"); + GraphTime nextCurrentTime = + SecondsToMediaTime((now - mCurrentTimeStamp).ToSeconds()) + mCurrentTime; + if (minBufferEndTime < nextCurrentTime) { + LOG(PR_LOG_WARNING, ("Reducing current time to minimum buffer end")); + nextCurrentTime = minBufferEndTime; + } + mCurrentTimeStamp = now; + + mBlockingDecisionsMadeUntilTime = + NS_MAX(nextCurrentTime, mBlockingDecisionsMadeUntilTime); + LOG(PR_LOG_DEBUG, ("Updating current time to %f (minBufferEndTime %f, real %f, mBlockingDecisionsMadeUntilTime %f)", + MediaTimeToSeconds(nextCurrentTime), + MediaTimeToSeconds(minBufferEndTime), + (now - mInitialTimeStamp).ToSeconds(), + MediaTimeToSeconds(mBlockingDecisionsMadeUntilTime))); + + if (prevCurrentTime >= nextCurrentTime) { + NS_ASSERTION(prevCurrentTime == nextCurrentTime, "Time can't go backwards!"); + // This could happen due to low clock resolution, maybe? + LOG(PR_LOG_DEBUG, ("Time did not advance")); + return; + } + + for (PRUint32 i = 0; i < mStreams.Length(); ++i) { + MediaStream* stream = mStreams[i]; + + // Calculate blocked time and fire Blocked/Unblocked events + GraphTime blockedTime = 0; + GraphTime t = prevCurrentTime; + // Save current blocked status + bool wasBlocked = stream->mBlocked.GetAt(prevCurrentTime); + while (t < nextCurrentTime) { + GraphTime end; + bool blocked = stream->mBlocked.GetAt(t, &end); + if (blocked) { + blockedTime += NS_MIN(end, nextCurrentTime) - t; + } + if (blocked != wasBlocked) { + for (PRUint32 j = 0; j < stream->mListeners.Length(); ++j) { + MediaStreamListener* l = stream->mListeners[j]; + l->NotifyBlockingChanged(this, + blocked ? MediaStreamListener::BLOCKED : MediaStreamListener::UNBLOCKED); + } + wasBlocked = blocked; + } + t = end; + } + + stream->AdvanceTimeVaryingValuesToCurrentTime(nextCurrentTime, blockedTime); + // Advance mBlocked last so that implementations of + // AdvanceTimeVaryingValuesToCurrentTime can rely on the value of mBlocked. + stream->mBlocked.AdvanceCurrentTime(nextCurrentTime); + + if (blockedTime < nextCurrentTime - mCurrentTime) { + for (PRUint32 i = 0; i < stream->mListeners.Length(); ++i) { + MediaStreamListener* l = stream->mListeners[i]; + l->NotifyOutput(this); + } + } + + if (stream->mFinished && !stream->mNotifiedFinished && + stream->mBufferStartTime + stream->GetBufferEnd() <= nextCurrentTime) { + stream->mNotifiedFinished = true; + for (PRUint32 j = 0; j < stream->mListeners.Length(); ++j) { + MediaStreamListener* l = stream->mListeners[j]; + l->NotifyFinished(this); + } + } + + LOG(PR_LOG_DEBUG, ("MediaStream %p bufferStartTime=%f blockedTime=%f", + stream, MediaTimeToSeconds(stream->mBufferStartTime), + MediaTimeToSeconds(blockedTime))); + } + + mCurrentTime = nextCurrentTime; +} + +void +MediaStreamGraphImpl::MarkStreamBlocked(MediaStream* aStream, + GraphTime aTime, GraphTime* aEnd) +{ + NS_ASSERTION(!aStream->mBlocked.GetAt(aTime), "MediaStream already blocked"); + + aStream->mBlocked.SetAtAndAfter(aTime, true); +} + +bool +MediaStreamGraphImpl::WillUnderrun(MediaStream* aStream, GraphTime aTime, + GraphTime aEndBlockingDecisions, GraphTime* aEnd) +{ + // Finished streams, or streams that aren't being played back, can't underrun. + if (aStream->mFinished || + (aStream->mAudioOutputs.IsEmpty() && aStream->mVideoOutputs.IsEmpty())) { + return false; + } + GraphTime bufferEnd = + StreamTimeToGraphTime(aStream, aStream->GetBufferEnd(), + INCLUDE_TRAILING_BLOCKED_INTERVAL); + NS_ASSERTION(bufferEnd >= mCurrentTime, "Buffer underran"); + // We should block after bufferEnd. + if (bufferEnd <= aTime) { + LOG(PR_LOG_DEBUG, ("MediaStream %p will block due to data underrun, " + "bufferEnd %f", + aStream, MediaTimeToSeconds(bufferEnd))); + return true; + } + // We should keep blocking if we're currently blocked and we don't have + // data all the way through to aEndBlockingDecisions. If we don't have + // data all the way through to aEndBlockingDecisions, we'll block soon, + // but we might as well remain unblocked and play the data we've got while + // we can. + if (bufferEnd <= aEndBlockingDecisions && aStream->mBlocked.GetBefore(aTime)) { + LOG(PR_LOG_DEBUG, ("MediaStream %p will block due to speculative data underrun, " + "bufferEnd %f", + aStream, MediaTimeToSeconds(bufferEnd))); + return true; + } + // Reconsider decisions at bufferEnd + *aEnd = NS_MIN(*aEnd, bufferEnd); + return false; +} + +void +MediaStreamGraphImpl::RecomputeBlocking() +{ + PRInt32 writeAudioUpTo = AUDIO_TARGET_MS; + GraphTime endBlockingDecisions = + mCurrentTime + MillisecondsToMediaTime(writeAudioUpTo); + + bool blockingDecisionsWillChange = false; + // mBlockingDecisionsMadeUntilTime has been set in UpdateCurrentTime + while (mBlockingDecisionsMadeUntilTime < endBlockingDecisions) { + LOG(PR_LOG_DEBUG, ("Media graph %p computing blocking for time %f", + this, MediaTimeToSeconds(mBlockingDecisionsMadeUntilTime))); + GraphTime end = GRAPH_TIME_MAX; + RecomputeBlockingAt(mBlockingDecisionsMadeUntilTime, endBlockingDecisions, &end); + LOG(PR_LOG_DEBUG, ("Media graph %p computed blocking for interval %f to %f", + this, MediaTimeToSeconds(mBlockingDecisionsMadeUntilTime), + MediaTimeToSeconds(end))); + mBlockingDecisionsMadeUntilTime = end; + if (end < GRAPH_TIME_MAX) { + blockingDecisionsWillChange = true; + } + } + mBlockingDecisionsMadeUntilTime = endBlockingDecisions; + + for (PRUint32 i = 0; i < mStreams.Length(); ++i) { + MediaStream* stream = mStreams[i]; + GraphTime end; + stream->mBlocked.GetAt(mCurrentTime, &end); + if (end < GRAPH_TIME_MAX) { + blockingDecisionsWillChange = true; + } + } + if (blockingDecisionsWillChange) { + // Make sure we wake up to notify listeners about these changes. + EnsureNextIteration(); + } +} + +void +MediaStreamGraphImpl::RecomputeBlockingAt(GraphTime aTime, + GraphTime aEndBlockingDecisions, + GraphTime* aEnd) +{ + for (PRUint32 i = 0; i < mStreams.Length(); ++i) { + MediaStream* stream = mStreams[i]; + stream->mBlocked.SetAtAndAfter(aTime, false); + } + + for (PRUint32 i = 0; i < mStreams.Length(); ++i) { + MediaStream* stream = mStreams[i]; + // Stream might be blocked by some other stream (due to processing + // constraints) + if (stream->mBlocked.GetAt(aTime)) { + continue; + } + + if (stream->mFinished) { + GraphTime endTime = StreamTimeToGraphTime(stream, stream->GetBufferEnd()); + if (endTime <= aTime) { + LOG(PR_LOG_DEBUG, ("MediaStream %p is blocked due to being finished", stream)); + MarkStreamBlocked(stream, aTime, aEnd); + continue; + } else { + LOG(PR_LOG_DEBUG, ("MediaStream %p is finished, but not blocked yet (end at %f, with blocking at %f)", + stream, MediaTimeToSeconds(stream->GetBufferEnd()), + MediaTimeToSeconds(endTime))); + *aEnd = NS_MIN(*aEnd, endTime); + } + } + + // We don't need to explicitly check for cycles; streams in a cycle will + // just never be able to produce data, and WillUnderrun will trigger. + GraphTime end; + bool explicitBlock = stream->mExplicitBlockerCount.GetAt(aTime, &end) > 0; + *aEnd = NS_MIN(*aEnd, end); + if (explicitBlock) { + LOG(PR_LOG_DEBUG, ("MediaStream %p is blocked due to explicit blocker", stream)); + MarkStreamBlocked(stream, aTime, aEnd); + continue; + } + + bool underrun = WillUnderrun(stream, aTime, aEndBlockingDecisions, aEnd); + if (underrun) { + MarkStreamBlocked(stream, aTime, aEnd); + continue; + } + + if (stream->mAudioOutputs.IsEmpty() && stream->mVideoOutputs.IsEmpty()) { + // See if the stream is being consumed anywhere. If not, it should block. + LOG(PR_LOG_DEBUG, ("MediaStream %p is blocked due to having no consumers", stream)); + MarkStreamBlocked(stream, aTime, aEnd); + continue; + } + } + + NS_ASSERTION(*aEnd > aTime, "Failed to advance!"); +} + +void +MediaStreamGraphImpl::UpdateFirstActiveTracks(MediaStream* aStream) +{ + StreamBuffer::Track* newTracksByType[MediaSegment::TYPE_COUNT]; + for (PRUint32 i = 0; i < ArrayLength(newTracksByType); ++i) { + newTracksByType[i] = nsnull; + } + + for (StreamBuffer::TrackIter iter(aStream->mBuffer); + !iter.IsEnded(); iter.Next()) { + MediaSegment::Type type = iter->GetType(); + if ((newTracksByType[type] && + iter->GetStartTimeRoundDown() < newTracksByType[type]->GetStartTimeRoundDown()) || + aStream->mFirstActiveTracks[type] == TRACK_NONE) { + newTracksByType[type] = &(*iter); + aStream->mFirstActiveTracks[type] = iter->GetID(); + } + } +} + +void +MediaStreamGraphImpl::CreateOrDestroyAudioStream(GraphTime aAudioOutputStartTime, + MediaStream* aStream) +{ + StreamBuffer::Track* track; + + if (aStream->mAudioOutputs.IsEmpty() || + !(track = aStream->mBuffer.FindTrack(aStream->mFirstActiveTracks[MediaSegment::AUDIO]))) { + if (aStream->mAudioOutput) { + aStream->mAudioOutput->Shutdown(); + aStream->mAudioOutput = nsnull; + } + return; + } + + if (aStream->mAudioOutput) + return; + + // No output stream created yet. Check if it's time to create one. + GraphTime startTime = + StreamTimeToGraphTime(aStream, track->GetStartTimeRoundDown(), + INCLUDE_TRAILING_BLOCKED_INTERVAL); + if (startTime >= mBlockingDecisionsMadeUntilTime) { + // The stream wants to play audio, but nothing will play for the forseeable + // future, so don't create the stream. + return; + } + + // Don't bother destroying the nsAudioStream for ended tracks yet. + + // XXX allocating a nsAudioStream could be slow so we're going to have to do + // something here ... preallocation, async allocation, multiplexing onto a single + // stream ... + + AudioSegment* audio = track->Get(); + aStream->mAudioPlaybackStartTime = aAudioOutputStartTime; + aStream->mAudioOutput = nsAudioStream::AllocateStream(); + aStream->mAudioOutput->Init(audio->GetChannels(), + track->GetRate(), + audio->GetFirstFrameFormat()); +} + +void +MediaStreamGraphImpl::PlayAudio(MediaStream* aStream, + GraphTime aFrom, GraphTime aTo) +{ + if (!aStream->mAudioOutput) + return; + + StreamBuffer::Track* track = + aStream->mBuffer.FindTrack(aStream->mFirstActiveTracks[MediaSegment::AUDIO]); + AudioSegment* audio = track->Get(); + + // When we're playing multiple copies of this stream at the same time, they're + // perfectly correlated so adding volumes is the right thing to do. + float volume = 0.0f; + for (PRUint32 i = 0; i < aStream->mAudioOutputs.Length(); ++i) { + volume += aStream->mAudioOutputs[i].mVolume; + } + + // We don't update aStream->mBufferStartTime here to account for + // time spent blocked. Instead, we'll update it in UpdateCurrentTime after the + // blocked period has completed. But we do need to make sure we play from the + // right offsets in the stream buffer, even if we've already written silence for + // some amount of blocked time after the current time. + GraphTime t = aFrom; + while (t < aTo) { + GraphTime end; + bool blocked = aStream->mBlocked.GetAt(t, &end); + end = NS_MIN(end, aTo); + + AudioSegment output; + if (blocked) { + // Track total blocked time in aStream->mBlockedAudioTime so that + // the amount of silent samples we've inserted for blocking never gets + // more than one sample away from the ideal amount. + TrackTicks startTicks = + TimeToTicksRoundDown(track->GetRate(), aStream->mBlockedAudioTime); + aStream->mBlockedAudioTime += end - t; + TrackTicks endTicks = + TimeToTicksRoundDown(track->GetRate(), aStream->mBlockedAudioTime); + + output.InitFrom(*audio); + output.InsertNullDataAtStart(endTicks - startTicks); + LOG(PR_LOG_DEBUG, ("MediaStream %p writing blocking-silence samples for %f to %f", + aStream, MediaTimeToSeconds(t), MediaTimeToSeconds(end))); + } else { + TrackTicks startTicks = + track->TimeToTicksRoundDown(GraphTimeToStreamTime(aStream, t)); + TrackTicks endTicks = + track->TimeToTicksRoundDown(GraphTimeToStreamTime(aStream, end)); + + output.SliceFrom(*audio, startTicks, endTicks); + output.ApplyVolume(volume); + LOG(PR_LOG_DEBUG, ("MediaStream %p writing samples for %f to %f (samples %lld to %lld)", + aStream, MediaTimeToSeconds(t), MediaTimeToSeconds(end), + startTicks, endTicks)); + } + output.WriteTo(aStream->mAudioOutput); + t = end; + } +} + +void +MediaStreamGraphImpl::PlayVideo(MediaStream* aStream) +{ + if (aStream->mVideoOutputs.IsEmpty()) + return; + + StreamBuffer::Track* track = + aStream->mBuffer.FindTrack(aStream->mFirstActiveTracks[MediaSegment::VIDEO]); + if (!track) + return; + VideoSegment* video = track->Get(); + + // Display the next frame a bit early. This is better than letting the current + // frame be displayed for too long. + GraphTime framePosition = mCurrentTime + MEDIA_GRAPH_TARGET_PERIOD_MS; + NS_ASSERTION(framePosition >= aStream->mBufferStartTime, "frame position before buffer?"); + StreamTime frameBufferTime = GraphTimeToStreamTime(aStream, framePosition); + TrackTicks start; + const VideoFrame* frame = + video->GetFrameAt(track->TimeToTicksRoundDown(frameBufferTime), &start); + if (!frame) { + frame = video->GetLastFrame(&start); + if (!frame) + return; + } + + if (*frame != aStream->mLastPlayedVideoFrame) { + LOG(PR_LOG_DEBUG, ("MediaStream %p writing video frame %p (%dx%d)", + aStream, frame->GetImage(), frame->GetIntrinsicSize().width, + frame->GetIntrinsicSize().height)); + GraphTime startTime = StreamTimeToGraphTime(aStream, + track->TicksToTimeRoundDown(start), INCLUDE_TRAILING_BLOCKED_INTERVAL); + TimeStamp targetTime = mCurrentTimeStamp + + TimeDuration::FromMilliseconds(double(startTime - mCurrentTime)); + for (PRUint32 i = 0; i < aStream->mVideoOutputs.Length(); ++i) { + VideoFrameContainer* output = aStream->mVideoOutputs[i]; + output->SetCurrentFrame(frame->GetIntrinsicSize(), frame->GetImage(), + targetTime); + nsCOMPtr event = + NS_NewRunnableMethod(output, &VideoFrameContainer::Invalidate); + NS_DispatchToMainThread(event, NS_DISPATCH_NORMAL); + } + aStream->mLastPlayedVideoFrame = *frame; + } +} + +void +MediaStreamGraphImpl::PrepareUpdatesToMainThreadState() +{ + mMonitor.AssertCurrentThreadOwns(); + + for (PRUint32 i = 0; i < mStreams.Length(); ++i) { + MediaStream* stream = mStreams[i]; + StreamUpdate* update = mStreamUpdates.AppendElement(); + update->mGraphUpdateIndex = stream->mGraphUpdateIndices.GetAt(mCurrentTime); + update->mStream = stream; + update->mNextMainThreadCurrentTime = + GraphTimeToStreamTime(stream, mCurrentTime); + update->mNextMainThreadFinished = + stream->mFinished && + StreamTimeToGraphTime(stream, stream->GetBufferEnd()) <= mCurrentTime; + } + mUpdateRunnables.MoveElementsFrom(mPendingUpdateRunnables); + + EnsureStableStateEventPosted(); +} + +void +MediaStreamGraphImpl::EnsureImmediateWakeUpLocked(MonitorAutoLock& aLock) +{ + if (mWaitState == WAITSTATE_WAITING_FOR_NEXT_ITERATION || + mWaitState == WAITSTATE_WAITING_INDEFINITELY) { + mWaitState = WAITSTATE_WAKING_UP; + aLock.Notify(); + } +} + +void +MediaStreamGraphImpl::EnsureNextIteration() +{ + MonitorAutoLock lock(mMonitor); + EnsureNextIterationLocked(lock); +} + +void +MediaStreamGraphImpl::EnsureNextIterationLocked(MonitorAutoLock& aLock) +{ + if (mNeedAnotherIteration) + return; + mNeedAnotherIteration = true; + if (mWaitState == WAITSTATE_WAITING_INDEFINITELY) { + mWaitState = WAITSTATE_WAKING_UP; + aLock.Notify(); + } +} + +void +MediaStreamGraphImpl::RunThread() +{ + nsTArray messageQueue; + { + MonitorAutoLock lock(mMonitor); + messageQueue.SwapElements(mMessageQueue); + } + NS_ASSERTION(!messageQueue.IsEmpty(), + "Shouldn't have started a graph with empty message queue!"); + + for (;;) { + // Update mCurrentTime to the min of the playing audio times, or using the + // wall-clock time change if no audio is playing. + UpdateCurrentTime(); + + // Calculate independent action times for each batch of messages (each + // batch corresponding to an event loop task). This isolates the performance + // of different scripts to some extent. + for (PRUint32 i = 0; i < messageQueue.Length(); ++i) { + mProcessingGraphUpdateIndex = messageQueue[i].mGraphUpdateIndex; + nsTArray >& messages = messageQueue[i].mMessages; + + for (PRUint32 j = 0; j < mStreams.Length(); ++j) { + mStreams[j]->mMessageAffectedTime = GRAPH_TIME_MAX; + } + for (PRUint32 j = 0; j < messages.Length(); ++j) { + messages[j]->UpdateAffectedStream(); + } + + ChooseActionTime(); + + for (PRUint32 j = 0; j < messages.Length(); ++j) { + messages[j]->Process(); + } + } + messageQueue.Clear(); + + GraphTime prevBlockingDecisionsMadeUntilTime = mBlockingDecisionsMadeUntilTime; + RecomputeBlocking(); + + PRUint32 audioStreamsActive = 0; + bool allBlockedForever = true; + // Figure out what each stream wants to do + for (PRUint32 i = 0; i < mStreams.Length(); ++i) { + MediaStream* stream = mStreams[i]; + UpdateFirstActiveTracks(stream); + CreateOrDestroyAudioStream(prevBlockingDecisionsMadeUntilTime, stream); + PlayAudio(stream, prevBlockingDecisionsMadeUntilTime, + mBlockingDecisionsMadeUntilTime); + if (stream->mAudioOutput) { + ++audioStreamsActive; + } + PlayVideo(stream); + GraphTime end; + if (!stream->mBlocked.GetAt(mCurrentTime, &end) || end < GRAPH_TIME_MAX) { + allBlockedForever = false; + } + } + if (!allBlockedForever || audioStreamsActive > 0) { + EnsureNextIteration(); + } + + { + MonitorAutoLock lock(mMonitor); + PrepareUpdatesToMainThreadState(); + if (mForceShutDown || (IsEmpty() && mMessageQueue.IsEmpty())) { + // Enter shutdown mode. The stable-state handler will detect this + // and complete shutdown. Destroy any streams immediately. + for (PRUint32 i = 0; i < mStreams.Length(); ++i) { + mStreams[i]->DestroyImpl(); + } + LOG(PR_LOG_DEBUG, ("MediaStreamGraph %p waiting for main thread cleanup", this)); + mLifecycleState = LIFECYCLE_WAITING_FOR_MAIN_THREAD_CLEANUP; + return; + } + + PRIntervalTime timeout = PR_INTERVAL_NO_TIMEOUT; + TimeStamp now = TimeStamp::Now(); + if (mNeedAnotherIteration) { + PRInt64 timeoutMS = MEDIA_GRAPH_TARGET_PERIOD_MS - + PRInt64((now - mCurrentTimeStamp).ToMilliseconds()); + // Make sure timeoutMS doesn't overflow 32 bits by waking up at + // least once a minute, if we need to wake up at all + timeoutMS = NS_MAX(0, NS_MIN(timeoutMS, 60*1000)); + timeout = PR_MillisecondsToInterval(PRUint32(timeoutMS)); + LOG(PR_LOG_DEBUG, ("Waiting for next iteration; at %f, timeout=%f", + (now - mInitialTimeStamp).ToSeconds(), timeoutMS/1000.0)); + mWaitState = WAITSTATE_WAITING_FOR_NEXT_ITERATION; + } else { + mWaitState = WAITSTATE_WAITING_INDEFINITELY; + } + if (timeout > 0) { + lock.Wait(timeout); + LOG(PR_LOG_DEBUG, ("Resuming after timeout; at %f, elapsed=%f", + (TimeStamp::Now() - mInitialTimeStamp).ToSeconds(), + (TimeStamp::Now() - now).ToSeconds())); + } + mWaitState = WAITSTATE_RUNNING; + mNeedAnotherIteration = false; + messageQueue.SwapElements(mMessageQueue); + } + } +} + +void +MediaStreamGraphImpl::ApplyStreamUpdate(StreamUpdate* aUpdate) +{ + mMonitor.AssertCurrentThreadOwns(); + + MediaStream* stream = aUpdate->mStream; + if (!stream) + return; + stream->mMainThreadCurrentTime = aUpdate->mNextMainThreadCurrentTime; + stream->mMainThreadFinished = aUpdate->mNextMainThreadFinished; +} + +void +MediaStreamGraphImpl::ShutdownThreads() +{ + NS_ASSERTION(NS_IsMainThread(), "Must be called on main thread"); + // mGraph's thread is not running so it's OK to do whatever here + LOG(PR_LOG_DEBUG, ("Stopping threads for MediaStreamGraph %p", this)); + + if (mThread) { + mThread->Shutdown(); + mThread = nsnull; + } +} + +void +MediaStreamGraphImpl::ForceShutDown() +{ + NS_ASSERTION(NS_IsMainThread(), "Must be called on main thread"); + LOG(PR_LOG_DEBUG, ("MediaStreamGraph %p ForceShutdown", this)); + { + MonitorAutoLock lock(mMonitor); + mForceShutDown = true; + EnsureImmediateWakeUpLocked(lock); + } +} + +namespace { + +class MediaStreamGraphThreadRunnable : public nsRunnable { +public: + NS_IMETHOD Run() + { + gGraph->RunThread(); + return NS_OK; + } +}; + +class MediaStreamGraphShutDownRunnable : public nsRunnable { +public: + MediaStreamGraphShutDownRunnable(MediaStreamGraphImpl* aGraph) : mGraph(aGraph) {} + NS_IMETHOD Run() + { + NS_ASSERTION(mGraph->mDetectedNotRunning, + "We should know the graph thread control loop isn't running!"); + // mGraph's thread is not running so it's OK to do whatever here + if (mGraph->IsEmpty()) { + // mGraph is no longer needed, so delete it. If the graph is not empty + // then we must be in a forced shutdown and some later AppendMessage will + // detect that the manager has been emptied, and delete it. + delete mGraph; + } else { + NS_ASSERTION(mGraph->mForceShutDown, "Not in forced shutdown?"); + mGraph->mLifecycleState = + MediaStreamGraphImpl::LIFECYCLE_WAITING_FOR_STREAM_DESTRUCTION; + } + return NS_OK; + } +private: + MediaStreamGraphImpl* mGraph; +}; + +class MediaStreamGraphStableStateRunnable : public nsRunnable { +public: + NS_IMETHOD Run() + { + if (gGraph) { + gGraph->RunInStableState(); + } + return NS_OK; + } +}; + +/* + * Control messages forwarded from main thread to graph manager thread + */ +class CreateMessage : public ControlMessage { +public: + CreateMessage(MediaStream* aStream) : ControlMessage(aStream) {} + virtual void UpdateAffectedStream() + { + mStream->GraphImpl()->AddStream(mStream); + } + virtual void Process() + { + mStream->Init(); + } +}; + +class MediaStreamGraphShutdownObserver : public nsIObserver +{ +public: + NS_DECL_ISUPPORTS + NS_DECL_NSIOBSERVER +}; + +} + +void +MediaStreamGraphImpl::RunInStableState() +{ + NS_ASSERTION(NS_IsMainThread(), "Must be called on main thread"); + + nsTArray > runnables; + + { + MonitorAutoLock lock(mMonitor); + mPostedRunInStableStateEvent = false; + + runnables.SwapElements(mUpdateRunnables); + for (PRUint32 i = 0; i < mStreamUpdates.Length(); ++i) { + StreamUpdate* update = &mStreamUpdates[i]; + if (update->mStream) { + ApplyStreamUpdate(update); + } + } + mStreamUpdates.Clear(); + + if (mLifecycleState == LIFECYCLE_WAITING_FOR_MAIN_THREAD_CLEANUP && mForceShutDown) { + for (PRUint32 i = 0; i < mMessageQueue.Length(); ++i) { + MessageBlock& mb = mMessageQueue[i]; + for (PRUint32 j = 0; j < mb.mMessages.Length(); ++j) { + mb.mMessages[j]->ProcessDuringShutdown(); + } + } + mMessageQueue.Clear(); + for (PRUint32 i = 0; i < mCurrentTaskMessageQueue.Length(); ++i) { + mCurrentTaskMessageQueue[i]->ProcessDuringShutdown(); + } + mCurrentTaskMessageQueue.Clear(); + // Stop MediaStreamGraph threads. Do not clear gGraph since + // we have outstanding DOM objects that may need it. + mLifecycleState = LIFECYCLE_WAITING_FOR_THREAD_SHUTDOWN; + nsCOMPtr event = new MediaStreamGraphShutDownRunnable(this); + NS_DispatchToMainThread(event); + } + + if (mLifecycleState == LIFECYCLE_THREAD_NOT_STARTED) { + mLifecycleState = LIFECYCLE_RUNNING; + // Start the thread now. We couldn't start it earlier because + // the graph might exit immediately on finding it has no streams. The + // first message for a new graph must create a stream. + nsCOMPtr event = new MediaStreamGraphThreadRunnable(); + NS_NewThread(getter_AddRefs(mThread), event); + } + + if (mCurrentTaskMessageQueue.IsEmpty()) { + if (mLifecycleState == LIFECYCLE_WAITING_FOR_MAIN_THREAD_CLEANUP && IsEmpty()) { + NS_ASSERTION(gGraph == this, "Not current graph??"); + // Complete shutdown. First, ensure that this graph is no longer used. + // A new graph graph will be created if one is needed. + LOG(PR_LOG_DEBUG, ("Disconnecting MediaStreamGraph %p", gGraph)); + gGraph = nsnull; + // Asynchronously clean up old graph. We don't want to do this + // synchronously because it spins the event loop waiting for threads + // to shut down, and we don't want to do that in a stable state handler. + mLifecycleState = LIFECYCLE_WAITING_FOR_THREAD_SHUTDOWN; + nsCOMPtr event = new MediaStreamGraphShutDownRunnable(this); + NS_DispatchToMainThread(event); + } + } else { + if (mLifecycleState <= LIFECYCLE_WAITING_FOR_MAIN_THREAD_CLEANUP) { + MessageBlock* block = mMessageQueue.AppendElement(); + block->mMessages.SwapElements(mCurrentTaskMessageQueue); + block->mGraphUpdateIndex = mGraphUpdatesSent; + ++mGraphUpdatesSent; + EnsureNextIterationLocked(lock); + } + + if (mLifecycleState == LIFECYCLE_WAITING_FOR_MAIN_THREAD_CLEANUP) { + mLifecycleState = LIFECYCLE_RUNNING; + // Revive the MediaStreamGraph since we have more messages going to it. + // Note that we need to put messages into its queue before reviving it, + // or it might exit immediately. + nsCOMPtr event = new MediaStreamGraphThreadRunnable(); + mThread->Dispatch(event, 0); + } + } + + mDetectedNotRunning = mLifecycleState > LIFECYCLE_RUNNING; + } + + // Make sure we get a new current time in the next event loop task + mPostedRunInStableState = false; + + for (PRUint32 i = 0; i < runnables.Length(); ++i) { + runnables[i]->Run(); + } +} + +static NS_DEFINE_CID(kAppShellCID, NS_APPSHELL_CID); + +void +MediaStreamGraphImpl::EnsureRunInStableState() +{ + NS_ASSERTION(NS_IsMainThread(), "main thread only"); + + if (mPostedRunInStableState) + return; + mPostedRunInStableState = true; + nsCOMPtr event = new MediaStreamGraphStableStateRunnable(); + nsCOMPtr appShell = do_GetService(kAppShellCID); + if (appShell) { + appShell->RunInStableState(event); + } else { + NS_ERROR("Appshell already destroyed?"); + } +} + +void +MediaStreamGraphImpl::EnsureStableStateEventPosted() +{ + mMonitor.AssertCurrentThreadOwns(); + + if (mPostedRunInStableStateEvent) + return; + mPostedRunInStableStateEvent = true; + nsCOMPtr event = new MediaStreamGraphStableStateRunnable(); + NS_DispatchToMainThread(event); +} + +void +MediaStreamGraphImpl::AppendMessage(ControlMessage* aMessage) +{ + NS_ASSERTION(NS_IsMainThread(), "main thread only"); + + if (mDetectedNotRunning && + mLifecycleState > LIFECYCLE_WAITING_FOR_MAIN_THREAD_CLEANUP) { + // The graph control loop is not running and main thread cleanup has + // happened. From now on we can't append messages to mCurrentTaskMessageQueue, + // because that will never be processed again, so just ProcessDuringShutdown + // this message. + // This should only happen during forced shutdown. + aMessage->ProcessDuringShutdown(); + delete aMessage; + if (IsEmpty()) { + NS_ASSERTION(gGraph == this, "Switched managers during forced shutdown?"); + gGraph = nsnull; + delete this; + } + return; + } + + mCurrentTaskMessageQueue.AppendElement(aMessage); + EnsureRunInStableState(); +} + +void +MediaStream::Init() +{ + MediaStreamGraphImpl* graph = GraphImpl(); + mBlocked.SetAtAndAfter(graph->mCurrentTime, true); + mExplicitBlockerCount.SetAtAndAfter(graph->mCurrentTime, true); + mExplicitBlockerCount.SetAtAndAfter(graph->mLastActionTime, false); +} + +MediaStreamGraphImpl* +MediaStream::GraphImpl() +{ + return gGraph; +} + +void +MediaStream::DestroyImpl() +{ + if (mAudioOutput) { + mAudioOutput->Shutdown(); + mAudioOutput = nsnull; + } +} + +void +MediaStream::Destroy() +{ + class Message : public ControlMessage { + public: + Message(MediaStream* aStream) : ControlMessage(aStream) {} + virtual void UpdateAffectedStream() + { + mStream->DestroyImpl(); + mStream->GraphImpl()->RemoveStream(mStream); + } + virtual void ProcessDuringShutdown() + { UpdateAffectedStream(); } + }; + mWrapper = nsnull; + GraphImpl()->AppendMessage(new Message(this)); +} + +void +MediaStream::AddAudioOutput(void* aKey) +{ + class Message : public ControlMessage { + public: + Message(MediaStream* aStream, void* aKey) : ControlMessage(aStream), mKey(aKey) {} + virtual void UpdateAffectedStream() + { + mStream->AddAudioOutputImpl(mKey); + } + void* mKey; + }; + GraphImpl()->AppendMessage(new Message(this, aKey)); +} + +void +MediaStream::SetAudioOutputVolumeImpl(void* aKey, float aVolume) +{ + for (PRUint32 i = 0; i < mAudioOutputs.Length(); ++i) { + if (mAudioOutputs[i].mKey == aKey) { + mAudioOutputs[i].mVolume = aVolume; + return; + } + } + NS_ERROR("Audio output key not found"); +} + +void +MediaStream::SetAudioOutputVolume(void* aKey, float aVolume) +{ + class Message : public ControlMessage { + public: + Message(MediaStream* aStream, void* aKey, float aVolume) : + ControlMessage(aStream), mKey(aKey), mVolume(aVolume) {} + virtual void UpdateAffectedStream() + { + mStream->SetAudioOutputVolumeImpl(mKey, mVolume); + } + void* mKey; + float mVolume; + }; + GraphImpl()->AppendMessage(new Message(this, aKey, aVolume)); +} + +void +MediaStream::RemoveAudioOutputImpl(void* aKey) +{ + for (PRUint32 i = 0; i < mAudioOutputs.Length(); ++i) { + if (mAudioOutputs[i].mKey == aKey) { + mAudioOutputs.RemoveElementAt(i); + return; + } + } + NS_ERROR("Audio output key not found"); +} + +void +MediaStream::RemoveAudioOutput(void* aKey) +{ + class Message : public ControlMessage { + public: + Message(MediaStream* aStream, void* aKey) : + ControlMessage(aStream), mKey(aKey) {} + virtual void UpdateAffectedStream() + { + mStream->RemoveAudioOutputImpl(mKey); + } + void* mKey; + }; + GraphImpl()->AppendMessage(new Message(this, aKey)); +} + +void +MediaStream::AddVideoOutput(VideoFrameContainer* aContainer) +{ + class Message : public ControlMessage { + public: + Message(MediaStream* aStream, VideoFrameContainer* aContainer) : + ControlMessage(aStream), mContainer(aContainer) {} + virtual void UpdateAffectedStream() + { + mStream->AddVideoOutputImpl(mContainer.forget()); + } + nsRefPtr mContainer; + }; + GraphImpl()->AppendMessage(new Message(this, aContainer)); +} + +void +MediaStream::RemoveVideoOutput(VideoFrameContainer* aContainer) +{ + class Message : public ControlMessage { + public: + Message(MediaStream* aStream, VideoFrameContainer* aContainer) : + ControlMessage(aStream), mContainer(aContainer) {} + virtual void UpdateAffectedStream() + { + mStream->RemoveVideoOutputImpl(mContainer); + } + nsRefPtr mContainer; + }; + GraphImpl()->AppendMessage(new Message(this, aContainer)); +} + +void +MediaStream::ChangeExplicitBlockerCount(PRInt32 aDelta) +{ + class Message : public ControlMessage { + public: + Message(MediaStream* aStream, PRInt32 aDelta) : + ControlMessage(aStream), mDelta(aDelta) {} + virtual void UpdateAffectedStream() + { + mStream->ChangeExplicitBlockerCountImpl( + mStream->GraphImpl()->mLastActionTime, mDelta); + } + PRInt32 mDelta; + }; + GraphImpl()->AppendMessage(new Message(this, aDelta)); +} + +void +MediaStream::AddListener(MediaStreamListener* aListener) +{ + class Message : public ControlMessage { + public: + Message(MediaStream* aStream, MediaStreamListener* aListener) : + ControlMessage(aStream), mListener(aListener) {} + virtual void UpdateAffectedStream() + { + mStream->AddListenerImpl(mListener.forget()); + } + nsRefPtr mListener; + }; + GraphImpl()->AppendMessage(new Message(this, aListener)); +} + +void +MediaStream::RemoveListener(MediaStreamListener* aListener) +{ + class Message : public ControlMessage { + public: + Message(MediaStream* aStream, MediaStreamListener* aListener) : + ControlMessage(aStream), mListener(aListener) {} + virtual void UpdateAffectedStream() + { + mStream->RemoveListenerImpl(mListener); + } + nsRefPtr mListener; + }; + GraphImpl()->AppendMessage(new Message(this, aListener)); +} + +static const PRUint32 kThreadLimit = 4; +static const PRUint32 kIdleThreadLimit = 4; +static const PRUint32 kIdleThreadTimeoutMs = 2000; + +MediaStreamGraphImpl::MediaStreamGraphImpl() + : mLastActionTime(1) + , mCurrentTime(1) + , mBlockingDecisionsMadeUntilTime(1) + , mProcessingGraphUpdateIndex(0) + , mMonitor("MediaStreamGraphImpl") + , mLifecycleState(LIFECYCLE_THREAD_NOT_STARTED) + , mWaitState(WAITSTATE_RUNNING) + , mNeedAnotherIteration(false) + , mForceShutDown(false) + , mPostedRunInStableStateEvent(false) + , mDetectedNotRunning(false) + , mPostedRunInStableState(false) +{ +#ifdef PR_LOGGING + if (!gMediaStreamGraphLog) { + gMediaStreamGraphLog = PR_NewLogModule("MediaStreamGraph"); + } +#endif + + mCurrentTimeStamp = mInitialTimeStamp = TimeStamp::Now(); +} + +NS_IMPL_ISUPPORTS1(MediaStreamGraphShutdownObserver, nsIObserver) + +static bool gShutdownObserverRegistered = false; + +NS_IMETHODIMP +MediaStreamGraphShutdownObserver::Observe(nsISupports *aSubject, + const char *aTopic, + const PRUnichar *aData) +{ + if (strcmp(aTopic, NS_XPCOM_SHUTDOWN_OBSERVER_ID) == 0) { + if (gGraph) { + gGraph->ForceShutDown(); + } + nsContentUtils::UnregisterShutdownObserver(this); + gShutdownObserverRegistered = false; + } + return NS_OK; +} + +MediaStreamGraph* +MediaStreamGraph::GetInstance() +{ + NS_ASSERTION(NS_IsMainThread(), "Main thread only"); + + if (!gGraph) { + if (!gShutdownObserverRegistered) { + gShutdownObserverRegistered = true; + nsContentUtils::RegisterShutdownObserver(new MediaStreamGraphShutdownObserver()); + } + + gGraph = new MediaStreamGraphImpl(); + LOG(PR_LOG_DEBUG, ("Starting up MediaStreamGraph %p", gGraph)); + } + + return gGraph; +} + +} diff --git a/content/media/MediaStreamGraph.h b/content/media/MediaStreamGraph.h new file mode 100644 index 00000000000..5179f1d0db9 --- /dev/null +++ b/content/media/MediaStreamGraph.h @@ -0,0 +1,394 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*-*/ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef MOZILLA_MEDIASTREAMGRAPH_H_ +#define MOZILLA_MEDIASTREAMGRAPH_H_ + +#include "mozilla/Mutex.h" +#include "nsAudioStream.h" +#include "nsTArray.h" +#include "nsIRunnable.h" +#include "nsISupportsImpl.h" +#include "StreamBuffer.h" +#include "TimeVarying.h" +#include "VideoFrameContainer.h" +#include "VideoSegment.h" + +class nsDOMMediaStream; + +namespace mozilla { + +/** + * Microseconds relative to the start of the graph timeline. + */ +typedef PRInt64 GraphTime; +const GraphTime GRAPH_TIME_MAX = MEDIA_TIME_MAX; + +/* + * MediaStreamGraph is a framework for synchronized audio/video processing + * and playback. It is designed to be used by other browser components such as + * HTML media elements, media capture APIs, real-time media streaming APIs, + * multitrack media APIs, and advanced audio APIs. + * + * The MediaStreamGraph uses a dedicated thread to process media --- the media + * graph thread. This ensures that we can process media through the graph + * without blocking on main-thread activity. The media graph is only modified + * on the media graph thread, to ensure graph changes can be processed without + * interfering with media processing. All interaction with the media graph + * thread is done with message passing. + * + * APIs that modify the graph or its properties are described as "control APIs". + * These APIs are asynchronous; they queue graph changes internally and + * those changes are processed all-at-once by the MediaStreamGraph. The + * MediaStreamGraph monitors the main thread event loop via nsIAppShell::RunInStableState + * to ensure that graph changes from a single event loop task are always + * processed all together. Control APIs should only be used on the main thread, + * currently; we may be able to relax that later. + * + * To allow precise synchronization of times in the control API, the + * MediaStreamGraph maintains a "media timeline". Control APIs that take or + * return times use that timeline. Those times never advance during + * an event loop task. This time is returned by MediaStreamGraph::GetCurrentTime(). + * + * Media decoding, audio processing and media playback use thread-safe APIs to + * the media graph to ensure they can continue while the main thread is blocked. + * + * When the graph is changed, we may need to throw out buffered data and + * reprocess it. This is triggered automatically by the MediaStreamGraph. + * + * Streams that use different sampling rates complicate things a lot. We + * considered forcing all streams to have the same audio sample rate, resampling + * at inputs and outputs only, but that would create situations where a stream + * is resampled from X to Y and then back to X unnecessarily. It seems easier + * to just live with streams having different sample rates. We do require that + * the sample rate for a stream be constant for the life of a stream. + * + * XXX does not yet support blockInput/blockOutput functionality. + */ + +class MediaStreamGraph; + +/** + * This is a base class for listener callbacks. Override methods to be + * notified of audio or video data or changes in stream state. + * + * This can be used by stream recorders or network connections that receive + * stream input. It could also be used for debugging. + * + * All notification methods are called from the media graph thread. Overriders + * of these methods are responsible for all synchronization. Beware! + * These methods are called without the media graph monitor held, so + * reentry into media graph methods is possible, although very much discouraged! + * You should do something non-blocking and non-reentrant (e.g. dispatch an + * event to some thread) and return. + */ +class MediaStreamListener { +public: + virtual ~MediaStreamListener() {} + + NS_INLINE_DECL_THREADSAFE_REFCOUNTING(MediaStreamListener) + + enum Blocking { + BLOCKED, + UNBLOCKED + }; + /** + * Notify that the blocking status of the stream changed. + */ + virtual void NotifyBlockingChanged(MediaStreamGraph* aGraph, Blocking aBlocked) {} + + /** + * Notify that the stream output is advancing. + */ + virtual void NotifyOutput(MediaStreamGraph* aGraph) {} + + /** + * Notify that the stream finished. + */ + virtual void NotifyFinished(MediaStreamGraph* aGraph) {} +}; + +class MediaStreamGraphImpl; + +/** + * A stream of synchronized audio and video data. All (not blocked) streams + * progress at the same rate --- "real time". Streams cannot seek. The only + * operation readers can perform on a stream is to read the next data. + * + * Consumers of a stream can be reading from it at different offsets, but that + * should only happen due to the order in which consumers are being run. + * Those offsets must not diverge in the long term, otherwise we would require + * unbounded buffering. + * + * Streams can be in a "blocked" state. While blocked, a stream does not + * produce data. A stream can be explicitly blocked via the control API, + * or implicitly blocked by whatever's generating it (e.g. an underrun in the + * source resource), or implicitly blocked because something consuming it + * blocks, or implicitly because it has finished. + * + * A stream can be in a "finished" state. "Finished" streams are permanently + * blocked. + * + * Transitions into and out of the "blocked" and "finished" states are managed + * by the MediaStreamGraph on the media graph thread. + * + * We buffer media data ahead of the consumers' reading offsets. It is possible + * to have buffered data but still be blocked. + * + * Any stream can have its audio and video playing when requested. The media + * stream graph plays audio by constructing audio output streams as necessary. + * Video is played by setting video frames into an VideoFrameContainer at the right + * time. To ensure video plays in sync with audio, make sure that the same + * stream is playing both the audio and video. + * + * The data in a stream is managed by StreamBuffer. It consists of a set of + * tracks of various types that can start and end over time. + * + * Streams are explicitly managed. The client creates them via + * MediaStreamGraph::CreateInput/ProcessedMediaStream, and releases them by calling + * Destroy() when no longer needed (actual destruction will be deferred). + * The actual object is owned by the MediaStreamGraph. The basic idea is that + * main thread objects will keep Streams alive as long as necessary (using the + * cycle collector to clean up whenever needed). + * + * We make them refcounted only so that stream-related messages with MediaStream* + * pointers can be sent to the main thread safely. + */ +class MediaStream { +public: + NS_INLINE_DECL_THREADSAFE_REFCOUNTING(MediaStream) + + MediaStream(nsDOMMediaStream* aWrapper) + : mBufferStartTime(0) + , mExplicitBlockerCount(0) + , mBlocked(false) + , mGraphUpdateIndices(0) + , mFinished(false) + , mNotifiedFinished(false) + , mAudioPlaybackStartTime(0) + , mBlockedAudioTime(0) + , mMessageAffectedTime(0) + , mWrapper(aWrapper) + , mMainThreadCurrentTime(0) + , mMainThreadFinished(false) + { + for (PRUint32 i = 0; i < ArrayLength(mFirstActiveTracks); ++i) { + mFirstActiveTracks[i] = TRACK_NONE; + } + } + virtual ~MediaStream() {} + + /** + * Returns the graph that owns this stream. + */ + MediaStreamGraphImpl* GraphImpl(); + + // Control API. + // Since a stream can be played multiple ways, we need to combine independent + // volume settings. The aKey parameter is used to keep volume settings + // separate. Since the stream is always playing the same contents, only + // a single audio output stream is used; the volumes are combined. + // Currently only the first enabled audio track is played. + // XXX change this so all enabled audio tracks are mixed and played. + void AddAudioOutput(void* aKey); + void SetAudioOutputVolume(void* aKey, float aVolume); + void RemoveAudioOutput(void* aKey); + // Since a stream can be played multiple ways, we need to be able to + // play to multiple VideoFrameContainers. + // Only the first enabled video track is played. + void AddVideoOutput(VideoFrameContainer* aContainer); + void RemoveVideoOutput(VideoFrameContainer* aContainer); + // Explicitly block. Useful for example if a media element is pausing + // and we need to stop its stream emitting its buffered data. + void ChangeExplicitBlockerCount(PRInt32 aDelta); + // Events will be dispatched by calling methods of aListener. + void AddListener(MediaStreamListener* aListener); + void RemoveListener(MediaStreamListener* aListener); + // Signal that the client is done with this MediaStream. It will be deleted later. + void Destroy(); + // Returns the main-thread's view of how much data has been processed by + // this stream. + StreamTime GetCurrentTime() { return mMainThreadCurrentTime; } + // Return the main thread's view of whether this stream has finished. + bool IsFinished() { return mMainThreadFinished; } + + friend class MediaStreamGraphImpl; + + // media graph thread only + void Init(); + // These Impl methods perform the core functionality of the control methods + // above, on the media graph thread. + /** + * Stop all stream activity and disconnect it from all inputs and outputs. + * This must be idempotent. + */ + virtual void DestroyImpl(); + StreamTime GetBufferEnd() { return mBuffer.GetEnd(); } + void SetAudioOutputVolumeImpl(void* aKey, float aVolume); + void AddAudioOutputImpl(void* aKey) + { + mAudioOutputs.AppendElement(AudioOutput(aKey)); + } + void RemoveAudioOutputImpl(void* aKey); + void AddVideoOutputImpl(already_AddRefed aContainer) + { + *mVideoOutputs.AppendElement() = aContainer; + } + void RemoveVideoOutputImpl(VideoFrameContainer* aContainer) + { + mVideoOutputs.RemoveElement(aContainer); + } + void ChangeExplicitBlockerCountImpl(StreamTime aTime, PRInt32 aDelta) + { + mExplicitBlockerCount.SetAtAndAfter(aTime, mExplicitBlockerCount.GetAt(aTime) + aDelta); + } + void AddListenerImpl(already_AddRefed aListener) + { + *mListeners.AppendElement() = aListener; + } + void RemoveListenerImpl(MediaStreamListener* aListener) + { + mListeners.RemoveElement(aListener); + } + +#ifdef DEBUG + const StreamBuffer& GetStreamBuffer() { return mBuffer; } +#endif + +protected: + virtual void AdvanceTimeVaryingValuesToCurrentTime(GraphTime aCurrentTime, GraphTime aBlockedTime) + { + mBufferStartTime += aBlockedTime; + mGraphUpdateIndices.InsertTimeAtStart(aBlockedTime); + mGraphUpdateIndices.AdvanceCurrentTime(aCurrentTime); + mExplicitBlockerCount.AdvanceCurrentTime(aCurrentTime); + + mBuffer.ForgetUpTo(aCurrentTime - mBufferStartTime); + } + + // This state is all initialized on the main thread but + // otherwise modified only on the media graph thread. + + // Buffered data. The start of the buffer corresponds to mBufferStartTime. + // Conceptually the buffer contains everything this stream has ever played, + // but we forget some prefix of the buffered data to bound the space usage. + StreamBuffer mBuffer; + // The time when the buffered data could be considered to have started playing. + // This increases over time to account for time the stream was blocked before + // mCurrentTime. + GraphTime mBufferStartTime; + + // Client-set volume of this stream + struct AudioOutput { + AudioOutput(void* aKey) : mKey(aKey), mVolume(1.0f) {} + void* mKey; + float mVolume; + }; + nsTArray mAudioOutputs; + nsTArray > mVideoOutputs; + // We record the last played video frame to avoid redundant setting + // of the current video frame. + VideoFrame mLastPlayedVideoFrame; + // The number of times this stream has been explicitly blocked by the control + // API, minus the number of times it has been explicitly unblocked. + TimeVarying mExplicitBlockerCount; + nsTArray > mListeners; + + // Precomputed blocking status (over GraphTime). + // This is only valid between the graph's mCurrentTime and + // mBlockingDecisionsMadeUntilTime. The stream is considered to have + // not been blocked before mCurrentTime (its mBufferStartTime is increased + // as necessary to account for that time instead) --- this avoids us having to + // record the entire history of the stream's blocking-ness in mBlocked. + TimeVarying mBlocked; + // Maps graph time to the graph update that affected this stream at that time + TimeVarying mGraphUpdateIndices; + + /** + * When true, this means the stream will be finished once all + * buffered data has been consumed. + */ + bool mFinished; + /** + * When true, mFinished is true and we've played all the data in this stream + * and fired NotifyFinished notifications. + */ + bool mNotifiedFinished; + + // Where audio output is going + nsRefPtr mAudioOutput; + // When we started audio playback for this stream. + // Add mAudioOutput->GetPosition() to find the current audio playback position. + GraphTime mAudioPlaybackStartTime; + // Amount of time that we've wanted to play silence because of the stream + // blocking. + MediaTime mBlockedAudioTime; + + // For each track type, this is the first active track found for that type. + // The first active track is the track that started earliest; if multiple + // tracks start at the same time, the one with the lowest ID. + TrackID mFirstActiveTracks[MediaSegment::TYPE_COUNT]; + + // Temporary data used by MediaStreamGraph on the graph thread + // The earliest time for which we would like to change this stream's output. + GraphTime mMessageAffectedTime; + + // This state is only used on the main thread. + nsDOMMediaStream* mWrapper; + // Main-thread views of state + StreamTime mMainThreadCurrentTime; + bool mMainThreadFinished; +}; + +/** + * Initially, at least, we will have a singleton MediaStreamGraph per + * process. + */ +class MediaStreamGraph { +public: + // Main thread only + static MediaStreamGraph* GetInstance(); + // Control API. + /** + * Returns the number of graph updates sent. This can be used to track + * whether a given update has been processed by the graph thread and reflected + * in main-thread stream state. + */ + PRInt64 GetCurrentGraphUpdateIndex() { return mGraphUpdatesSent; } + + /** + * Media graph thread only. + * Dispatches a runnable that will run on the main thread after all + * main-thread stream state has been next updated. + * Should only be called during MediaStreamListener callbacks. + */ + void DispatchToMainThreadAfterStreamStateUpdate(nsIRunnable* aRunnable) + { + mPendingUpdateRunnables.AppendElement(aRunnable); + } + +protected: + MediaStreamGraph() + : mGraphUpdatesSent(1) + { + MOZ_COUNT_CTOR(MediaStreamGraph); + } + ~MediaStreamGraph() + { + MOZ_COUNT_DTOR(MediaStreamGraph); + } + + // Media graph thread only + nsTArray > mPendingUpdateRunnables; + + // Main thread only + // The number of updates we have sent to the media graph thread. We start + // this at 1 just to ensure that 0 is usable as a special value. + PRInt64 mGraphUpdatesSent; +}; + +} + +#endif /* MOZILLA_MEDIASTREAMGRAPH_H_ */ From 00a409b20fe763bb7b974209f6be61dbb61bc3c4 Mon Sep 17 00:00:00 2001 From: Robert O'Callahan Date: Mon, 30 Apr 2012 15:11:34 +1200 Subject: [PATCH 010/162] Bug 664918. Part 4: Create nsDOMMediaStream, a DOM object wrapping an underlying MediaStream. r=smaug,jesup --- b2g/installer/package-manifest.in | 1 + browser/installer/package-manifest.in | 1 + content/media/Makefile.in | 9 ++++ content/media/nsDOMMediaStream.cpp | 49 +++++++++++++++++ content/media/nsDOMMediaStream.h | 55 ++++++++++++++++++++ content/media/nsIDOMMediaStream.idl | 12 +++++ dom/base/nsDOMClassInfo.cpp | 7 +++ dom/base/nsDOMClassInfoClasses.h | 3 ++ mobile/android/installer/package-manifest.in | 1 + mobile/xul/installer/package-manifest.in | 1 + 10 files changed, 139 insertions(+) create mode 100644 content/media/nsDOMMediaStream.cpp create mode 100644 content/media/nsDOMMediaStream.h create mode 100644 content/media/nsIDOMMediaStream.idl diff --git a/b2g/installer/package-manifest.in b/b2g/installer/package-manifest.in index 70befef465b..41143b1e652 100644 --- a/b2g/installer/package-manifest.in +++ b/b2g/installer/package-manifest.in @@ -138,6 +138,7 @@ @BINPATH@/components/content_canvas.xpt @BINPATH@/components/content_htmldoc.xpt @BINPATH@/components/content_html.xpt +@BINPATH@/components/content_media.xpt @BINPATH@/components/content_xslt.xpt @BINPATH@/components/content_xtf.xpt @BINPATH@/components/cookie.xpt diff --git a/browser/installer/package-manifest.in b/browser/installer/package-manifest.in index cef9d6ee054..24329988630 100644 --- a/browser/installer/package-manifest.in +++ b/browser/installer/package-manifest.in @@ -142,6 +142,7 @@ @BINPATH@/components/content_canvas.xpt @BINPATH@/components/content_htmldoc.xpt @BINPATH@/components/content_html.xpt +@BINPATH@/components/content_media.xpt @BINPATH@/components/content_xslt.xpt @BINPATH@/components/content_xtf.xpt @BINPATH@/components/cookie.xpt diff --git a/content/media/Makefile.in b/content/media/Makefile.in index e1b217e4e39..63f974a0304 100644 --- a/content/media/Makefile.in +++ b/content/media/Makefile.in @@ -44,6 +44,11 @@ include $(DEPTH)/config/autoconf.mk MODULE = content LIBRARY_NAME = gkconmedia_s LIBXUL_LIBRARY = 1 +XPIDL_MODULE = content_media + +XPIDLSRCS = \ + nsIDOMMediaStream.idl \ + $(NULL) EXPORTS = \ AudioSegment.h \ @@ -55,6 +60,7 @@ EXPORTS = \ nsBuiltinDecoder.h \ nsBuiltinDecoderStateMachine.h \ nsBuiltinDecoderReader.h \ + nsDOMMediaStream.h \ nsMediaCache.h \ nsMediaDecoder.h \ SharedBuffer.h \ @@ -74,6 +80,7 @@ CPPSRCS = \ nsBuiltinDecoder.cpp \ nsBuiltinDecoderStateMachine.cpp \ nsBuiltinDecoderReader.cpp \ + nsDOMMediaStream.cpp \ nsMediaCache.cpp \ nsMediaDecoder.cpp \ StreamBuffer.cpp \ @@ -124,3 +131,5 @@ INCLUDES += \ -I$(srcdir)/../base/src \ -I$(srcdir)/../html/content/src \ $(NULL) + +DEFINES += -D_IMPL_NS_LAYOUT diff --git a/content/media/nsDOMMediaStream.cpp b/content/media/nsDOMMediaStream.cpp new file mode 100644 index 00000000000..928d1d3f7a9 --- /dev/null +++ b/content/media/nsDOMMediaStream.cpp @@ -0,0 +1,49 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "nsDOMMediaStream.h" +#include "nsDOMClassInfoID.h" +#include "nsContentUtils.h" + +using namespace mozilla; + +DOMCI_DATA(MediaStream, nsDOMMediaStream) + +NS_INTERFACE_MAP_BEGIN_CYCLE_COLLECTION(nsDOMMediaStream) + NS_INTERFACE_MAP_ENTRY(nsISupports) + NS_INTERFACE_MAP_ENTRY(nsIDOMMediaStream) + NS_DOM_INTERFACE_MAP_ENTRY_CLASSINFO(MediaStream) +NS_INTERFACE_MAP_END + +NS_IMPL_CYCLE_COLLECTING_ADDREF(nsDOMMediaStream) +NS_IMPL_CYCLE_COLLECTING_RELEASE(nsDOMMediaStream) + +NS_IMPL_CYCLE_COLLECTION_CLASS(nsDOMMediaStream) + +NS_IMPL_CYCLE_COLLECTION_UNLINK_BEGIN(nsDOMMediaStream) +NS_IMPL_CYCLE_COLLECTION_UNLINK_END + +NS_IMPL_CYCLE_COLLECTION_TRAVERSE_BEGIN(nsDOMMediaStream) +NS_IMPL_CYCLE_COLLECTION_TRAVERSE_END + +nsDOMMediaStream::~nsDOMMediaStream() +{ + if (mStream) { + mStream->Destroy(); + } +} + +NS_IMETHODIMP +nsDOMMediaStream::GetCurrentTime(double *aCurrentTime) +{ + *aCurrentTime = mStream ? MediaTimeToSeconds(mStream->GetCurrentTime()) : 0.0; + return NS_OK; +} + +bool +nsDOMMediaStream::CombineWithPrincipal(nsIPrincipal* aPrincipal) +{ + return nsContentUtils::CombineResourcePrincipals(&mPrincipal, aPrincipal); +} diff --git a/content/media/nsDOMMediaStream.h b/content/media/nsDOMMediaStream.h new file mode 100644 index 00000000000..0b4827c981b --- /dev/null +++ b/content/media/nsDOMMediaStream.h @@ -0,0 +1,55 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*-*/ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef NSDOMMEDIASTREAM_H_ +#define NSDOMMEDIASTREAM_H_ + +#include "nsIDOMMediaStream.h" +#include "MediaStreamGraph.h" +#include "nsCycleCollectionParticipant.h" +#include "nsIPrincipal.h" + +/** + * DOM wrapper for MediaStreams. + */ +class nsDOMMediaStream : public nsIDOMMediaStream +{ + typedef mozilla::MediaStream MediaStream; + +public: + nsDOMMediaStream() : mStream(nsnull) {} + virtual ~nsDOMMediaStream(); + + NS_DECL_CYCLE_COLLECTION_CLASS(nsDOMMediaStream) + NS_DECL_CYCLE_COLLECTING_ISUPPORTS + + NS_DECL_NSIDOMMEDIASTREAM + + MediaStream* GetStream() { return mStream; } + bool IsFinished() { return !mStream || mStream->IsFinished(); } + /** + * Returns a principal indicating who may access this stream. The stream contents + * can only be accessed by principals subsuming this principal. + */ + nsIPrincipal* GetPrincipal() { return mPrincipal; } + + /** + * Indicate that data will be contributed to this stream from origin aPrincipal. + * If aPrincipal is null, this is ignored. Otherwise, from now on the contents + * of this stream can only be accessed by principals that subsume aPrincipal. + * Returns true if the stream's principal changed. + */ + bool CombineWithPrincipal(nsIPrincipal* aPrincipal); + +protected: + // MediaStream is owned by the graph, but we tell it when to die, and it won't + // die until we let it. + MediaStream* mStream; + // Principal identifying who may access the contents of this stream. + // If null, this stream can be used by anyone because it has no content yet. + nsCOMPtr mPrincipal; +}; + +#endif /* NSDOMMEDIASTREAM_H_ */ diff --git a/content/media/nsIDOMMediaStream.idl b/content/media/nsIDOMMediaStream.idl new file mode 100644 index 00000000000..cc748ae1752 --- /dev/null +++ b/content/media/nsIDOMMediaStream.idl @@ -0,0 +1,12 @@ +/* -*- Mode: IDL; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "nsISupports.idl" + +[scriptable, builtinclass, uuid(f37c2871-4cb7-4672-bb28-c2d601f7cc9e)] +interface nsIDOMMediaStream : nsISupports +{ + readonly attribute double currentTime; +}; diff --git a/dom/base/nsDOMClassInfo.cpp b/dom/base/nsDOMClassInfo.cpp index c9ca767e9ca..a7ddf188f88 100644 --- a/dom/base/nsDOMClassInfo.cpp +++ b/dom/base/nsDOMClassInfo.cpp @@ -309,6 +309,7 @@ #include "nsIDOMHTMLSourceElement.h" #include "nsIDOMHTMLVideoElement.h" #include "nsIDOMHTMLAudioElement.h" +#include "nsIDOMMediaStream.h" #include "nsIDOMProgressEvent.h" #include "nsIDOMCSS2Properties.h" #include "nsIDOMCSSCharsetRule.h" @@ -1480,6 +1481,8 @@ static nsDOMClassInfoData sClassInfoData[] = { ELEMENT_SCRIPTABLE_FLAGS) NS_DEFINE_CLASSINFO_DATA(TimeRanges, nsDOMGenericSH, DOM_DEFAULT_SCRIPTABLE_FLAGS) + NS_DEFINE_CLASSINFO_DATA(MediaStream, nsDOMGenericSH, + DOM_DEFAULT_SCRIPTABLE_FLAGS) #endif NS_DEFINE_CLASSINFO_DATA(ProgressEvent, nsDOMGenericSH, @@ -4135,6 +4138,10 @@ nsDOMClassInfo::Init() DOM_CLASSINFO_MAP_BEGIN(TimeRanges, nsIDOMTimeRanges) DOM_CLASSINFO_MAP_ENTRY(nsIDOMTimeRanges) DOM_CLASSINFO_MAP_END + + DOM_CLASSINFO_MAP_BEGIN(MediaStream, nsIDOMMediaStream) + DOM_CLASSINFO_MAP_ENTRY(nsIDOMMediaStream) + DOM_CLASSINFO_MAP_END #endif DOM_CLASSINFO_MAP_BEGIN(ProgressEvent, nsIDOMProgressEvent) diff --git a/dom/base/nsDOMClassInfoClasses.h b/dom/base/nsDOMClassInfoClasses.h index 4becd364d34..e2415471a49 100644 --- a/dom/base/nsDOMClassInfoClasses.h +++ b/dom/base/nsDOMClassInfoClasses.h @@ -449,6 +449,9 @@ DOMCI_CLASS(HTMLSourceElement) DOMCI_CLASS(MediaError) DOMCI_CLASS(HTMLAudioElement) DOMCI_CLASS(TimeRanges) + +// Media streams +DOMCI_CLASS(MediaStream) #endif DOMCI_CLASS(ProgressEvent) diff --git a/mobile/android/installer/package-manifest.in b/mobile/android/installer/package-manifest.in index 34d7235c33a..60f1bb1e9ee 100644 --- a/mobile/android/installer/package-manifest.in +++ b/mobile/android/installer/package-manifest.in @@ -144,6 +144,7 @@ @BINPATH@/components/content_canvas.xpt @BINPATH@/components/content_htmldoc.xpt @BINPATH@/components/content_html.xpt +@BINPATH@/components/content_media.xpt @BINPATH@/components/content_xslt.xpt @BINPATH@/components/content_xtf.xpt @BINPATH@/components/cookie.xpt diff --git a/mobile/xul/installer/package-manifest.in b/mobile/xul/installer/package-manifest.in index e836e6d08c3..897891a1e5e 100644 --- a/mobile/xul/installer/package-manifest.in +++ b/mobile/xul/installer/package-manifest.in @@ -144,6 +144,7 @@ @BINPATH@/components/content_canvas.xpt @BINPATH@/components/content_htmldoc.xpt @BINPATH@/components/content_html.xpt +@BINPATH@/components/content_media.xpt @BINPATH@/components/content_xslt.xpt @BINPATH@/components/content_xtf.xpt @BINPATH@/components/cookie.xpt From a94e4412e86c80f15c4e2c2e2182b0f387b5ac8c Mon Sep 17 00:00:00 2001 From: Robert O'Callahan Date: Mon, 30 Apr 2012 15:11:40 +1200 Subject: [PATCH 011/162] Bug 664918. Part 5: Create SourceMediaStream, a MediaStream with an API allowing data to be injected into it by some source. r=jesup --- content/media/MediaStreamGraph.cpp | 172 +++++++++++++++++++++++++++++ content/media/MediaStreamGraph.h | 121 ++++++++++++++++++++ content/media/nsDOMMediaStream.cpp | 9 ++ content/media/nsDOMMediaStream.h | 5 + 4 files changed, 307 insertions(+) diff --git a/content/media/MediaStreamGraph.cpp b/content/media/MediaStreamGraph.cpp index 96e09babc27..bc346493326 100644 --- a/content/media/MediaStreamGraph.cpp +++ b/content/media/MediaStreamGraph.cpp @@ -205,6 +205,14 @@ public: * will take effect. */ void ChooseActionTime(); + /** + * Extract any state updates pending in aStream, and apply them. + */ + void ExtractPendingInput(SourceMediaStream* aStream); + /** + * Update "have enough data" flags in aStream. + */ + void UpdateBufferSufficiencyState(SourceMediaStream* aStream); /** * Compute the blocking states of streams from mBlockingDecisionsMadeUntilTime * until the desired future time (determined by heuristic). @@ -604,6 +612,72 @@ MediaStreamGraphImpl::ChooseActionTime() mLastActionTime = GetEarliestActionTime(); } +void +MediaStreamGraphImpl::ExtractPendingInput(SourceMediaStream* aStream) +{ + bool finished; + { + MutexAutoLock lock(aStream->mMutex); + finished = aStream->mUpdateFinished; + for (PRInt32 i = aStream->mUpdateTracks.Length() - 1; i >= 0; --i) { + SourceMediaStream::TrackData* data = &aStream->mUpdateTracks[i]; + if (data->mCommands & SourceMediaStream::TRACK_CREATE) { + MediaSegment* segment = data->mData.forget(); + LOG(PR_LOG_DEBUG, ("SourceMediaStream %p creating track %d, rate %d, start %lld, initial end %lld", + aStream, data->mID, data->mRate, PRInt64(data->mStart), + PRInt64(segment->GetDuration()))); + aStream->mBuffer.AddTrack(data->mID, data->mRate, data->mStart, segment); + // The track has taken ownership of data->mData, so let's replace + // data->mData with an empty clone. + data->mData = segment->CreateEmptyClone(); + data->mCommands &= ~SourceMediaStream::TRACK_CREATE; + } else if (data->mData->GetDuration() > 0) { + MediaSegment* dest = aStream->mBuffer.FindTrack(data->mID)->GetSegment(); + LOG(PR_LOG_DEBUG, ("SourceMediaStream %p track %d, advancing end from %lld to %lld", + aStream, data->mID, + PRInt64(dest->GetDuration()), + PRInt64(dest->GetDuration() + data->mData->GetDuration()))); + dest->AppendFrom(data->mData); + } + if (data->mCommands & SourceMediaStream::TRACK_END) { + aStream->mBuffer.FindTrack(data->mID)->SetEnded(); + aStream->mUpdateTracks.RemoveElementAt(i); + } + } + aStream->mBuffer.AdvanceKnownTracksTime(aStream->mUpdateKnownTracksTime); + } + if (finished) { + FinishStream(aStream); + } +} + +void +MediaStreamGraphImpl::UpdateBufferSufficiencyState(SourceMediaStream* aStream) +{ + StreamTime desiredEnd = GetDesiredBufferEnd(aStream); + nsTArray runnables; + + { + MutexAutoLock lock(aStream->mMutex); + for (PRUint32 i = 0; i < aStream->mUpdateTracks.Length(); ++i) { + SourceMediaStream::TrackData* data = &aStream->mUpdateTracks[i]; + if (data->mCommands & SourceMediaStream::TRACK_CREATE) { + continue; + } + StreamBuffer::Track* track = aStream->mBuffer.FindTrack(data->mID); + data->mHaveEnough = track->GetEndTimeRoundDown() >= desiredEnd; + if (!data->mHaveEnough) { + runnables.MoveElementsFrom(data->mDispatchWhenNotEnough); + } + } + } + + for (PRUint32 i = 0; i < runnables.Length(); ++i) { + runnables[i].mThread->Dispatch(runnables[i].mRunnable, 0); + } +} + + StreamTime MediaStreamGraphImpl::GraphTimeToStreamTime(MediaStream* aStream, GraphTime aTime) @@ -1186,6 +1260,14 @@ MediaStreamGraphImpl::RunThread() } messageQueue.Clear(); + // Grab pending ProcessingEngine results. + for (PRUint32 i = 0; i < mStreams.Length(); ++i) { + SourceMediaStream* is = mStreams[i]->AsSourceStream(); + if (is) { + ExtractPendingInput(is); + } + } + GraphTime prevBlockingDecisionsMadeUntilTime = mBlockingDecisionsMadeUntilTime; RecomputeBlocking(); @@ -1202,6 +1284,10 @@ MediaStreamGraphImpl::RunThread() ++audioStreamsActive; } PlayVideo(stream); + SourceMediaStream* is = stream->AsSourceStream(); + if (is) { + UpdateBufferSufficiencyState(is); + } GraphTime end; if (!stream->mBlocked.GetAt(mCurrentTime, &end) || end < GRAPH_TIME_MAX) { allBlockedForever = false; @@ -1705,6 +1791,83 @@ MediaStream::RemoveListener(MediaStreamListener* aListener) GraphImpl()->AppendMessage(new Message(this, aListener)); } +void +SourceMediaStream::AddTrack(TrackID aID, TrackRate aRate, TrackTicks aStart, + MediaSegment* aSegment) +{ + { + MutexAutoLock lock(mMutex); + TrackData* data = mUpdateTracks.AppendElement(); + data->mID = aID; + data->mRate = aRate; + data->mStart = aStart; + data->mCommands = TRACK_CREATE; + data->mData = aSegment; + data->mHaveEnough = false; + } + GraphImpl()->EnsureNextIteration(); +} + +void +SourceMediaStream::AppendToTrack(TrackID aID, MediaSegment* aSegment) +{ + { + MutexAutoLock lock(mMutex); + FindDataForTrack(aID)->mData->AppendFrom(aSegment); + } + GraphImpl()->EnsureNextIteration(); +} + +bool +SourceMediaStream::HaveEnoughBuffered(TrackID aID) +{ + MutexAutoLock lock(mMutex); + return FindDataForTrack(aID)->mHaveEnough; +} + +void +SourceMediaStream::DispatchWhenNotEnoughBuffered(TrackID aID, + nsIThread* aSignalThread, nsIRunnable* aSignalRunnable) +{ + MutexAutoLock lock(mMutex); + TrackData* data = FindDataForTrack(aID); + if (data->mHaveEnough) { + data->mDispatchWhenNotEnough.AppendElement()->Init(aSignalThread, aSignalRunnable); + } else { + aSignalThread->Dispatch(aSignalRunnable, 0); + } +} + +void +SourceMediaStream::EndTrack(TrackID aID) +{ + { + MutexAutoLock lock(mMutex); + FindDataForTrack(aID)->mCommands |= TRACK_END; + } + GraphImpl()->EnsureNextIteration(); +} + +void +SourceMediaStream::AdvanceKnownTracksTime(StreamTime aKnownTime) +{ + { + MutexAutoLock lock(mMutex); + mUpdateKnownTracksTime = aKnownTime; + } + GraphImpl()->EnsureNextIteration(); +} + +void +SourceMediaStream::Finish() +{ + { + MutexAutoLock lock(mMutex); + mUpdateFinished = true; + } + GraphImpl()->EnsureNextIteration(); +} + static const PRUint32 kThreadLimit = 4; static const PRUint32 kIdleThreadLimit = 4; static const PRUint32 kIdleThreadTimeoutMs = 2000; @@ -1769,4 +1932,13 @@ MediaStreamGraph::GetInstance() return gGraph; } +SourceMediaStream* +MediaStreamGraph::CreateInputStream(nsDOMMediaStream* aWrapper) +{ + SourceMediaStream* stream = new SourceMediaStream(aWrapper); + NS_ADDREF(stream); + static_cast(this)->AppendMessage(new CreateMessage(stream)); + return stream; +} + } diff --git a/content/media/MediaStreamGraph.h b/content/media/MediaStreamGraph.h index 5179f1d0db9..8d372f9eb5f 100644 --- a/content/media/MediaStreamGraph.h +++ b/content/media/MediaStreamGraph.h @@ -111,6 +111,7 @@ public: }; class MediaStreamGraphImpl; +class SourceMediaStream; /** * A stream of synchronized audio and video data. All (not blocked) streams @@ -216,6 +217,8 @@ public: friend class MediaStreamGraphImpl; + virtual SourceMediaStream* AsSourceStream() { return nsnull; } + // media graph thread only void Init(); // These Impl methods perform the core functionality of the control methods @@ -342,6 +345,119 @@ protected: bool mMainThreadFinished; }; +/** + * This is a stream into which a decoder can write audio and video. + * + * Audio and video can be written on any thread, but you probably want to + * always write from the same thread to avoid unexpected interleavings. + */ +class SourceMediaStream : public MediaStream { +public: + SourceMediaStream(nsDOMMediaStream* aWrapper) : + MediaStream(aWrapper), mMutex("mozilla::media::SourceMediaStream"), + mUpdateKnownTracksTime(0), mUpdateFinished(false) + {} + + virtual SourceMediaStream* AsSourceStream() { return this; } + + // Call these on any thread. + /** + * Add a new track to the stream starting at the given base time (which + * must be greater than or equal to the last time passed to + * AdvanceKnownTracksTime). Takes ownership of aSegment. aSegment should + * contain data starting after aStart. + */ + void AddTrack(TrackID aID, TrackRate aRate, TrackTicks aStart, + MediaSegment* aSegment); + /** + * Append media data to a track. Ownership of aSegment remains with the caller, + * but aSegment is emptied. + */ + void AppendToTrack(TrackID aID, MediaSegment* aSegment); + /** + * Returns true if the buffer currently has enough data. + */ + bool HaveEnoughBuffered(TrackID aID); + /** + * Ensures that aSignalRunnable will be dispatched to aSignalThread + * when we don't have enough buffered data in the track (which could be + * immediately). + */ + void DispatchWhenNotEnoughBuffered(TrackID aID, + nsIThread* aSignalThread, nsIRunnable* aSignalRunnable); + /** + * Indicate that a track has ended. Do not do any more API calls + * affecting this track. + */ + void EndTrack(TrackID aID); + /** + * Indicate that no tracks will be added starting before time aKnownTime. + * aKnownTime must be >= its value at the last call to AdvanceKnownTracksTime. + */ + void AdvanceKnownTracksTime(StreamTime aKnownTime); + /** + * Indicate that this stream should enter the "finished" state. All tracks + * must have been ended via EndTrack. The finish time of the stream is + * when all tracks have ended and when latest time sent to + * AdvanceKnownTracksTime() has been reached. + */ + void Finish(); + + // XXX need a Reset API + + friend class MediaStreamGraph; + friend class MediaStreamGraphImpl; + + struct ThreadAndRunnable { + void Init(nsIThread* aThread, nsIRunnable* aRunnable) + { + mThread = aThread; + mRunnable = aRunnable; + } + + nsCOMPtr mThread; + nsCOMPtr mRunnable; + }; + enum TrackCommands { + TRACK_CREATE = 0x01, + TRACK_END = 0x02 + }; + /** + * Data for each track that hasn't ended. + */ + struct TrackData { + TrackID mID; + TrackRate mRate; + TrackTicks mStart; + // Each time the track updates are flushed to the media graph thread, + // this is cleared. + PRUint32 mCommands; + // Each time the track updates are flushed to the media graph thread, + // the segment buffer is emptied. + nsAutoPtr mData; + nsTArray mDispatchWhenNotEnough; + bool mHaveEnough; + }; + +protected: + TrackData* FindDataForTrack(TrackID aID) + { + for (PRUint32 i = 0; i < mUpdateTracks.Length(); ++i) { + if (mUpdateTracks[i].mID == aID) { + return &mUpdateTracks[i]; + } + } + NS_ERROR("Bad track ID!"); + return nsnull; + } + + Mutex mMutex; + // protected by mMutex + StreamTime mUpdateKnownTracksTime; + nsTArray mUpdateTracks; + bool mUpdateFinished; +}; + /** * Initially, at least, we will have a singleton MediaStreamGraph per * process. @@ -351,6 +467,11 @@ public: // Main thread only static MediaStreamGraph* GetInstance(); // Control API. + /** + * Create a stream that a media decoder (or some other source of + * media data, such as a camera) can write to. + */ + SourceMediaStream* CreateInputStream(nsDOMMediaStream* aWrapper); /** * Returns the number of graph updates sent. This can be used to track * whether a given update has been processed by the graph thread and reflected diff --git a/content/media/nsDOMMediaStream.cpp b/content/media/nsDOMMediaStream.cpp index 928d1d3f7a9..2b217bcb45b 100644 --- a/content/media/nsDOMMediaStream.cpp +++ b/content/media/nsDOMMediaStream.cpp @@ -42,6 +42,15 @@ nsDOMMediaStream::GetCurrentTime(double *aCurrentTime) return NS_OK; } +already_AddRefed +nsDOMMediaStream::CreateInputStream() +{ + nsRefPtr stream = new nsDOMMediaStream(); + MediaStreamGraph* gm = MediaStreamGraph::GetInstance(); + stream->mStream = gm->CreateInputStream(stream); + return stream.forget(); +} + bool nsDOMMediaStream::CombineWithPrincipal(nsIPrincipal* aPrincipal) { diff --git a/content/media/nsDOMMediaStream.h b/content/media/nsDOMMediaStream.h index 0b4827c981b..3b42c335f84 100644 --- a/content/media/nsDOMMediaStream.h +++ b/content/media/nsDOMMediaStream.h @@ -43,6 +43,11 @@ public: */ bool CombineWithPrincipal(nsIPrincipal* aPrincipal); + /** + * Create an nsDOMMediaStream whose underlying stream is a SourceMediaStream. + */ + static already_AddRefed CreateInputStream(); + protected: // MediaStream is owned by the graph, but we tell it when to die, and it won't // die until we let it. From cb775dfed8a010e11b87749565150f31b20e3de7 Mon Sep 17 00:00:00 2001 From: Robert O'Callahan Date: Mon, 30 Apr 2012 15:11:47 +1200 Subject: [PATCH 012/162] Bug 664918. Part 6: ImageContainer::GetCurrentAsSurface shouldn't crash when mActiveImage is null. r=bas --- gfx/layers/ImageLayers.cpp | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/gfx/layers/ImageLayers.cpp b/gfx/layers/ImageLayers.cpp index b561ef69c81..19b93722a23 100644 --- a/gfx/layers/ImageLayers.cpp +++ b/gfx/layers/ImageLayers.cpp @@ -231,12 +231,15 @@ ImageContainer::GetCurrentAsSurface(gfxIntSize *aSize) CrossProcessMutexAutoLock autoLock(*mRemoteDataMutex); EnsureActiveImage(); + if (!mActiveImage) + return nsnull; *aSize = mRemoteData->mSize; - return mActiveImage ? mActiveImage->GetAsSurface() : nsnull; + } else { + if (!mActiveImage) + return nsnull; + *aSize = mActiveImage->GetSize(); } - - *aSize = mActiveImage->GetSize(); - return mActiveImage ? mActiveImage->GetAsSurface() : nsnull; + return mActiveImage->GetAsSurface(); } gfxIntSize From 53a92c834ad5b4e03c221ab13ec611732e09d39f Mon Sep 17 00:00:00 2001 From: Robert O'Callahan Date: Mon, 30 Apr 2012 15:12:28 +1200 Subject: [PATCH 013/162] Bug 664918. Part 7: Let the "src" attribute of HTML media elements accept a MediaStream DOM object, to make the media element play back the contents of the given stream. r=cpearce,jesup --- .../html/content/public/nsHTMLMediaElement.h | 46 ++- content/html/content/src/Makefile.in | 1 + .../html/content/src/nsHTMLAudioElement.cpp | 20 +- .../html/content/src/nsHTMLMediaElement.cpp | 356 +++++++++++++++--- .../html/nsIDOMHTMLAudioElement.idl | 2 +- .../html/nsIDOMHTMLMediaElement.idl | 4 +- .../html/nsIDOMHTMLVideoElement.idl | 2 +- 7 files changed, 354 insertions(+), 77 deletions(-) diff --git a/content/html/content/public/nsHTMLMediaElement.h b/content/html/content/public/nsHTMLMediaElement.h index 0c40ac5ae14..8f11d18a318 100644 --- a/content/html/content/public/nsHTMLMediaElement.h +++ b/content/html/content/public/nsHTMLMediaElement.h @@ -51,6 +51,8 @@ #include "nsAudioStream.h" #include "VideoFrameContainer.h" #include "mozilla/CORSMode.h" +#include "nsDOMMediaStream.h" +#include "mozilla/Mutex.h" // Define to output information on decoding and painting framerate /* #define DEBUG_FRAME_RATE 1 */ @@ -58,6 +60,10 @@ typedef PRUint16 nsMediaNetworkState; typedef PRUint16 nsMediaReadyState; +namespace mozilla { +class MediaResource; +} + class nsHTMLMediaElement : public nsGenericHTMLElement, public nsIObserver { @@ -65,6 +71,8 @@ public: typedef mozilla::TimeStamp TimeStamp; typedef mozilla::layers::ImageContainer ImageContainer; typedef mozilla::VideoFrameContainer VideoFrameContainer; + typedef mozilla::MediaStream MediaStream; + typedef mozilla::MediaResource MediaResource; enum CanPlayStatus { CANPLAY_NO, @@ -252,7 +260,9 @@ public: // http://www.whatwg.org/specs/web-apps/current-work/#ended bool IsPlaybackEnded() const; - // principal of the currently playing stream + // principal of the currently playing resource. Anything accessing the contents + // of this element must have a principal that subsumes this principal. + // Returns null if nothing is playing. already_AddRefed GetCurrentPrincipal(); // Update the visual size of the media. Called from the decoder on the @@ -370,8 +380,15 @@ public: */ void FireTimeUpdate(bool aPeriodic); + MediaStream* GetMediaStream() + { + NS_ASSERTION(mStream, "Don't call this when not playing a stream"); + return mStream->GetStream(); + } + protected: class MediaLoadListener; + class StreamListener; /** * Logs a warning message to the web console to report various failures. @@ -390,6 +407,15 @@ protected: */ void SetPlayedOrSeeked(bool aValue); + /** + * Initialize the media element for playback of mSrcAttrStream + */ + void SetupMediaStreamPlayback(); + /** + * Stop playback on mStream. + */ + void EndMediaStreamPlayback(); + /** * Create a decoder for the given aMIMEType. Returns null if we * were unable to create the decoder. @@ -415,7 +441,10 @@ protected: * Finish setting up the decoder after Load() has been called on it. * Called by InitializeDecoderForChannel/InitializeDecoderAsClone. */ - nsresult FinishDecoderSetup(nsMediaDecoder* aDecoder); + nsresult FinishDecoderSetup(nsMediaDecoder* aDecoder, + MediaResource* aStream, + nsIStreamListener **aListener, + nsMediaDecoder* aCloneDonor); /** * Call this after setting up mLoadingSrc and mDecoder. @@ -588,12 +617,25 @@ protected: void ProcessMediaFragmentURI(); // The current decoder. Load() has been called on this decoder. + // At most one of mDecoder and mStream can be non-null. nsRefPtr mDecoder; // A reference to the VideoFrameContainer which contains the current frame // of video to display. nsRefPtr mVideoFrameContainer; + // Holds a reference to the DOM wrapper for the MediaStream that has been + // set in the src attribute. + nsRefPtr mSrcAttrStream; + + // Holds a reference to the DOM wrapper for the MediaStream that we're + // actually playing. + // At most one of mDecoder and mStream can be non-null. + nsRefPtr mStream; + + // Holds a reference to the MediaStreamListener attached to mStream. STRONG! + StreamListener* mStreamListener; + // Holds a reference to the first channel we open to the media resource. // Once the decoder is created, control over the channel passes to the // decoder, and we null out this reference. We must store this in case diff --git a/content/html/content/src/Makefile.in b/content/html/content/src/Makefile.in index 2df9999372d..961b89e7110 100644 --- a/content/html/content/src/Makefile.in +++ b/content/html/content/src/Makefile.in @@ -149,6 +149,7 @@ INCLUDES += \ -I$(srcdir)/../../../../editor/libeditor/base \ -I$(srcdir)/../../../../editor/libeditor/text \ -I$(srcdir) \ + -I$(topsrcdir)/js/xpconnect/src \ -I$(topsrcdir)/xpcom/ds \ $(NULL) diff --git a/content/html/content/src/nsHTMLAudioElement.cpp b/content/html/content/src/nsHTMLAudioElement.cpp index d43ad8db685..398b5dbadcd 100644 --- a/content/html/content/src/nsHTMLAudioElement.cpp +++ b/content/html/content/src/nsHTMLAudioElement.cpp @@ -132,23 +132,9 @@ nsHTMLAudioElement::Initialize(nsISupports* aOwner, JSContext* aContext, return NS_OK; } - // The only (optional) argument is the url of the audio - JSString* jsstr = JS_ValueToString(aContext, argv[0]); - if (!jsstr) - return NS_ERROR_FAILURE; - - nsDependentJSString str; - if (!str.init(aContext, jsstr)) - return NS_ERROR_FAILURE; - - rv = SetAttr(kNameSpaceID_None, nsGkAtoms::src, str, true); - if (NS_FAILED(rv)) - return rv; - - // We have been specified with a src URL. Begin a load. - QueueSelectResourceTask(); - - return NS_OK; + // The only (optional) argument is the src of the audio (which can + // be a URL string or a MediaStream object) + return SetSrc(aContext, argv[0]); } NS_IMETHODIMP diff --git a/content/html/content/src/nsHTMLMediaElement.cpp b/content/html/content/src/nsHTMLMediaElement.cpp index c3f02cf2f34..b96adc4aea9 100644 --- a/content/html/content/src/nsHTMLMediaElement.cpp +++ b/content/html/content/src/nsHTMLMediaElement.cpp @@ -54,6 +54,7 @@ #include "nsNodeInfoManager.h" #include "nsNetUtil.h" #include "nsXPCOMStrings.h" +#include "xpcpublic.h" #include "nsThreadUtils.h" #include "nsIThreadInternal.h" #include "nsContentUtils.h" @@ -92,6 +93,9 @@ #include "nsIDOMNotifyAudioAvailableEvent.h" #include "nsMediaFragmentURIParser.h" #include "nsURIHashKey.h" +#include "nsJSUtils.h" +#include "MediaStreamGraph.h" +#include "nsDOMMediaStream.h" #include "nsIScriptError.h" #ifdef MOZ_OGG @@ -418,12 +422,20 @@ NS_IMPL_RELEASE_INHERITED(nsHTMLMediaElement, nsGenericHTMLElement) NS_IMPL_CYCLE_COLLECTION_CLASS(nsHTMLMediaElement) NS_IMPL_CYCLE_COLLECTION_TRAVERSE_BEGIN_INHERITED(nsHTMLMediaElement, nsGenericHTMLElement) + NS_IMPL_CYCLE_COLLECTION_TRAVERSE_NSCOMPTR(mStream) + NS_IMPL_CYCLE_COLLECTION_TRAVERSE_NSCOMPTR(mSrcAttrStream) NS_IMPL_CYCLE_COLLECTION_TRAVERSE_NSCOMPTR(mSourcePointer) NS_IMPL_CYCLE_COLLECTION_TRAVERSE_NSCOMPTR(mLoadBlockedDoc) NS_IMPL_CYCLE_COLLECTION_TRAVERSE_NSCOMPTR(mSourceLoadCandidate) NS_IMPL_CYCLE_COLLECTION_TRAVERSE_END NS_IMPL_CYCLE_COLLECTION_UNLINK_BEGIN_INHERITED(nsHTMLMediaElement, nsGenericHTMLElement) + if (tmp->mStream) { + // Need to EndMediaStreamPlayback to clear mStream and make sure everything + // gets unhooked correctly. + tmp->EndMediaStreamPlayback(); + } + NS_IMPL_CYCLE_COLLECTION_UNLINK_NSCOMPTR(mSrcAttrStream) NS_IMPL_CYCLE_COLLECTION_UNLINK_NSCOMPTR(mSourcePointer) NS_IMPL_CYCLE_COLLECTION_UNLINK_NSCOMPTR(mLoadBlockedDoc) NS_IMPL_CYCLE_COLLECTION_UNLINK_NSCOMPTR(mSourceLoadCandidate) @@ -434,13 +446,56 @@ NS_INTERFACE_MAP_BEGIN_CYCLE_COLLECTION_INHERITED(nsHTMLMediaElement) NS_INTERFACE_MAP_END_INHERITING(nsGenericHTMLElement) // nsIDOMHTMLMediaElement -NS_IMPL_URI_ATTR(nsHTMLMediaElement, Src, src) NS_IMPL_BOOL_ATTR(nsHTMLMediaElement, Controls, controls) NS_IMPL_BOOL_ATTR(nsHTMLMediaElement, Autoplay, autoplay) NS_IMPL_BOOL_ATTR(nsHTMLMediaElement, Loop, loop) NS_IMPL_BOOL_ATTR(nsHTMLMediaElement, DefaultMuted, muted) NS_IMPL_ENUM_ATTR_DEFAULT_VALUE(nsHTMLMediaElement, Preload, preload, NULL) +NS_IMETHODIMP +nsHTMLMediaElement::GetSrc(JSContext* aCtx, jsval *aParams) +{ + if (mSrcAttrStream) { + NS_ASSERTION(mSrcAttrStream->GetStream(), "MediaStream should have been set up properly"); + return nsContentUtils::WrapNative(aCtx, JS_GetGlobalForScopeChain(aCtx), + mSrcAttrStream, aParams); + } + + nsAutoString str; + nsresult rv = GetURIAttr(nsGkAtoms::src, nsnull, str); + NS_ENSURE_SUCCESS(rv, rv); + if (!xpc::StringToJsval(aCtx, str, aParams)) { + return NS_ERROR_FAILURE; + } + return NS_OK; +} + +NS_IMETHODIMP +nsHTMLMediaElement::SetSrc(JSContext* aCtx, const jsval & aParams) +{ + if (JSVAL_IS_OBJECT(aParams)) { + nsCOMPtr stream; + stream = do_QueryInterface(nsContentUtils::XPConnect()-> + GetNativeOfWrapper(aCtx, JSVAL_TO_OBJECT(aParams))); + if (stream) { + mSrcAttrStream = static_cast(stream.get()); + UnsetAttr(kNameSpaceID_None, nsGkAtoms::src, true); + Load(); + return NS_OK; + } + } + + mSrcAttrStream = nsnull; + JSString* jsStr = JS_ValueToString(aCtx, aParams); + if (!jsStr) + return NS_ERROR_DOM_TYPE_MISMATCH_ERR; + nsDependentJSString str; + if (!str.init(aCtx, jsStr)) + return NS_ERROR_DOM_TYPE_MISMATCH_ERR; + // Will trigger Load() + return SetAttrHelper(nsGkAtoms::src, str); +} + /* readonly attribute nsIDOMHTMLMediaElement mozAutoplayEnabled; */ NS_IMETHODIMP nsHTMLMediaElement::GetMozAutoplayEnabled(bool *aAutoplayEnabled) { @@ -460,8 +515,11 @@ NS_IMETHODIMP nsHTMLMediaElement::GetError(nsIDOMMediaError * *aError) /* readonly attribute boolean ended; */ NS_IMETHODIMP nsHTMLMediaElement::GetEnded(bool *aEnded) { - *aEnded = mDecoder ? mDecoder->IsEnded() : false; - + if (mStream) { + *aEnded = GetMediaStream()->IsFinished(); + } else if (mDecoder) { + *aEnded = mDecoder->IsEnded(); + } return NS_OK; } @@ -528,6 +586,9 @@ void nsHTMLMediaElement::AbortExistingLoads() mDecoder->Shutdown(); mDecoder = nsnull; } + if (mStream) { + EndMediaStreamPlayback(); + } mLoadingSrc = nsnull; if (mNetworkState == nsIDOMHTMLMediaElement::NETWORK_LOADING || @@ -550,6 +611,7 @@ void nsHTMLMediaElement::AbortExistingLoads() if (mNetworkState != nsIDOMHTMLMediaElement::NETWORK_EMPTY) { mNetworkState = nsIDOMHTMLMediaElement::NETWORK_EMPTY; + NS_ASSERTION(!mDecoder && !mStream, "How did someone setup a new stream/decoder already?"); ChangeReadyState(nsIDOMHTMLMediaElement::HAVE_NOTHING); mPaused = true; @@ -672,7 +734,8 @@ void nsHTMLMediaElement::SelectResourceWrapper() void nsHTMLMediaElement::SelectResource() { - if (!HasAttr(kNameSpaceID_None, nsGkAtoms::src) && !HasSourceChildren(this)) { + if (!mSrcAttrStream && !HasAttr(kNameSpaceID_None, nsGkAtoms::src) && + !HasSourceChildren(this)) { // The media element has neither a src attribute nor any source // element children, abort the load. mNetworkState = nsIDOMHTMLMediaElement::NETWORK_EMPTY; @@ -696,7 +759,9 @@ void nsHTMLMediaElement::SelectResource() // If we have a 'src' attribute, use that exclusively. nsAutoString src; - if (GetAttr(kNameSpaceID_None, nsGkAtoms::src, src)) { + if (mSrcAttrStream) { + SetupMediaStreamPlayback(); + } else if (GetAttr(kNameSpaceID_None, nsGkAtoms::src, src)) { nsCOMPtr uri; nsresult rv = NewURIFromString(src, getter_AddRefs(uri)); if (NS_SUCCEEDED(rv)) { @@ -1064,7 +1129,12 @@ nsresult nsHTMLMediaElement::LoadWithChannel(nsIChannel *aChannel, *aListener = nsnull; + // Make sure we don't reenter during synchronous abort events. + if (mIsRunningLoadMethod) + return NS_OK; + mIsRunningLoadMethod = true; AbortExistingLoads(); + mIsRunningLoadMethod = false; nsresult rv = aChannel->GetOriginalURI(getter_AddRefs(mLoadingSrc)); NS_ENSURE_SUCCESS(rv, rv); @@ -1085,7 +1155,12 @@ NS_IMETHODIMP nsHTMLMediaElement::MozLoadFrom(nsIDOMHTMLMediaElement* aOther) { NS_ENSURE_ARG_POINTER(aOther); + // Make sure we don't reenter during synchronous abort events. + if (mIsRunningLoadMethod) + return NS_OK; + mIsRunningLoadMethod = true; AbortExistingLoads(); + mIsRunningLoadMethod = false; nsCOMPtr content = do_QueryInterface(aOther); nsHTMLMediaElement* other = static_cast(content.get()); @@ -1125,7 +1200,13 @@ NS_IMETHODIMP nsHTMLMediaElement::GetSeeking(bool *aSeeking) /* attribute double currentTime; */ NS_IMETHODIMP nsHTMLMediaElement::GetCurrentTime(double *aCurrentTime) { - *aCurrentTime = mDecoder ? mDecoder->GetCurrentTime() : 0.0; + if (mStream) { + *aCurrentTime = MediaTimeToSeconds(GetMediaStream()->GetCurrentTime()); + } else if (mDecoder) { + *aCurrentTime = mDecoder->GetCurrentTime(); + } else { + *aCurrentTime = 0.0; + } return NS_OK; } @@ -1133,6 +1214,12 @@ NS_IMETHODIMP nsHTMLMediaElement::SetCurrentTime(double aCurrentTime) { StopSuspendingAfterFirstFrame(); + if (mStream) { + // do nothing since streams aren't seekable; we effectively clamp to + // the current time. + return NS_ERROR_DOM_INVALID_STATE_ERR; + } + if (!mDecoder) { LOG(PR_LOG_DEBUG, ("%p SetCurrentTime(%f) failed: no decoder", this, aCurrentTime)); return NS_ERROR_DOM_INVALID_STATE_ERR; @@ -1171,7 +1258,13 @@ NS_IMETHODIMP nsHTMLMediaElement::SetCurrentTime(double aCurrentTime) /* readonly attribute double duration; */ NS_IMETHODIMP nsHTMLMediaElement::GetDuration(double *aDuration) { - *aDuration = mDecoder ? mDecoder->GetDuration() : std::numeric_limits::quiet_NaN(); + if (mStream) { + *aDuration = std::numeric_limits::infinity(); + } else if (mDecoder) { + *aDuration = mDecoder->GetDuration(); + } else { + *aDuration = std::numeric_limits::quiet_NaN(); + } return NS_OK; } @@ -1213,6 +1306,9 @@ NS_IMETHODIMP nsHTMLMediaElement::Pause() AddRemoveSelfReference(); if (!oldPaused) { + if (mStream) { + GetMediaStream()->ChangeExplicitBlockerCount(1); + } FireTimeUpdate(false); DispatchAsyncEvent(NS_LITERAL_STRING("pause")); } @@ -1238,10 +1334,14 @@ NS_IMETHODIMP nsHTMLMediaElement::SetVolume(double aVolume) mVolume = aVolume; - if (mDecoder && !mMuted) { - mDecoder->SetVolume(mVolume); - } else if (mAudioStream && !mMuted) { - mAudioStream->SetVolume(mVolume); + if (!mMuted) { + if (mDecoder) { + mDecoder->SetVolume(mVolume); + } else if (mAudioStream) { + mAudioStream->SetVolume(mVolume); + } else if (mStream) { + GetMediaStream()->SetAudioOutputVolume(this, float(mVolume)); + } } DispatchAsyncEvent(NS_LITERAL_STRING("volumechange")); @@ -1308,10 +1408,13 @@ NS_IMETHODIMP nsHTMLMediaElement::SetMuted(bool aMuted) mMuted = aMuted; + float effectiveVolume = mMuted ? 0.0f : float(mVolume); if (mDecoder) { - mDecoder->SetVolume(mMuted ? 0.0 : mVolume); + mDecoder->SetVolume(effectiveVolume); } else if (mAudioStream) { - mAudioStream->SetVolume(mMuted ? 0.0 : mVolume); + mAudioStream->SetVolume(effectiveVolume); + } else if (mStream) { + GetMediaStream()->SetAudioOutputVolume(this, effectiveVolume); } DispatchAsyncEvent(NS_LITERAL_STRING("volumechange")); @@ -1425,6 +1528,7 @@ nsHTMLMediaElement::LookupMediaElementURITable(nsIURI* aURI) nsHTMLMediaElement::nsHTMLMediaElement(already_AddRefed aNodeInfo) : nsGenericHTMLElement(aNodeInfo), + mStreamListener(nsnull), mCurrentLoadID(0), mNetworkState(nsIDOMHTMLMediaElement::NETWORK_EMPTY), mReadyState(nsIDOMHTMLMediaElement::HAVE_NOTHING), @@ -1488,6 +1592,9 @@ nsHTMLMediaElement::~nsHTMLMediaElement() RemoveMediaElementFromURITable(); mDecoder->Shutdown(); } + if (mStream) { + EndMediaStreamPlayback(); + } NS_ASSERTION(MediaElementTableCount(this, mLoadingSrc) == 0, "Destroyed media element should no longer be in element table"); @@ -1557,6 +1664,9 @@ NS_IMETHODIMP nsHTMLMediaElement::Play() // seek to the effective start. // TODO: The playback rate must be set to the default playback rate. if (mPaused) { + if (mStream) { + GetMediaStream()->ChangeExplicitBlockerCount(-1); + } DispatchAsyncEvent(NS_LITERAL_STRING("play")); switch (mReadyState) { case nsIDOMHTMLMediaElement::HAVE_NOTHING: @@ -2110,15 +2220,7 @@ nsresult nsHTMLMediaElement::InitializeDecoderAsClone(nsMediaDecoder* aOriginal) return NS_ERROR_FAILURE; } - mNetworkState = nsIDOMHTMLMediaElement::NETWORK_LOADING; - - nsresult rv = decoder->Load(resource, nsnull, aOriginal); - if (NS_FAILED(rv)) { - LOG(PR_LOG_DEBUG, ("%p Failed to load decoder/stream for decoder %p", this, decoder.get())); - return rv; - } - - return FinishDecoderSetup(decoder); + return FinishDecoderSetup(decoder, resource, nsnull, aOriginal); } nsresult nsHTMLMediaElement::InitializeDecoderForChannel(nsIChannel *aChannel, @@ -2142,14 +2244,34 @@ nsresult nsHTMLMediaElement::InitializeDecoderForChannel(nsIChannel *aChannel, LOG(PR_LOG_DEBUG, ("%p Created decoder %p for type %s", this, decoder.get(), mimeType.get())); - mNetworkState = nsIDOMHTMLMediaElement::NETWORK_LOADING; - MediaResource* resource = MediaResource::Create(decoder, aChannel); if (!resource) return NS_ERROR_OUT_OF_MEMORY; - nsresult rv = decoder->Load(resource, aListener, nsnull); + // stream successfully created, the stream now owns the channel. + mChannel = nsnull; + + return FinishDecoderSetup(decoder, resource, aListener, nsnull); +} + +nsresult nsHTMLMediaElement::FinishDecoderSetup(nsMediaDecoder* aDecoder, + MediaResource* aStream, + nsIStreamListener **aListener, + nsMediaDecoder* aCloneDonor) +{ + mNetworkState = nsIDOMHTMLMediaElement::NETWORK_LOADING; + + // Force a same-origin check before allowing events for this media resource. + mMediaSecurityVerified = false; + + // The new stream has not been suspended by us. + mPausedForInactiveDocument = false; + + aDecoder->SetVolume(mMuted ? 0.0 : mVolume); + + nsresult rv = aDecoder->Load(aStream, aListener, aCloneDonor); if (NS_FAILED(rv)) { + LOG(PR_LOG_DEBUG, ("%p Failed to load for decoder %p", this, aDecoder)); return rv; } @@ -2157,29 +2279,13 @@ nsresult nsHTMLMediaElement::InitializeDecoderForChannel(nsIChannel *aChannel, // which owns the channel. mChannel = nsnull; - return FinishDecoderSetup(decoder); -} - -nsresult nsHTMLMediaElement::FinishDecoderSetup(nsMediaDecoder* aDecoder) -{ - NS_ASSERTION(mLoadingSrc, "mLoadingSrc set up"); - mDecoder = aDecoder; AddMediaElementToURITable(); - // Force a same-origin check before allowing events for this media resource. - mMediaSecurityVerified = false; - - // The new resource has not been suspended by us. - mPausedForInactiveDocument = false; - // But we may want to suspend it now. + // We may want to suspend the new stream now. // This will also do an AddRemoveSelfReference. NotifyOwnerDocumentActivityChanged(); - nsresult rv = NS_OK; - - mDecoder->SetVolume(mMuted ? 0.0 : mVolume); - if (!mPaused) { SetPlayedOrSeeked(true); if (!mPausedForInactiveDocument) { @@ -2204,6 +2310,128 @@ nsresult nsHTMLMediaElement::FinishDecoderSetup(nsMediaDecoder* aDecoder) return rv; } +class nsHTMLMediaElement::StreamListener : public MediaStreamListener { +public: + StreamListener(nsHTMLMediaElement* aElement) : + mElement(aElement), + mMutex("nsHTMLMediaElement::StreamListener"), + mPendingNotifyOutput(false) + {} + void Forget() { mElement = nsnull; } + + // Main thread + void DoNotifyFinished() + { + if (mElement) { + mElement->PlaybackEnded(); + } + } + void DoNotifyBlocked() + { + if (mElement) { + mElement->UpdateReadyStateForData(NEXT_FRAME_UNAVAILABLE_BUFFERING); + } + } + void DoNotifyUnblocked() + { + if (mElement) { + mElement->UpdateReadyStateForData(NEXT_FRAME_AVAILABLE); + } + } + void DoNotifyOutput() + { + { + MutexAutoLock lock(mMutex); + mPendingNotifyOutput = false; + } + if (mElement) { + mElement->FireTimeUpdate(true); + } + } + + // These notifications run on the media graph thread so we need to + // dispatch events to the main thread. + virtual void NotifyBlockingChanged(MediaStreamGraph* aGraph, Blocking aBlocked) + { + nsCOMPtr event; + if (aBlocked == BLOCKED) { + event = NS_NewRunnableMethod(this, &StreamListener::DoNotifyBlocked); + } else { + event = NS_NewRunnableMethod(this, &StreamListener::DoNotifyUnblocked); + } + aGraph->DispatchToMainThreadAfterStreamStateUpdate(event); + } + virtual void NotifyFinished(MediaStreamGraph* aGraph) + { + nsCOMPtr event = + NS_NewRunnableMethod(this, &StreamListener::DoNotifyFinished); + aGraph->DispatchToMainThreadAfterStreamStateUpdate(event); + } + virtual void NotifyOutput(MediaStreamGraph* aGraph) + { + MutexAutoLock lock(mMutex); + if (mPendingNotifyOutput) + return; + mPendingNotifyOutput = true; + nsCOMPtr event = + NS_NewRunnableMethod(this, &StreamListener::DoNotifyOutput); + aGraph->DispatchToMainThreadAfterStreamStateUpdate(event); + } + +private: + nsHTMLMediaElement* mElement; + + Mutex mMutex; + bool mPendingNotifyOutput; +}; + +void nsHTMLMediaElement::SetupMediaStreamPlayback() +{ + NS_ASSERTION(!mStream && !mStreamListener, "Should have been ended already"); + + mStream = mSrcAttrStream; + mStreamListener = new StreamListener(this); + NS_ADDREF(mStreamListener); + GetMediaStream()->AddListener(mStreamListener); + if (mPaused) { + GetMediaStream()->ChangeExplicitBlockerCount(1); + } + if (mPausedForInactiveDocument) { + GetMediaStream()->ChangeExplicitBlockerCount(1); + } + ChangeDelayLoadStatus(false); + GetMediaStream()->AddAudioOutput(this); + GetMediaStream()->SetAudioOutputVolume(this, float(mMuted ? 0.0 : mVolume)); + VideoFrameContainer* container = GetVideoFrameContainer(); + if (container) { + GetMediaStream()->AddVideoOutput(container); + } + ChangeReadyState(nsIDOMHTMLMediaElement::HAVE_METADATA); + DispatchAsyncEvent(NS_LITERAL_STRING("durationchange")); + DispatchAsyncEvent(NS_LITERAL_STRING("loadedmetadata")); + ResourceLoaded(); +} + +void nsHTMLMediaElement::EndMediaStreamPlayback() +{ + GetMediaStream()->RemoveListener(mStreamListener); + // Kill its reference to this element + mStreamListener->Forget(); + NS_RELEASE(mStreamListener); // sets to null + GetMediaStream()->RemoveAudioOutput(this); + VideoFrameContainer* container = GetVideoFrameContainer(); + if (container) { + GetMediaStream()->RemoveVideoOutput(container); + } + if (mPaused) { + GetMediaStream()->ChangeExplicitBlockerCount(-1); + } + if (mPausedForInactiveDocument) { + GetMediaStream()->ChangeExplicitBlockerCount(-1); + } + mStream = nsnull; +} + nsresult nsHTMLMediaElement::NewURIFromString(const nsAutoString& aURISpec, nsIURI** aURI) { NS_ENSURE_ARG_POINTER(aURI); @@ -2362,11 +2590,12 @@ void nsHTMLMediaElement::Error(PRUint16 aErrorCode) void nsHTMLMediaElement::PlaybackEnded() { - NS_ASSERTION(mDecoder->IsEnded(), "Decoder fired ended, but not in ended state"); - // We changed the state of IsPlaybackEnded which can affect AddRemoveSelfReference + // We changed state which can affect AddRemoveSelfReference AddRemoveSelfReference(); - if (mDecoder && mDecoder->IsInfinite()) { + NS_ASSERTION(!mDecoder || mDecoder->IsEnded(), + "Decoder fired ended, but not in ended state"); + if (mStream || (mDecoder && mDecoder->IsInfinite())) { LOG(PR_LOG_DEBUG, ("%p, got duration by reaching the end of the resource", this)); DispatchAsyncEvent(NS_LITERAL_STRING("durationchange")); } @@ -2445,6 +2674,11 @@ void nsHTMLMediaElement::UpdateReadyStateForData(NextFrameStatus aNextFrame) return; } + if (mStream) { + ChangeReadyState(nsIDOMHTMLMediaElement::HAVE_ENOUGH_DATA); + return; + } + // Now see if we should set HAVE_ENOUGH_DATA. // If it's something we don't know the size of, then we can't // make a real estimate, so we go straight to HAVE_ENOUGH_DATA once @@ -2544,6 +2778,9 @@ void nsHTMLMediaElement::NotifyAutoplayDataReady() if (mDecoder) { SetPlayedOrSeeked(true); mDecoder->Play(); + } else if (mStream) { + SetPlayedOrSeeked(true); + GetMediaStream()->ChangeExplicitBlockerCount(-1); } DispatchAsyncEvent(NS_LITERAL_STRING("play")); } @@ -2664,10 +2901,14 @@ bool nsHTMLMediaElement::IsPlaybackEnded() const already_AddRefed nsHTMLMediaElement::GetCurrentPrincipal() { - if (!mDecoder) - return nsnull; - - return mDecoder->GetCurrentPrincipal(); + if (mDecoder) { + return mDecoder->GetCurrentPrincipal(); + } + if (mStream) { + nsRefPtr principal = mStream->GetPrincipal(); + return principal.forget(); + } + return nsnull; } void nsHTMLMediaElement::UpdateMediaSize(nsIntSize size) @@ -2683,17 +2924,23 @@ void nsHTMLMediaElement::NotifyOwnerDocumentActivityChanged() if (pauseForInactiveDocument != mPausedForInactiveDocument) { mPausedForInactiveDocument = pauseForInactiveDocument; - if (mDecoder) { - if (pauseForInactiveDocument) { + if (pauseForInactiveDocument) { + if (mDecoder) { mDecoder->Pause(); mDecoder->Suspend(); - } else { + } else if (mStream) { + GetMediaStream()->ChangeExplicitBlockerCount(1); + } + } else { + if (mDecoder) { mDecoder->Resume(false); - DispatchPendingMediaEvents(); if (!mPaused && !mDecoder->IsEnded()) { mDecoder->Play(); } + } else if (mStream) { + GetMediaStream()->ChangeExplicitBlockerCount(-1); } + DispatchPendingMediaEvents(); } } @@ -2715,6 +2962,7 @@ void nsHTMLMediaElement::AddRemoveSelfReference() ownerDoc->IsActive() && (mDelayingLoadEvent || (!mPaused && mDecoder && !mDecoder->IsEnded()) || + (!mPaused && mStream && !mStream->IsFinished()) || (mDecoder && mDecoder->IsSeeking()) || CanActivateAutoplay() || mNetworkState == nsIDOMHTMLMediaElement::NETWORK_LOADING); @@ -2948,7 +3196,7 @@ void nsHTMLMediaElement::FireTimeUpdate(bool aPeriodic) double time = 0; GetCurrentTime(&time); - // Fire a timupdate event if this is not a periodic update (i.e. it's a + // Fire a timeupdate event if this is not a periodic update (i.e. it's a // timeupdate event mandated by the spec), or if it's a periodic update // and TIMEUPDATE_MS has passed since the last timeupdate event fired and // the time has changed. diff --git a/dom/interfaces/html/nsIDOMHTMLAudioElement.idl b/dom/interfaces/html/nsIDOMHTMLAudioElement.idl index ead49377877..6869fa3a5f9 100644 --- a/dom/interfaces/html/nsIDOMHTMLAudioElement.idl +++ b/dom/interfaces/html/nsIDOMHTMLAudioElement.idl @@ -52,7 +52,7 @@ * @status UNDER_DEVELOPMENT */ -[scriptable, uuid(D5844B73-30E2-46D5-894C-108967E05C80)] +[scriptable, uuid(e1a11e83-255b-4350-81cf-f1f3e7d59712)] interface nsIDOMHTMLAudioElement : nsIDOMHTMLMediaElement { // Setup the audio stream for writing diff --git a/dom/interfaces/html/nsIDOMHTMLMediaElement.idl b/dom/interfaces/html/nsIDOMHTMLMediaElement.idl index 9c5edf58abb..b3cf00c55b2 100644 --- a/dom/interfaces/html/nsIDOMHTMLMediaElement.idl +++ b/dom/interfaces/html/nsIDOMHTMLMediaElement.idl @@ -57,14 +57,14 @@ #endif %} -[scriptable, uuid(02FB205D-68B5-4722-982B-1D12238FBF72)] +[scriptable, uuid(3e672e79-a0ea-45ef-87de-828402f1f6d7)] interface nsIDOMHTMLMediaElement : nsIDOMHTMLElement { // error state readonly attribute nsIDOMMediaError error; // network state - attribute DOMString src; + [implicit_jscontext] attribute jsval src; readonly attribute DOMString currentSrc; attribute DOMString crossorigin; const unsigned short NETWORK_EMPTY = 0; diff --git a/dom/interfaces/html/nsIDOMHTMLVideoElement.idl b/dom/interfaces/html/nsIDOMHTMLVideoElement.idl index a1734b3b523..f93c0735909 100644 --- a/dom/interfaces/html/nsIDOMHTMLVideoElement.idl +++ b/dom/interfaces/html/nsIDOMHTMLVideoElement.idl @@ -48,7 +48,7 @@ * @status UNDER_DEVELOPMENT */ -[scriptable, uuid(0a1c8b44-12a4-4316-b39f-feeee48475f8)] +[scriptable, uuid(e1a11e83-255b-4350-81cf-f1f3e7d59712)] interface nsIDOMHTMLVideoElement : nsIDOMHTMLMediaElement { attribute long width; From 9eef1b9f614ac7db97fe2039df2aa2a9a65f5259 Mon Sep 17 00:00:00 2001 From: Robert O'Callahan Date: Mon, 30 Apr 2012 15:12:42 +1200 Subject: [PATCH 014/162] Bug 664918. Part 8: Add mozCaptureStream()/mozCaptureStreamUntilEnded() APIs to HTML media elements, returning a MediaStream representing the contents of the media element. r=cpearce,jesup This is currently not fully functional. The MediaStream always ends when the underlying resource ends. You can't use these APIs on a media element whose src is a MediaStream. Seeking or pausing the resource will cause problems. The media element does not play back in sync with the MediaStream. --- .../html/content/public/nsHTMLMediaElement.h | 25 +- .../html/content/src/nsHTMLMediaElement.cpp | 62 +++ content/media/MediaResource.cpp | 8 + content/media/MediaResource.h | 2 + .../media/nsAudioAvailableEventManager.cpp | 20 +- content/media/nsBuiltinDecoder.cpp | 31 ++ content/media/nsBuiltinDecoder.h | 56 +++ content/media/nsBuiltinDecoderReader.cpp | 25 +- content/media/nsBuiltinDecoderReader.h | 40 +- .../media/nsBuiltinDecoderStateMachine.cpp | 375 ++++++++++++++++-- content/media/nsBuiltinDecoderStateMachine.h | 32 ++ content/media/nsMediaCache.cpp | 25 +- content/media/nsMediaCache.h | 22 +- content/media/nsMediaDecoder.h | 17 +- .../html/nsIDOMHTMLAudioElement.idl | 2 +- .../html/nsIDOMHTMLMediaElement.idl | 9 +- .../html/nsIDOMHTMLVideoElement.idl | 2 +- 17 files changed, 691 insertions(+), 62 deletions(-) diff --git a/content/html/content/public/nsHTMLMediaElement.h b/content/html/content/public/nsHTMLMediaElement.h index 8f11d18a318..c9484fd00be 100644 --- a/content/html/content/public/nsHTMLMediaElement.h +++ b/content/html/content/public/nsHTMLMediaElement.h @@ -265,6 +265,9 @@ public: // Returns null if nothing is playing. already_AddRefed GetCurrentPrincipal(); + // called to notify that the principal of the decoder's media resource has changed. + void NotifyDecoderPrincipalChanged(); + // Update the visual size of the media. Called from the decoder on the // main thread when/if the size changes. void UpdateMediaSize(nsIntSize size); @@ -416,6 +419,15 @@ protected: */ void EndMediaStreamPlayback(); + /** + * Returns an nsDOMMediaStream containing the played contents of this + * element. When aFinishWhenEnded is true, when this element ends playback + * we will finish the stream and not play any more into it. + * When aFinishWhenEnded is false, ending playback does not finish the stream. + * The stream will never finish. + */ + already_AddRefed CaptureStreamInternal(bool aFinishWhenEnded); + /** * Create a decoder for the given aMIMEType. Returns null if we * were unable to create the decoder. @@ -633,6 +645,14 @@ protected: // At most one of mDecoder and mStream can be non-null. nsRefPtr mStream; + // Holds references to the DOM wrappers for the MediaStreams that we're + // writing to. + struct OutputMediaStream { + nsRefPtr mStream; + bool mFinishWhenEnded; + }; + nsTArray mOutputStreams; + // Holds a reference to the MediaStreamListener attached to mStream. STRONG! StreamListener* mStreamListener; @@ -769,9 +789,12 @@ protected: // 'Pause' method, or playback not yet having started. bool mPaused; - // True if the sound is muted + // True if the sound is muted. bool mMuted; + // True if the sound is being captured. + bool mAudioCaptured; + // If TRUE then the media element was actively playing before the currently // in progress seeking. If FALSE then the media element is either not seeking // or was not actively playing before the current seek. Used to decide whether diff --git a/content/html/content/src/nsHTMLMediaElement.cpp b/content/html/content/src/nsHTMLMediaElement.cpp index b96adc4aea9..8fbb6da2940 100644 --- a/content/html/content/src/nsHTMLMediaElement.cpp +++ b/content/html/content/src/nsHTMLMediaElement.cpp @@ -427,6 +427,9 @@ NS_IMPL_CYCLE_COLLECTION_TRAVERSE_BEGIN_INHERITED(nsHTMLMediaElement, nsGenericH NS_IMPL_CYCLE_COLLECTION_TRAVERSE_NSCOMPTR(mSourcePointer) NS_IMPL_CYCLE_COLLECTION_TRAVERSE_NSCOMPTR(mLoadBlockedDoc) NS_IMPL_CYCLE_COLLECTION_TRAVERSE_NSCOMPTR(mSourceLoadCandidate) + for (PRUint32 i = 0; i < tmp->mOutputStreams.Length(); ++i) { + NS_IMPL_CYCLE_COLLECTION_TRAVERSE_NSCOMPTR(mOutputStreams[i].mStream); + } NS_IMPL_CYCLE_COLLECTION_TRAVERSE_END NS_IMPL_CYCLE_COLLECTION_UNLINK_BEGIN_INHERITED(nsHTMLMediaElement, nsGenericHTMLElement) @@ -439,6 +442,9 @@ NS_IMPL_CYCLE_COLLECTION_UNLINK_BEGIN_INHERITED(nsHTMLMediaElement, nsGenericHTM NS_IMPL_CYCLE_COLLECTION_UNLINK_NSCOMPTR(mSourcePointer) NS_IMPL_CYCLE_COLLECTION_UNLINK_NSCOMPTR(mLoadBlockedDoc) NS_IMPL_CYCLE_COLLECTION_UNLINK_NSCOMPTR(mSourceLoadCandidate) + for (PRUint32 i = 0; i < tmp->mOutputStreams.Length(); ++i) { + NS_IMPL_CYCLE_COLLECTION_UNLINK_NSCOMPTR(mOutputStreams[i].mStream); + } NS_IMPL_CYCLE_COLLECTION_UNLINK_END NS_INTERFACE_MAP_BEGIN_CYCLE_COLLECTION_INHERITED(nsHTMLMediaElement) @@ -1422,6 +1428,43 @@ NS_IMETHODIMP nsHTMLMediaElement::SetMuted(bool aMuted) return NS_OK; } +already_AddRefed +nsHTMLMediaElement::CaptureStreamInternal(bool aFinishWhenEnded) +{ + OutputMediaStream* out = mOutputStreams.AppendElement(); + out->mStream = nsDOMMediaStream::CreateInputStream(); + nsRefPtr principal = GetCurrentPrincipal(); + out->mStream->CombineWithPrincipal(principal); + out->mFinishWhenEnded = aFinishWhenEnded; + + mAudioCaptured = true; + if (mDecoder) { + mDecoder->SetAudioCaptured(true); + mDecoder->AddOutputStream( + out->mStream->GetStream()->AsSourceStream(), aFinishWhenEnded); + } + nsRefPtr result = out->mStream; + return result.forget(); +} + +NS_IMETHODIMP nsHTMLMediaElement::MozCaptureStream(nsIDOMMediaStream** aStream) +{ + *aStream = CaptureStreamInternal(false).get(); + return NS_OK; +} + +NS_IMETHODIMP nsHTMLMediaElement::MozCaptureStreamUntilEnded(nsIDOMMediaStream** aStream) +{ + *aStream = CaptureStreamInternal(true).get(); + return NS_OK; +} + +NS_IMETHODIMP nsHTMLMediaElement::GetMozAudioCaptured(bool *aCaptured) +{ + *aCaptured = mAudioCaptured; + return NS_OK; +} + class MediaElementSetForURI : public nsURIHashKey { public: MediaElementSetForURI(const nsIURI* aKey) : nsURIHashKey(aKey) {} @@ -1548,6 +1591,7 @@ nsHTMLMediaElement::nsHTMLMediaElement(already_AddRefed aNodeInfo) mAutoplayEnabled(true), mPaused(true), mMuted(false), + mAudioCaptured(false), mPlayingBeforeSeek(false), mPausedForInactiveDocument(false), mWaitingFired(false), @@ -2267,7 +2311,13 @@ nsresult nsHTMLMediaElement::FinishDecoderSetup(nsMediaDecoder* aDecoder, // The new stream has not been suspended by us. mPausedForInactiveDocument = false; + aDecoder->SetAudioCaptured(mAudioCaptured); aDecoder->SetVolume(mMuted ? 0.0 : mVolume); + for (PRUint32 i = 0; i < mOutputStreams.Length(); ++i) { + OutputMediaStream* ms = &mOutputStreams[i]; + aDecoder->AddOutputStream(ms->mStream->GetStream()->AsSourceStream(), + ms->mFinishWhenEnded); + } nsresult rv = aDecoder->Load(aStream, aListener, aCloneDonor); if (NS_FAILED(rv)) { @@ -2281,6 +2331,7 @@ nsresult nsHTMLMediaElement::FinishDecoderSetup(nsMediaDecoder* aDecoder, mDecoder = aDecoder; AddMediaElementToURITable(); + NotifyDecoderPrincipalChanged(); // We may want to suspend the new stream now. // This will also do an AddRemoveSelfReference. @@ -2390,6 +2441,8 @@ void nsHTMLMediaElement::SetupMediaStreamPlayback() NS_ASSERTION(!mStream && !mStreamListener, "Should have been ended already"); mStream = mSrcAttrStream; + // XXX if we ever support capturing the output of a media element which is + // playing a stream, we'll need to add a CombineWithPrincipal call here. mStreamListener = new StreamListener(this); NS_ADDREF(mStreamListener); GetMediaStream()->AddListener(mStreamListener); @@ -2911,6 +2964,15 @@ already_AddRefed nsHTMLMediaElement::GetCurrentPrincipal() return nsnull; } +void nsHTMLMediaElement::NotifyDecoderPrincipalChanged() +{ + for (PRUint32 i = 0; i < mOutputStreams.Length(); ++i) { + OutputMediaStream* ms = &mOutputStreams[i]; + nsRefPtr principal = GetCurrentPrincipal(); + ms->mStream->CombineWithPrincipal(principal); + } +} + void nsHTMLMediaElement::UpdateMediaSize(nsIntSize size) { mMediaSize = size; diff --git a/content/media/MediaResource.cpp b/content/media/MediaResource.cpp index 43ee9e81b11..a7ca48134c2 100644 --- a/content/media/MediaResource.cpp +++ b/content/media/MediaResource.cpp @@ -762,6 +762,14 @@ ChannelMediaResource::CacheClientNotifyDataEnded(nsresult aStatus) NS_DispatchToMainThread(event, NS_DISPATCH_NORMAL); } +void +ChannelMediaResource::CacheClientNotifyPrincipalChanged() +{ + NS_ASSERTION(NS_IsMainThread(), "Don't call on non-main thread"); + + mDecoder->NotifyPrincipalChanged(); +} + nsresult ChannelMediaResource::CacheClientSeek(PRInt64 aOffset, bool aResume) { diff --git a/content/media/MediaResource.h b/content/media/MediaResource.h index 832462ef415..72bc6e9ad1f 100644 --- a/content/media/MediaResource.h +++ b/content/media/MediaResource.h @@ -375,6 +375,8 @@ public: // if this stream didn't read any data, since another stream might have // received data for the same resource. void CacheClientNotifyDataEnded(nsresult aStatus); + // Notify that the principal for the cached resource changed. + void CacheClientNotifyPrincipalChanged(); // These are called on the main thread by nsMediaCache. These shouldn't block, // but they may grab locks --- the media cache is not holding its lock diff --git a/content/media/nsAudioAvailableEventManager.cpp b/content/media/nsAudioAvailableEventManager.cpp index 9fc35b83edc..10e4a7d2543 100644 --- a/content/media/nsAudioAvailableEventManager.cpp +++ b/content/media/nsAudioAvailableEventManager.cpp @@ -161,10 +161,16 @@ void nsAudioAvailableEventManager::QueueWrittenAudioData(AudioDataValue* aAudioD // Fill the signalBuffer. PRUint32 i; float *signalBuffer = mSignalBuffer.get() + mSignalBufferPosition; - for (i = 0; i < signalBufferTail; ++i) { - signalBuffer[i] = MOZ_CONVERT_AUDIO_SAMPLE(audioData[i]); + if (audioData) { + for (i = 0; i < signalBufferTail; ++i) { + signalBuffer[i] = MOZ_CONVERT_AUDIO_SAMPLE(audioData[i]); + } + } else { + memset(signalBuffer, 0, signalBufferTail*sizeof(signalBuffer[0])); + } + if (audioData) { + audioData += signalBufferTail; } - audioData += signalBufferTail; NS_ASSERTION(audioDataLength >= signalBufferTail, "audioDataLength about to wrap past zero to +infinity!"); @@ -204,8 +210,12 @@ void nsAudioAvailableEventManager::QueueWrittenAudioData(AudioDataValue* aAudioD // Add data to the signalBuffer. PRUint32 i; float *signalBuffer = mSignalBuffer.get() + mSignalBufferPosition; - for (i = 0; i < audioDataLength; ++i) { - signalBuffer[i] = MOZ_CONVERT_AUDIO_SAMPLE(audioData[i]); + if (audioData) { + for (i = 0; i < audioDataLength; ++i) { + signalBuffer[i] = MOZ_CONVERT_AUDIO_SAMPLE(audioData[i]); + } + } else { + memset(signalBuffer, 0, audioDataLength*sizeof(signalBuffer[0])); } mSignalBufferPosition += audioDataLength; } diff --git a/content/media/nsBuiltinDecoder.cpp b/content/media/nsBuiltinDecoder.cpp index dc0af2eb8d2..a9352e5f5a1 100644 --- a/content/media/nsBuiltinDecoder.cpp +++ b/content/media/nsBuiltinDecoder.cpp @@ -82,6 +82,30 @@ void nsBuiltinDecoder::SetVolume(double aVolume) } } +void nsBuiltinDecoder::SetAudioCaptured(bool aCaptured) +{ + NS_ASSERTION(NS_IsMainThread(), "Should be on main thread."); + mInitialAudioCaptured = aCaptured; + if (mDecoderStateMachine) { + mDecoderStateMachine->SetAudioCaptured(aCaptured); + } +} + +void nsBuiltinDecoder::AddOutputStream(SourceMediaStream* aStream, bool aFinishWhenEnded) +{ + NS_ASSERTION(NS_IsMainThread(), "Should be on main thread."); + + { + ReentrantMonitorAutoEnter mon(mReentrantMonitor); + OutputMediaStream* ms = mOutputStreams.AppendElement(); + ms->Init(PRInt64(mCurrentTime*USECS_PER_S), aStream, aFinishWhenEnded); + } + + // Make sure the state machine thread runs so that any buffered data + // is fed into our strema. + ScheduleStateMachineThread(); +} + double nsBuiltinDecoder::GetDuration() { NS_ASSERTION(NS_IsMainThread(), "Should be on main thread."); @@ -692,6 +716,13 @@ void nsBuiltinDecoder::NotifyDownloadEnded(nsresult aStatus) UpdateReadyStateForData(); } +void nsBuiltinDecoder::NotifyPrincipalChanged() +{ + if (mElement) { + mElement->NotifyDecoderPrincipalChanged(); + } +} + void nsBuiltinDecoder::NotifyBytesConsumed(PRInt64 aBytes) { ReentrantMonitorAutoEnter mon(mReentrantMonitor); diff --git a/content/media/nsBuiltinDecoder.h b/content/media/nsBuiltinDecoder.h index e1e44f43399..71e6e4d6a3e 100644 --- a/content/media/nsBuiltinDecoder.h +++ b/content/media/nsBuiltinDecoder.h @@ -262,6 +262,7 @@ public: // Set the audio volume. The decoder monitor must be obtained before // calling this. virtual void SetVolume(double aVolume) = 0; + virtual void SetAudioCaptured(bool aCapture) = 0; virtual void Shutdown() = 0; @@ -397,6 +398,54 @@ public: virtual void Pause(); virtual void SetVolume(double aVolume); + virtual void SetAudioCaptured(bool aCaptured); + + virtual void AddOutputStream(SourceMediaStream* aStream, bool aFinishWhenEnded); + // Protected by mReentrantMonitor. All decoder output is copied to these streams. + struct OutputMediaStream { + void Init(PRInt64 aInitialTime, SourceMediaStream* aStream, bool aFinishWhenEnded) + { + mLastAudioPacketTime = -1; + mLastAudioPacketEndTime = -1; + mAudioFramesWrittenBaseTime = aInitialTime; + mAudioFramesWritten = 0; + mNextVideoTime = aInitialTime; + mStream = aStream; + mStreamInitialized = false; + mFinishWhenEnded = aFinishWhenEnded; + mHaveSentFinish = false; + mHaveSentFinishAudio = false; + mHaveSentFinishVideo = false; + } + PRInt64 mLastAudioPacketTime; // microseconds + PRInt64 mLastAudioPacketEndTime; // microseconds + // Count of audio frames written to the stream + PRInt64 mAudioFramesWritten; + // Timestamp of the first audio packet whose frames we wrote. + PRInt64 mAudioFramesWrittenBaseTime; // microseconds + // mNextVideoTime is the end timestamp for the last packet sent to the stream. + // Therefore video packets starting at or after this time need to be copied + // to the output stream. + PRInt64 mNextVideoTime; // microseconds + // The last video image sent to the stream. Useful if we need to replicate + // the image. + nsRefPtr mLastVideoImage; + nsRefPtr mStream; + gfxIntSize mLastVideoImageDisplaySize; + // This is set to true when the stream is initialized (audio and + // video tracks added). + bool mStreamInitialized; + bool mFinishWhenEnded; + bool mHaveSentFinish; + bool mHaveSentFinishAudio; + bool mHaveSentFinishVideo; + }; + nsTArray& OutputStreams() + { + GetReentrantMonitor().AssertCurrentThreadIn(); + return mOutputStreams; + } + virtual double GetDuration(); virtual void SetInfinite(bool aInfinite); @@ -408,6 +457,7 @@ public: virtual void NotifySuspendedStatusChanged(); virtual void NotifyBytesDownloaded(); virtual void NotifyDownloadEnded(nsresult aStatus); + virtual void NotifyPrincipalChanged(); // Called by the decode thread to keep track of the number of bytes read // from the resource. void NotifyBytesConsumed(PRInt64 aBytes); @@ -663,6 +713,9 @@ public: // only. PRInt64 mDuration; + // True when playback should start with audio captured (not playing). + bool mInitialAudioCaptured; + // True if the media resource is seekable (server supports byte range // requests). bool mSeekable; @@ -686,6 +739,9 @@ public: // state change. ReentrantMonitor mReentrantMonitor; + // Data about MediaStreams that are being fed by this decoder. + nsTArray mOutputStreams; + // Set to one of the valid play states. It is protected by the // monitor mReentrantMonitor. This monitor must be acquired when reading or // writing the state. Any change to the state on the main thread diff --git a/content/media/nsBuiltinDecoderReader.cpp b/content/media/nsBuiltinDecoderReader.cpp index e370b270e03..782c5a2baf1 100644 --- a/content/media/nsBuiltinDecoderReader.cpp +++ b/content/media/nsBuiltinDecoderReader.cpp @@ -71,6 +71,21 @@ extern PRLogModuleInfo* gBuiltinDecoderLog; #define SEEK_LOG(type, msg) #endif +void +AudioData::EnsureAudioBuffer() +{ + if (mAudioBuffer) + return; + mAudioBuffer = SharedBuffer::Create(mFrames*mChannels*sizeof(AudioDataValue)); + + AudioDataValue* data = static_cast(mAudioBuffer->Data()); + for (PRUint32 i = 0; i < mFrames; ++i) { + for (PRUint32 j = 0; j < mChannels; ++j) { + data[j*mFrames + i] = mAudioData[i*mChannels + j]; + } + } +} + static bool ValidatePlane(const VideoData::YCbCrBuffer::Plane& aPlane) { @@ -115,7 +130,15 @@ VideoData* VideoData::Create(nsVideoInfo& aInfo, nsIntRect aPicture) { if (!aContainer) { - return nsnull; + // Create a dummy VideoData with no image. This gives us something to + // send to media streams if necessary. + nsAutoPtr v(new VideoData(aOffset, + aTime, + aEndTime, + aKeyframe, + aTimecode, + aInfo.mDisplay)); + return v.forget(); } // The following situation should never happen unless there is a bug diff --git a/content/media/nsBuiltinDecoderReader.h b/content/media/nsBuiltinDecoderReader.h index fe8b498aceb..0fb2d675202 100644 --- a/content/media/nsBuiltinDecoderReader.h +++ b/content/media/nsBuiltinDecoderReader.h @@ -43,13 +43,15 @@ #include "ImageLayers.h" #include "nsSize.h" #include "mozilla/ReentrantMonitor.h" +#include "MediaStreamGraph.h" +#include "SharedBuffer.h" // Stores info relevant to presenting media frames. class nsVideoInfo { public: nsVideoInfo() - : mAudioRate(0), - mAudioChannels(0), + : mAudioRate(44100), + mAudioChannels(2), mDisplay(0,0), mStereoMode(mozilla::layers::STEREO_MODE_MONO), mHasAudio(false), @@ -113,6 +115,8 @@ typedef float AudioDataValue; // Holds chunk a decoded audio frames. class AudioData { public: + typedef mozilla::SharedBuffer SharedBuffer; + AudioData(PRInt64 aOffset, PRInt64 aTime, PRInt64 aDuration, @@ -134,6 +138,11 @@ public: MOZ_COUNT_DTOR(AudioData); } + // If mAudioBuffer is null, creates it from mAudioData. + void EnsureAudioBuffer(); + + PRInt64 GetEnd() { return mTime + mDuration; } + // Approximate byte offset of the end of the page on which this chunk // ends. const PRInt64 mOffset; @@ -142,6 +151,10 @@ public: const PRInt64 mDuration; // In usecs. const PRUint32 mFrames; const PRUint32 mChannels; + // At least one of mAudioBuffer/mAudioData must be non-null. + // mChannels channels, each with mFrames frames + nsRefPtr mAudioBuffer; + // mFrames frames, each with mChannels values nsAutoArrayPtr mAudioData; }; @@ -198,6 +211,8 @@ public: MOZ_COUNT_DTOR(VideoData); } + PRInt64 GetEnd() { return mEndTime; } + // Dimensions at which to display the video frame. The picture region // will be scaled to this size. This is should be the picture region's // dimensions scaled with respect to its aspect ratio. @@ -370,6 +385,25 @@ template class MediaQueue : private nsDeque { ForEach(aFunctor); } + // Extracts elements from the queue into aResult, in order. + // Elements whose start time is before aTime are ignored. + void GetElementsAfter(PRInt64 aTime, nsTArray* aResult) { + ReentrantMonitorAutoEnter mon(mReentrantMonitor); + if (!GetSize()) + return; + PRInt32 i; + for (i = GetSize() - 1; i > 0; --i) { + T* v = static_cast(ObjectAt(i)); + if (v->GetEnd() < aTime) + break; + } + // Elements less than i have a end time before aTime. It's also possible + // that the element at i has a end time before aTime, but that's OK. + for (; i < GetSize(); ++i) { + aResult->AppendElement(static_cast(ObjectAt(i))); + } + } + private: mutable ReentrantMonitor mReentrantMonitor; @@ -408,7 +442,7 @@ public: // than aTimeThreshold will be decoded (unless they're not keyframes // and aKeyframeSkip is true), but will not be added to the queue. virtual bool DecodeVideoFrame(bool &aKeyframeSkip, - PRInt64 aTimeThreshold) = 0; + PRInt64 aTimeThreshold) = 0; virtual bool HasAudio() = 0; virtual bool HasVideo() = 0; diff --git a/content/media/nsBuiltinDecoderStateMachine.cpp b/content/media/nsBuiltinDecoderStateMachine.cpp index 2881398d8d3..712bf3fc8e5 100644 --- a/content/media/nsBuiltinDecoderStateMachine.cpp +++ b/content/media/nsBuiltinDecoderStateMachine.cpp @@ -46,6 +46,8 @@ #include "VideoUtils.h" #include "nsTimeRanges.h" #include "nsDeque.h" +#include "AudioSegment.h" +#include "VideoSegment.h" #include "mozilla/Preferences.h" #include "mozilla/StandardInteger.h" @@ -420,6 +422,7 @@ nsBuiltinDecoderStateMachine::nsBuiltinDecoderStateMachine(nsBuiltinDecoder* aDe mAudioEndTime(-1), mVideoFrameEndTime(-1), mVolume(1.0), + mAudioCaptured(false), mSeekable(true), mPositionChangeQueued(false), mAudioCompleted(false), @@ -434,6 +437,8 @@ nsBuiltinDecoderStateMachine::nsBuiltinDecoderStateMachine(nsBuiltinDecoder* aDe mDecodeThreadWaiting(false), mRealTime(aRealTime), mRequestedNewDecodeThread(false), + mDidThrottleAudioDecoding(false), + mDidThrottleVideoDecoding(false), mEventManager(aDecoder) { MOZ_COUNT_CTOR(nsBuiltinDecoderStateMachine); @@ -521,6 +526,276 @@ void nsBuiltinDecoderStateMachine::DecodeThreadRun() LOG(PR_LOG_DEBUG, ("%p Decode thread finished", mDecoder.get())); } +void nsBuiltinDecoderStateMachine::SendOutputStreamAudio(AudioData* aAudio, + OutputMediaStream* aStream, + AudioSegment* aOutput) +{ + mDecoder->GetReentrantMonitor().AssertCurrentThreadIn(); + + if (aAudio->mTime <= aStream->mLastAudioPacketTime) { + // ignore packet that we've already processed + return; + } + aStream->mLastAudioPacketTime = aAudio->mTime; + aStream->mLastAudioPacketEndTime = aAudio->GetEnd(); + + NS_ASSERTION(aOutput->GetChannels() == aAudio->mChannels, + "Wrong number of channels"); + + // This logic has to mimic AudioLoop closely to make sure we write + // the exact same silences + CheckedInt64 audioWrittenOffset = UsecsToFrames(mInfo.mAudioRate, + aStream->mAudioFramesWrittenBaseTime + mStartTime) + aStream->mAudioFramesWritten; + CheckedInt64 frameOffset = UsecsToFrames(mInfo.mAudioRate, aAudio->mTime); + if (!audioWrittenOffset.valid() || !frameOffset.valid()) + return; + if (audioWrittenOffset.value() < frameOffset.value()) { + // Write silence to catch up + LOG(PR_LOG_DEBUG, ("%p Decoder writing %d frames of silence to MediaStream", + mDecoder.get(), PRInt32(frameOffset.value() - audioWrittenOffset.value()))); + AudioSegment silence; + silence.InitFrom(*aOutput); + silence.InsertNullDataAtStart(frameOffset.value() - audioWrittenOffset.value()); + aStream->mAudioFramesWritten += silence.GetDuration(); + aOutput->AppendFrom(&silence); + } + + PRInt64 offset; + if (aStream->mAudioFramesWritten == 0) { + NS_ASSERTION(frameOffset.value() <= audioWrittenOffset.value(), + "Otherwise we'd have taken the write-silence path"); + // We're starting in the middle of a packet. Split the packet. + offset = audioWrittenOffset.value() - frameOffset.value(); + } else { + // Write the entire packet. + offset = 0; + } + + if (offset >= aAudio->mFrames) + return; + + aAudio->EnsureAudioBuffer(); + nsRefPtr buffer = aAudio->mAudioBuffer; + aOutput->AppendFrames(buffer.forget(), aAudio->mFrames, PRInt32(offset), aAudio->mFrames, + MOZ_AUDIO_DATA_FORMAT); + LOG(PR_LOG_DEBUG, ("%p Decoder writing %d frames of data to MediaStream for AudioData at %lld", + mDecoder.get(), aAudio->mFrames - PRInt32(offset), aAudio->mTime)); + aStream->mAudioFramesWritten += aAudio->mFrames - PRInt32(offset); +} + +static void WriteVideoToMediaStream(Image* aImage, + PRInt64 aDuration, const gfxIntSize& aIntrinsicSize, + VideoSegment* aOutput) +{ + nsRefPtr image = aImage; + aOutput->AppendFrame(image.forget(), aDuration, aIntrinsicSize); +} + +static const TrackID TRACK_AUDIO = 1; +static const TrackID TRACK_VIDEO = 2; +static const TrackRate RATE_VIDEO = USECS_PER_S; + +void nsBuiltinDecoderStateMachine::SendOutputStreamData() +{ + mDecoder->GetReentrantMonitor().AssertCurrentThreadIn(); + + if (mState == DECODER_STATE_DECODING_METADATA) + return; + + nsTArray& streams = mDecoder->OutputStreams(); + PRInt64 minLastAudioPacketTime = PR_INT64_MAX; + + bool finished = + (!mInfo.mHasAudio || mReader->mAudioQueue.IsFinished()) && + (!mInfo.mHasVideo || mReader->mVideoQueue.IsFinished()); + + for (PRUint32 i = 0; i < streams.Length(); ++i) { + OutputMediaStream* stream = &streams[i]; + SourceMediaStream* mediaStream = stream->mStream; + StreamTime endPosition = 0; + + if (!stream->mStreamInitialized) { + if (mInfo.mHasAudio) { + AudioSegment* audio = new AudioSegment(); + audio->Init(mInfo.mAudioChannels); + mediaStream->AddTrack(TRACK_AUDIO, mInfo.mAudioRate, 0, audio); + } + if (mInfo.mHasVideo) { + VideoSegment* video = new VideoSegment(); + mediaStream->AddTrack(TRACK_VIDEO, RATE_VIDEO, 0, video); + } + stream->mStreamInitialized = true; + } + + if (mInfo.mHasAudio) { + nsAutoTArray audio; + // It's OK to hold references to the AudioData because while audio + // is captured, only the decoder thread pops from the queue (see below). + mReader->mAudioQueue.GetElementsAfter(stream->mLastAudioPacketTime, &audio); + AudioSegment output; + output.Init(mInfo.mAudioChannels); + for (PRUint32 i = 0; i < audio.Length(); ++i) { + AudioData* a = audio[i]; + SendOutputStreamAudio(audio[i], stream, &output); + } + if (output.GetDuration() > 0) { + mediaStream->AppendToTrack(TRACK_AUDIO, &output); + } + if (mReader->mAudioQueue.IsFinished() && !stream->mHaveSentFinishAudio) { + mediaStream->EndTrack(TRACK_AUDIO); + stream->mHaveSentFinishAudio = true; + } + minLastAudioPacketTime = NS_MIN(minLastAudioPacketTime, stream->mLastAudioPacketTime); + endPosition = NS_MAX(endPosition, + TicksToTimeRoundDown(mInfo.mAudioRate, stream->mAudioFramesWritten)); + } + + if (mInfo.mHasVideo) { + nsAutoTArray video; + // It's OK to hold references to the VideoData only the decoder thread + // pops from the queue. + mReader->mVideoQueue.GetElementsAfter(stream->mNextVideoTime + mStartTime, &video); + VideoSegment output; + for (PRUint32 i = 0; i < video.Length(); ++i) { + VideoData* v = video[i]; + if (stream->mNextVideoTime + mStartTime < v->mTime) { + LOG(PR_LOG_DEBUG, ("%p Decoder writing last video to MediaStream for %lld ms", + mDecoder.get(), v->mTime - (stream->mNextVideoTime + mStartTime))); + // Write last video frame to catch up. mLastVideoImage can be null here + // which is fine, it just means there's no video. + WriteVideoToMediaStream(stream->mLastVideoImage, + v->mTime - (stream->mNextVideoTime + mStartTime), stream->mLastVideoImageDisplaySize, + &output); + stream->mNextVideoTime = v->mTime - mStartTime; + } + if (stream->mNextVideoTime + mStartTime < v->mEndTime) { + LOG(PR_LOG_DEBUG, ("%p Decoder writing video frame %lld to MediaStream", + mDecoder.get(), v->mTime)); + WriteVideoToMediaStream(v->mImage, + v->mEndTime - (stream->mNextVideoTime + mStartTime), v->mDisplay, + &output); + stream->mNextVideoTime = v->mEndTime - mStartTime; + stream->mLastVideoImage = v->mImage; + stream->mLastVideoImageDisplaySize = v->mDisplay; + } else { + LOG(PR_LOG_DEBUG, ("%p Decoder skipping writing video frame %lld to MediaStream", + mDecoder.get(), v->mTime)); + } + } + if (output.GetDuration() > 0) { + mediaStream->AppendToTrack(TRACK_VIDEO, &output); + } + if (mReader->mVideoQueue.IsFinished() && !stream->mHaveSentFinishVideo) { + mediaStream->EndTrack(TRACK_VIDEO); + stream->mHaveSentFinishVideo = true; + } + endPosition = NS_MAX(endPosition, + TicksToTimeRoundDown(RATE_VIDEO, stream->mNextVideoTime)); + } + + if (!stream->mHaveSentFinish) { + stream->mStream->AdvanceKnownTracksTime(endPosition); + } + + if (finished && !stream->mHaveSentFinish) { + stream->mHaveSentFinish = true; + stream->mStream->Finish(); + } + } + + if (mAudioCaptured) { + // Discard audio packets that are no longer needed. + PRInt64 audioPacketTimeToDiscard = + NS_MIN(minLastAudioPacketTime, mStartTime + mCurrentFrameTime); + while (true) { + nsAutoPtr a(mReader->mAudioQueue.PopFront()); + if (!a) + break; + // Packet times are not 100% reliable so this may discard packets that + // actually contain data for mCurrentFrameTime. This means if someone might + // create a new output stream and we actually don't have the audio for the + // very start. That's OK, we'll play silence instead for a brief moment. + // That's OK. Seeking to this time would have a similar issue for such + // badly muxed resources. + if (a->GetEnd() >= audioPacketTimeToDiscard) { + mReader->mAudioQueue.PushFront(a.forget()); + break; + } + } + + if (finished) { + mAudioCompleted = true; + UpdateReadyState(); + } + } +} + +bool nsBuiltinDecoderStateMachine::HaveEnoughDecodedAudio(PRInt64 aAmpleAudioUSecs) +{ + mDecoder->GetReentrantMonitor().AssertCurrentThreadIn(); + + if (mReader->mAudioQueue.GetSize() == 0 || + GetDecodedAudioDuration() < aAmpleAudioUSecs) { + return false; + } + if (!mAudioCaptured) { + return true; + } + + nsTArray& streams = mDecoder->OutputStreams(); + for (PRUint32 i = 0; i < streams.Length(); ++i) { + OutputMediaStream* stream = &streams[i]; + if (!stream->mHaveSentFinishAudio && + !stream->mStream->HaveEnoughBuffered(TRACK_AUDIO)) { + return false; + } + } + + nsIThread* thread = GetStateMachineThread(); + nsCOMPtr callback = NS_NewRunnableMethod(this, + &nsBuiltinDecoderStateMachine::ScheduleStateMachineWithLockAndWakeDecoder); + for (PRUint32 i = 0; i < streams.Length(); ++i) { + OutputMediaStream* stream = &streams[i]; + if (!stream->mHaveSentFinishAudio) { + stream->mStream->DispatchWhenNotEnoughBuffered(TRACK_AUDIO, thread, callback); + } + } + return true; +} + +bool nsBuiltinDecoderStateMachine::HaveEnoughDecodedVideo() +{ + mDecoder->GetReentrantMonitor().AssertCurrentThreadIn(); + + if (static_cast(mReader->mVideoQueue.GetSize()) < AMPLE_VIDEO_FRAMES) { + return false; + } + + nsTArray& streams = mDecoder->OutputStreams(); + if (streams.IsEmpty()) { + return true; + } + + for (PRUint32 i = 0; i < streams.Length(); ++i) { + OutputMediaStream* stream = &streams[i]; + if (!stream->mHaveSentFinishVideo && + !stream->mStream->HaveEnoughBuffered(TRACK_VIDEO)) { + return false; + } + } + + nsIThread* thread = GetStateMachineThread(); + nsCOMPtr callback = NS_NewRunnableMethod(this, + &nsBuiltinDecoderStateMachine::ScheduleStateMachineWithLockAndWakeDecoder); + for (PRUint32 i = 0; i < streams.Length(); ++i) { + OutputMediaStream* stream = &streams[i]; + if (!stream->mHaveSentFinishVideo) { + stream->mStream->DispatchWhenNotEnoughBuffered(TRACK_VIDEO, thread, callback); + } + } + return true; +} + void nsBuiltinDecoderStateMachine::DecodeLoop() { LOG(PR_LOG_DEBUG, ("%p Start DecodeLoop()", mDecoder.get())); @@ -558,7 +833,6 @@ void nsBuiltinDecoderStateMachine::DecodeLoop() PRInt64 ampleAudioThreshold = AMPLE_AUDIO_USECS; MediaQueue& videoQueue = mReader->mVideoQueue; - MediaQueue& audioQueue = mReader->mAudioQueue; // Main decode loop. bool videoPlaying = HasVideo(); @@ -592,10 +866,9 @@ void nsBuiltinDecoderStateMachine::DecodeLoop() if (mState == DECODER_STATE_DECODING && !skipToNextKeyframe && videoPlaying && - ((!audioPump && audioPlaying && GetDecodedAudioDuration() < lowAudioThreshold) || - (!videoPump && - videoPlaying && - static_cast(videoQueue.GetSize()) < LOW_VIDEO_FRAMES)) && + ((!audioPump && audioPlaying && !mDidThrottleAudioDecoding && GetDecodedAudioDuration() < lowAudioThreshold) || + (!videoPump && videoPlaying && !mDidThrottleVideoDecoding && + static_cast(videoQueue.GetSize()) < LOW_VIDEO_FRAMES)) && !HasLowUndecodedData()) { @@ -604,8 +877,12 @@ void nsBuiltinDecoderStateMachine::DecodeLoop() } // Video decode. - if (videoPlaying && - static_cast(videoQueue.GetSize()) < AMPLE_VIDEO_FRAMES) + bool throttleVideoDecoding = !videoPlaying || HaveEnoughDecodedVideo(); + if (mDidThrottleVideoDecoding && !throttleVideoDecoding) { + videoPump = true; + } + mDidThrottleVideoDecoding = throttleVideoDecoding; + if (!throttleVideoDecoding) { // Time the video decode, so that if it's slow, we can increase our low // audio threshold to reduce the chance of an audio underrun while we're @@ -632,13 +909,18 @@ void nsBuiltinDecoderStateMachine::DecodeLoop() } // Audio decode. - if (audioPlaying && - (GetDecodedAudioDuration() < ampleAudioThreshold || audioQueue.GetSize() == 0)) - { + bool throttleAudioDecoding = !audioPlaying || HaveEnoughDecodedAudio(ampleAudioThreshold); + if (mDidThrottleAudioDecoding && !throttleAudioDecoding) { + audioPump = true; + } + mDidThrottleAudioDecoding = throttleAudioDecoding; + if (!mDidThrottleAudioDecoding) { ReentrantMonitorAutoExit exitMon(mDecoder->GetReentrantMonitor()); audioPlaying = mReader->DecodeAudioData(); } + SendOutputStreamData(); + // Notify to ensure that the AudioLoop() is not waiting, in case it was // waiting for more audio to be decoded. mDecoder->GetReentrantMonitor().NotifyAll(); @@ -650,11 +932,7 @@ void nsBuiltinDecoderStateMachine::DecodeLoop() if ((mState == DECODER_STATE_DECODING || mState == DECODER_STATE_BUFFERING) && !mStopDecodeThread && (videoPlaying || audioPlaying) && - (!audioPlaying || (GetDecodedAudioDuration() >= ampleAudioThreshold && - audioQueue.GetSize() > 0)) - && - (!videoPlaying || - static_cast(videoQueue.GetSize()) >= AMPLE_VIDEO_FRAMES)) + throttleAudioDecoding && throttleVideoDecoding) { // All active bitstreams' decode is well ahead of the playback // position, we may as well wait for the playback to catch up. Note the @@ -697,6 +975,15 @@ bool nsBuiltinDecoderStateMachine::IsPlaying() return !mPlayStartTime.IsNull(); } +static void WriteSilence(nsAudioStream* aStream, PRUint32 aFrames) +{ + PRUint32 numSamples = aFrames * aStream->GetChannels(); + nsAutoTArray buf; + buf.SetLength(numSamples); + memset(buf.Elements(), 0, numSamples * sizeof(AudioDataValue)); + aStream->Write(buf.Elements(), aFrames); +} + void nsBuiltinDecoderStateMachine::AudioLoop() { NS_ASSERTION(OnAudioThread(), "Should be on audio thread."); @@ -760,6 +1047,7 @@ void nsBuiltinDecoderStateMachine::AudioLoop() } // If we're shutting down, break out and exit the audio thread. + // Also break out if audio is being captured. if (mState == DECODER_STATE_SHUTDOWN || mStopAudioThread || mReader->mAudioQueue.AtEndOfStream()) @@ -813,6 +1101,8 @@ void nsBuiltinDecoderStateMachine::AudioLoop() // time. missingFrames = NS_MIN(static_cast(PR_UINT32_MAX), missingFrames.value()); + LOG(PR_LOG_DEBUG, ("%p Decoder playing %d frames of silence", + mDecoder.get(), PRInt32(missingFrames.value()))); framesWritten = PlaySilence(static_cast(missingFrames.value()), channels, playedFrames.value()); } else { @@ -850,10 +1140,7 @@ void nsBuiltinDecoderStateMachine::AudioLoop() if (framesToWrite < PR_UINT32_MAX / channels) { // Write silence manually rather than using PlaySilence(), so that // the AudioAPI doesn't get a copy of the audio frames. - PRUint32 numSamples = framesToWrite * channels; - nsAutoArrayPtr buf(new AudioDataValue[numSamples]); - memset(buf.get(), 0, numSamples * sizeof(AudioDataValue)); - mAudioStream->Write(buf, framesToWrite); + WriteSilence(mAudioStream, framesToWrite); } } @@ -885,10 +1172,12 @@ void nsBuiltinDecoderStateMachine::AudioLoop() ReentrantMonitorAutoEnter mon(mDecoder->GetReentrantMonitor()); mAudioStream = nsnull; mEventManager.Clear(); - mAudioCompleted = true; - UpdateReadyState(); - // Kick the decode thread; it may be sleeping waiting for this to finish. - mDecoder->GetReentrantMonitor().NotifyAll(); + if (!mAudioCaptured) { + mAudioCompleted = true; + UpdateReadyState(); + // Kick the decode thread; it may be sleeping waiting for this to finish. + mDecoder->GetReentrantMonitor().NotifyAll(); + } } // Must not hold the decoder monitor while we shutdown the audio stream, as @@ -908,12 +1197,9 @@ PRUint32 nsBuiltinDecoderStateMachine::PlaySilence(PRUint32 aFrames, NS_ASSERTION(!mAudioStream->IsPaused(), "Don't play when paused"); PRUint32 maxFrames = SILENCE_BYTES_CHUNK / aChannels / sizeof(AudioDataValue); PRUint32 frames = NS_MIN(aFrames, maxFrames); - PRUint32 numSamples = frames * aChannels; - nsAutoArrayPtr buf(new AudioDataValue[numSamples]); - memset(buf.get(), 0, numSamples * sizeof(AudioDataValue)); - mAudioStream->Write(buf, frames); + WriteSilence(mAudioStream, frames); // Dispatch events to the DOM for the audio just written. - mEventManager.QueueWrittenAudioData(buf.get(), frames * aChannels, + mEventManager.QueueWrittenAudioData(nsnull, frames * aChannels, (aFrameOffset + frames) * aChannels); return frames; } @@ -927,6 +1213,7 @@ PRUint32 nsBuiltinDecoderStateMachine::PlayFromAudioQueue(PRUint64 aFrameOffset, { ReentrantMonitorAutoEnter mon(mDecoder->GetReentrantMonitor()); NS_WARN_IF_FALSE(IsPlaying(), "Should be playing"); + NS_ASSERTION(!mAudioCaptured, "Audio cannot be captured here!"); // Awaken the decode loop if it's waiting for space to free up in the // audio queue. mDecoder->GetReentrantMonitor().NotifyAll(); @@ -941,6 +1228,8 @@ PRUint32 nsBuiltinDecoderStateMachine::PlayFromAudioQueue(PRUint64 aFrameOffset, // able to acquire the audio monitor in order to resume or destroy the // audio stream. if (!mAudioStream->IsPaused()) { + LOG(PR_LOG_DEBUG, ("%p Decoder playing %d frames of data to stream for AudioData at %lld", + mDecoder.get(), audio->mFrames, audio->mTime)); mAudioStream->Write(audio->mAudioData, audio->mFrames); @@ -1077,6 +1366,16 @@ void nsBuiltinDecoderStateMachine::SetVolume(double volume) mVolume = volume; } +void nsBuiltinDecoderStateMachine::SetAudioCaptured(bool aCaptured) +{ + NS_ASSERTION(NS_IsMainThread(), "Should be on main thread."); + ReentrantMonitorAutoEnter mon(mDecoder->GetReentrantMonitor()); + if (!mAudioCaptured && aCaptured) { + StopAudioThread(); + } + mAudioCaptured = aCaptured; +} + double nsBuiltinDecoderStateMachine::GetCurrentTime() const { NS_ASSERTION(NS_IsMainThread() || @@ -1363,7 +1662,7 @@ nsBuiltinDecoderStateMachine::StartAudioThread() "Should be on state machine or decode thread."); mDecoder->GetReentrantMonitor().AssertCurrentThreadIn(); mStopAudioThread = false; - if (HasAudio() && !mAudioThread) { + if (HasAudio() && !mAudioThread && !mAudioCaptured) { nsresult rv = NS_NewThread(getter_AddRefs(mAudioThread), nsnull, MEDIA_THREAD_STACK_SIZE); @@ -1542,6 +1841,9 @@ void nsBuiltinDecoderStateMachine::DecodeSeek() NS_ASSERTION(mState == DECODER_STATE_SEEKING, "Only call when in seeking state"); + mDidThrottleAudioDecoding = false; + mDidThrottleVideoDecoding = false; + // During the seek, don't have a lock on the decoder state, // otherwise long seek operations can block the main thread. // The events dispatched to the main thread are SYNC calls. @@ -1596,7 +1898,7 @@ void nsBuiltinDecoderStateMachine::DecodeSeek() mAudioStartTime = startTime; mPlayDuration = startTime - mStartTime; if (HasVideo()) { - nsAutoPtr video(mReader->mVideoQueue.PeekFront()); + VideoData* video = mReader->mVideoQueue.PeekFront(); if (video) { NS_ASSERTION(video->mTime <= seekTime && seekTime <= video->mEndTime, "Seek target should lie inside the first frame after seek"); @@ -1604,7 +1906,6 @@ void nsBuiltinDecoderStateMachine::DecodeSeek() ReentrantMonitorAutoExit exitMon(mDecoder->GetReentrantMonitor()); RenderVideoFrame(video, TimeStamp::Now()); } - mReader->mVideoQueue.PopFront(); nsCOMPtr event = NS_NewRunnableMethod(mDecoder, &nsBuiltinDecoder::Invalidate); NS_DispatchToMainThread(event, NS_DISPATCH_NORMAL); @@ -1882,6 +2183,9 @@ void nsBuiltinDecoderStateMachine::RenderVideoFrame(VideoData* aData, return; } + LOG(PR_LOG_DEBUG, ("%p Decoder playing video frame %lld", + mDecoder.get(), aData->mTime)); + VideoFrameContainer* container = mDecoder->GetVideoFrameContainer(); if (container) { container->SetCurrentFrame(aData->mDisplay, aData->mImage, aTarget); @@ -1893,7 +2197,7 @@ nsBuiltinDecoderStateMachine::GetAudioClock() { NS_ASSERTION(OnStateMachineThread(), "Should be on state machine thread."); mDecoder->GetReentrantMonitor().AssertCurrentThreadIn(); - if (!HasAudio()) + if (!HasAudio() || mAudioCaptured) return -1; // We must hold the decoder monitor while using the audio stream off the // audio thread to ensure that it doesn't get destroyed on the audio thread @@ -1953,6 +2257,7 @@ void nsBuiltinDecoderStateMachine::AdvanceFrame() while (mRealTime || clock_time >= frame->mTime) { mVideoFrameEndTime = frame->mEndTime; currentFrame = frame; + LOG(PR_LOG_DEBUG, ("%p Decoder discarding video frame %lld", mDecoder.get(), frame->mTime)); mReader->mVideoQueue.PopFront(); // Notify the decode thread that the video queue's buffers may have // free'd up space for more frames. @@ -2240,6 +2545,12 @@ nsresult nsBuiltinDecoderStateMachine::ScheduleStateMachine() { return ScheduleStateMachine(0); } +void nsBuiltinDecoderStateMachine::ScheduleStateMachineWithLockAndWakeDecoder() { + ReentrantMonitorAutoEnter mon(mDecoder->GetReentrantMonitor()); + mon.NotifyAll(); + ScheduleStateMachine(0); +} + nsresult nsBuiltinDecoderStateMachine::ScheduleStateMachine(PRInt64 aUsecs) { mDecoder->GetReentrantMonitor().AssertCurrentThreadIn(); NS_ABORT_IF_FALSE(GetStateMachineThread(), diff --git a/content/media/nsBuiltinDecoderStateMachine.h b/content/media/nsBuiltinDecoderStateMachine.h index 6a211ccb6c9..4ea50f246b8 100644 --- a/content/media/nsBuiltinDecoderStateMachine.h +++ b/content/media/nsBuiltinDecoderStateMachine.h @@ -117,6 +117,8 @@ hardware (via nsAudioStream and libsydneyaudio). #include "nsHTMLMediaElement.h" #include "mozilla/ReentrantMonitor.h" #include "nsITimer.h" +#include "AudioSegment.h" +#include "VideoSegment.h" /* The state machine class. This manages the decoding and seeking in the @@ -137,6 +139,10 @@ public: typedef mozilla::TimeStamp TimeStamp; typedef mozilla::TimeDuration TimeDuration; typedef mozilla::VideoFrameContainer VideoFrameContainer; + typedef nsBuiltinDecoder::OutputMediaStream OutputMediaStream; + typedef mozilla::SourceMediaStream SourceMediaStream; + typedef mozilla::AudioSegment AudioSegment; + typedef mozilla::VideoSegment VideoSegment; nsBuiltinDecoderStateMachine(nsBuiltinDecoder* aDecoder, nsBuiltinDecoderReader* aReader, bool aRealTime = false); ~nsBuiltinDecoderStateMachine(); @@ -149,6 +155,7 @@ public: return mState; } virtual void SetVolume(double aVolume); + virtual void SetAudioCaptured(bool aCapture); virtual void Shutdown(); virtual PRInt64 GetDuration(); virtual void SetDuration(PRInt64 aDuration); @@ -249,6 +256,10 @@ public: // machine again. nsresult ScheduleStateMachine(); + // Calls ScheduleStateMachine() after taking the decoder lock. Also + // notifies the decoder thread in case it's waiting on the decoder lock. + void ScheduleStateMachineWithLockAndWakeDecoder(); + // Schedules the shared state machine thread to run the state machine // in aUsecs microseconds from now, if it's not already scheduled to run // earlier, in which case the request is discarded. @@ -273,6 +284,12 @@ public: // element. Called on the main thread. void NotifyAudioAvailableListener(); + // Copy queued audio/video data in the reader to any output MediaStreams that + // need it. + void SendOutputStreamData(); + bool HaveEnoughDecodedAudio(PRInt64 aAmpleAudioUSecs); + bool HaveEnoughDecodedVideo(); + protected: // Returns true if we've got less than aAudioUsecs microseconds of decoded @@ -436,6 +453,11 @@ protected: // to call. void DecodeThreadRun(); + // Copy audio from an AudioData packet to aOutput. This may require + // inserting silence depending on the timing of the audio packet. + void SendOutputStreamAudio(AudioData* aAudio, OutputMediaStream* aStream, + AudioSegment* aOutput); + // State machine thread run function. Defers to RunStateMachine(). nsresult CallRunStateMachine(); @@ -569,6 +591,10 @@ protected: // Time at which we started decoding. Synchronised via decoder monitor. TimeStamp mDecodeStartTime; + // True if we shouldn't play our audio (but still write it to any capturing + // streams). + bool mAudioCaptured; + // True if the media resource can be seeked. Accessed from the state // machine and main threads. Synchronised via decoder monitor. bool mSeekable; @@ -636,6 +662,12 @@ protected: // True is we are decoding a realtime stream, like a camera stream bool mRealTime; + // Record whether audio and video decoding were throttled during the + // previous iteration of DecodeLooop. When we transition from + // throttled to not-throttled we need to pump decoding. + bool mDidThrottleAudioDecoding; + bool mDidThrottleVideoDecoding; + // True if we've requested a new decode thread, but it has not yet been // created. Synchronized by the decoder monitor. bool mRequestedNewDecodeThread; diff --git a/content/media/nsMediaCache.cpp b/content/media/nsMediaCache.cpp index 0ee5f98dca8..e12e154a562 100644 --- a/content/media/nsMediaCache.cpp +++ b/content/media/nsMediaCache.cpp @@ -239,6 +239,7 @@ public: /** * An iterator that makes it easy to iterate through all streams that * have a given resource ID and are not closed. + * Can be used on the main thread or while holding the media cache lock. */ class ResourceStreamIterator { public: @@ -351,13 +352,14 @@ protected: // This member is main-thread only. It's used to allocate unique // resource IDs to streams. PRInt64 mNextResourceID; - // This member is main-thread only. It contains all the streams. - nsTArray mStreams; // The monitor protects all the data members here. Also, off-main-thread // readers that need to block will Wait() on this monitor. When new // data becomes available in the cache, we NotifyAll() on this monitor. ReentrantMonitor mReentrantMonitor; + // This is only written while on the main thread and the monitor is held. + // Thus, it can be safely read from the main thread or while holding the monitor. + nsTArray mStreams; // The Blocks describing the cache entries. nsTArray mIndex; // Writer which performs IO, asynchronously writing cache blocks. @@ -1703,10 +1705,10 @@ nsMediaCacheStream::NotifyDataStarted(PRInt64 aOffset) } } -void +bool nsMediaCacheStream::UpdatePrincipal(nsIPrincipal* aPrincipal) { - nsContentUtils::CombineResourcePrincipals(&mPrincipal, aPrincipal); + return nsContentUtils::CombineResourcePrincipals(&mPrincipal, aPrincipal); } void @@ -1715,6 +1717,20 @@ nsMediaCacheStream::NotifyDataReceived(PRInt64 aSize, const char* aData, { NS_ASSERTION(NS_IsMainThread(), "Only call on main thread"); + // Update principals before putting the data in the cache. This is important, + // we want to make sure all principals are updated before any consumer + // can see the new data. + // We do this without holding the cache monitor, in case the client wants + // to do something that takes a lock. + { + nsMediaCache::ResourceStreamIterator iter(mResourceID); + while (nsMediaCacheStream* stream = iter.Next()) { + if (stream->UpdatePrincipal(aPrincipal)) { + stream->mClient->CacheClientNotifyPrincipalChanged(); + } + } + } + ReentrantMonitorAutoEnter mon(gMediaCache->GetReentrantMonitor()); PRInt64 size = aSize; const char* data = aData; @@ -1769,7 +1785,6 @@ nsMediaCacheStream::NotifyDataReceived(PRInt64 aSize, const char* aData, // The stream is at least as long as what we've read stream->mStreamLength = NS_MAX(stream->mStreamLength, mChannelOffset); } - stream->UpdatePrincipal(aPrincipal); stream->mClient->CacheClientNotifyDataReceived(); } diff --git a/content/media/nsMediaCache.h b/content/media/nsMediaCache.h index fe0a17b700f..3142ddf585c 100644 --- a/content/media/nsMediaCache.h +++ b/content/media/nsMediaCache.h @@ -230,10 +230,10 @@ public: // aClient provides the underlying transport that cache will use to read // data for this stream. nsMediaCacheStream(ChannelMediaResource* aClient) - : mClient(aClient), mResourceID(0), mInitialized(false), + : mClient(aClient), mInitialized(false), mHasHadUpdate(false), mClosed(false), - mDidNotifyDataEnded(false), + mDidNotifyDataEnded(false), mResourceID(0), mIsSeekable(false), mCacheSuspended(false), mChannelEnded(false), mChannelOffset(0), mStreamLength(-1), @@ -326,7 +326,8 @@ public: // If we've successfully read data beyond the originally reported length, // we return the end of the data we've read. PRInt64 GetLength(); - // Returns the unique resource ID + // Returns the unique resource ID. Call only on the main thread or while + // holding the media cache lock. PRInt64 GetResourceID() { return mResourceID; } // Returns the end of the bytes starting at the given offset // which are in cache. @@ -459,15 +460,11 @@ private: // blocked on reading from this stream. void CloseInternal(ReentrantMonitorAutoEnter& aReentrantMonitor); // Update mPrincipal given that data has been received from aPrincipal - void UpdatePrincipal(nsIPrincipal* aPrincipal); + bool UpdatePrincipal(nsIPrincipal* aPrincipal); // These fields are main-thread-only. ChannelMediaResource* mClient; nsCOMPtr mPrincipal; - // This is a unique ID representing the resource we're loading. - // All streams with the same mResourceID are loading the same - // underlying resource and should share data. - PRInt64 mResourceID; // Set to true when Init or InitAsClone has been called bool mInitialized; // Set to true when nsMediaCache::Update() has finished while this stream @@ -479,9 +476,14 @@ private: // True if CacheClientNotifyDataEnded has been called for this stream. bool mDidNotifyDataEnded; - // The following fields are protected by the cache's monitor but are - // only written on the main thread. + // The following fields must be written holding the cache's monitor and + // only on the main thread, thus can be read either on the main thread + // or while holding the cache's monitor. + // This is a unique ID representing the resource we're loading. + // All streams with the same mResourceID are loading the same + // underlying resource and should share data. + PRInt64 mResourceID; // The last reported seekability state for the underlying channel bool mIsSeekable; // True if the cache has suspended our channel because the cache is diff --git a/content/media/nsMediaDecoder.h b/content/media/nsMediaDecoder.h index 8418d35c7ee..7a2a24f38f3 100644 --- a/content/media/nsMediaDecoder.h +++ b/content/media/nsMediaDecoder.h @@ -41,6 +41,7 @@ #include "ImageLayers.h" #include "mozilla/ReentrantMonitor.h" #include "VideoFrameContainer.h" +#include "MediaStreamGraph.h" class nsHTMLMediaElement; class nsIStreamListener; @@ -69,13 +70,14 @@ static const PRUint32 FRAMEBUFFER_LENGTH_MAX = 16384; class nsMediaDecoder : public nsIObserver { public: + typedef mozilla::layers::Image Image; + typedef mozilla::layers::ImageContainer ImageContainer; typedef mozilla::MediaResource MediaResource; typedef mozilla::ReentrantMonitor ReentrantMonitor; + typedef mozilla::SourceMediaStream SourceMediaStream; typedef mozilla::TimeStamp TimeStamp; typedef mozilla::TimeDuration TimeDuration; typedef mozilla::VideoFrameContainer VideoFrameContainer; - typedef mozilla::layers::Image Image; - typedef mozilla::layers::ImageContainer ImageContainer; nsMediaDecoder(); virtual ~nsMediaDecoder(); @@ -129,6 +131,13 @@ public: // Set the audio volume. It should be a value from 0 to 1.0. virtual void SetVolume(double aVolume) = 0; + // Sets whether audio is being captured. If it is, we won't play any + // of our audio. + virtual void SetAudioCaptured(bool aCaptured) = 0; + + // Add an output stream. All decoder output will be sent to the stream. + virtual void AddOutputStream(SourceMediaStream* aStream, bool aFinishWhenEnded) = 0; + // Start playback of a video. 'Load' must have previously been // called. virtual nsresult Play() = 0; @@ -331,6 +340,10 @@ public: // the result from OnStopRequest. virtual void NotifyDownloadEnded(nsresult aStatus) = 0; + // Called by MediaResource when the principal of the resource has + // changed. Called on main thread only. + virtual void NotifyPrincipalChanged() = 0; + // Called as data arrives on the stream and is read into the cache. Called // on the main thread only. virtual void NotifyDataArrived(const char* aBuffer, PRUint32 aLength, PRInt64 aOffset) = 0; diff --git a/dom/interfaces/html/nsIDOMHTMLAudioElement.idl b/dom/interfaces/html/nsIDOMHTMLAudioElement.idl index 6869fa3a5f9..a3983a73ee7 100644 --- a/dom/interfaces/html/nsIDOMHTMLAudioElement.idl +++ b/dom/interfaces/html/nsIDOMHTMLAudioElement.idl @@ -52,7 +52,7 @@ * @status UNDER_DEVELOPMENT */ -[scriptable, uuid(e1a11e83-255b-4350-81cf-f1f3e7d59712)] +[scriptable, uuid(32c54e30-5063-4e35-8fc9-890e50fed147)] interface nsIDOMHTMLAudioElement : nsIDOMHTMLMediaElement { // Setup the audio stream for writing diff --git a/dom/interfaces/html/nsIDOMHTMLMediaElement.idl b/dom/interfaces/html/nsIDOMHTMLMediaElement.idl index b3cf00c55b2..979c88a1dfe 100644 --- a/dom/interfaces/html/nsIDOMHTMLMediaElement.idl +++ b/dom/interfaces/html/nsIDOMHTMLMediaElement.idl @@ -40,6 +40,8 @@ #include "nsIDOMMediaError.idl" #include "nsIDOMTimeRanges.idl" +interface nsIDOMMediaStream; + /** * The nsIDOMHTMLMediaElement interface is an interface to be implemented by the HTML *