From edb7a03da519f0b0d9f7ff592b1b0deafceb54b8 Mon Sep 17 00:00:00 2001 From: Alex Soto Date: Mon, 16 Oct 2017 03:11:27 -0500 Subject: [PATCH] [CoreMedia] Adds API bindings from Xcode 9 Beta 1 to stable (#2886) * [CoreMedia] Adds API bindings from Xcode 9 Beta 1 to stable * Bindings for CoreMedia API from Xcode 9 beta 1 to Stable * Adds test for manual bindings of CMVideoFormatDescription HEVC APIs * Fixed TODO of pending exposure of CMSampleBufferAttachmentSettings API * Derived of the TODO renamed some internal keys inside CMSampleAttachmentKey static class in order for them to follow the [StrongDictionary] conventions to avoid uneccesary [Export] on StrongDictionary members. Also removed a #if !MONOMAC conditional in favour of [NoMac]. * implement feedback --- src/CoreMedia/CMFormatDescription.cs | 76 +++++++++++ src/CoreMedia/CMSampleBuffer.cs | 13 +- src/coremedia.cs | 122 +++++++++++++++--- .../CoreMedia/CMFormatDescriptionTest.cs | 40 ++++++ 4 files changed, 222 insertions(+), 29 deletions(-) diff --git a/src/CoreMedia/CMFormatDescription.cs b/src/CoreMedia/CMFormatDescription.cs index 4d8ad580d0..4c5b2e3eb4 100644 --- a/src/CoreMedia/CMFormatDescription.cs +++ b/src/CoreMedia/CMFormatDescription.cs @@ -538,6 +538,82 @@ namespace XamCore.CoreMedia { return CMVideoFormatDescriptionMatchesImageBuffer (handle, imageBuffer.Handle); } #endif + + [iOS (11,0), Mac (10,13), TV (11,0)] + [DllImport (Constants.CoreMediaLibrary)] + static extern /* OSStatus */ CMFormatDescriptionError CMVideoFormatDescriptionCreateFromHEVCParameterSets ( + /* CFAllocatorRef */ IntPtr allocator, + /* size_t */ nuint parameterSetCount, + /* const uint8_t* const* */ IntPtr [] parameterSetPointers, + /* size_t* */ nuint[] parameterSetSizes, + /* int */ int NALUnitHeaderLength, + /* CFDictionaryRef */ IntPtr extensions, + /* CMFormatDescriptionRef* */ out IntPtr formatDescriptionOut); + + [iOS (11,0), Mac (10,13), TV (11,0)] + public static CMVideoFormatDescription FromHevcParameterSets (List parameterSets, int nalUnitHeaderLength, NSDictionary extensions, out CMFormatDescriptionError error) + { + if (parameterSets == null) + throw new ArgumentNullException (nameof (parameterSets)); + + if (parameterSets.Count < 3) + throw new ArgumentException ($"{nameof (parameterSets)} must contain at least three elements"); + + if (nalUnitHeaderLength != 1 && nalUnitHeaderLength != 2 && nalUnitHeaderLength != 4) + throw new ArgumentOutOfRangeException (nameof (nalUnitHeaderLength), "must be 1, 2 or 4"); + + var handles = new GCHandle [parameterSets.Count]; + try { + var parameterSetSizes = new nuint [parameterSets.Count]; + var parameterSetPtrs = new IntPtr [parameterSets.Count]; + + for (int i = 0; i < parameterSets.Count; i++) { + handles [i] = GCHandle.Alloc (parameterSets [i], GCHandleType.Pinned); // This can't use unsafe code because we need to get the pointer for an unbound number of objects. + parameterSetPtrs [i] = handles [i].AddrOfPinnedObject (); + parameterSetSizes [i] = (nuint) parameterSets [i].Length; + } + + IntPtr desc; + error = CMVideoFormatDescriptionCreateFromHEVCParameterSets (IntPtr.Zero, (nuint) parameterSets.Count, parameterSetPtrs, parameterSetSizes, nalUnitHeaderLength, extensions.GetHandle (), out desc); + if (error != CMFormatDescriptionError.None || desc == IntPtr.Zero) + return null; + + return new CMVideoFormatDescription (desc, true); + } finally { + for (int i = 0; i < handles.Length; i++) { + if (handles [i].IsAllocated) + handles [i].Free (); + } + } + } + + [iOS (11,0), Mac (10,13), TV (11,0)] + [DllImport (Constants.CoreMediaLibrary)] + static extern /* OSStatus */ CMFormatDescriptionError CMVideoFormatDescriptionGetHEVCParameterSetAtIndex ( + /* CMFormatDescriptionRef */ IntPtr videoDesc, + /* size_t */ nuint parameterSetIndex, + /* const uint8_t** */ out IntPtr parameterSetPointerOut, + /* size_t* */ out nuint parameterSetSizeOut, + /* size_t* */ out nuint parameterSetCountOut, + /* int* */ out int nalUnitHeaderLengthOut); + + [iOS (11,0), Mac (10,13), TV (11,0)] + public byte [] GetHevcParameterSet (nuint index, out nuint parameterSetCount, out int nalUnitHeaderLength, out CMFormatDescriptionError error) + { + if (Handle == IntPtr.Zero) + throw new ObjectDisposedException ("VideoFormatDescription"); + + IntPtr ret; + nuint parameterSetSizeOut; + error = CMVideoFormatDescriptionGetHEVCParameterSetAtIndex (Handle, index, out ret, out parameterSetSizeOut, out parameterSetCount, out nalUnitHeaderLength); + if (error != CMFormatDescriptionError.None) + return null; + + var arr = new byte [(int) parameterSetSizeOut]; + Marshal.Copy (ret, arr, 0, (int) parameterSetSizeOut); + + return arr; + } #endif } } diff --git a/src/CoreMedia/CMSampleBuffer.cs b/src/CoreMedia/CMSampleBuffer.cs index b7b18023d1..2077d90c34 100644 --- a/src/CoreMedia/CMSampleBuffer.cs +++ b/src/CoreMedia/CMSampleBuffer.cs @@ -841,7 +841,7 @@ namespace XamCore.CoreMedia { public enum LensStabilizationStatus { Active, OutOfRange, Unavailable, Off, None } #if !COREBUILD - public class CMSampleBufferAttachmentSettings : DictionaryContainer { + public partial class CMSampleBufferAttachmentSettings : DictionaryContainer { internal CMSampleBufferAttachmentSettings (NSMutableDictionary dictionary) : base (dictionary) @@ -1020,17 +1020,6 @@ namespace XamCore.CoreMedia { } } #endif - - // TODO: Implement remaining selector properties - // PostNotificationWhenConsumed - // ResumeOutput - // TransitionID - // TrimDurationAtStart - // TrimDurationAtEnd - // SpeedMultiplier - // SampleReferenceURL - // SampleReferenceByteOffset - // GradualDecoderRefresh } #endif } diff --git a/src/coremedia.cs b/src/coremedia.cs index a3eaa87469..4496844f10 100644 --- a/src/coremedia.cs +++ b/src/coremedia.cs @@ -76,22 +76,22 @@ namespace XamCore.CoreMedia { NSString DrainAfterDecoding { get; } [Field ("kCMSampleBufferAttachmentKey_PostNotificationWhenConsumed")] - NSString PostNotificationWhenConsumed { get; } + NSString PostNotificationWhenConsumedKey { get; } [Field ("kCMSampleBufferAttachmentKey_ResumeOutput")] - NSString ResumeOutput { get; } + NSString ResumeOutputKey { get; } [Field ("kCMSampleBufferAttachmentKey_TransitionID")] - NSString TransitionID { get; } + NSString TransitionIdKey { get; } [Field ("kCMSampleBufferAttachmentKey_TrimDurationAtStart")] - NSString TrimDurationAtStart { get; } + NSString TrimDurationAtStartKey { get; } [Field ("kCMSampleBufferAttachmentKey_TrimDurationAtEnd")] - NSString TrimDurationAtEnd { get; } + NSString TrimDurationAtEndKey { get; } [Field ("kCMSampleBufferAttachmentKey_SpeedMultiplier")] - NSString SpeedMultiplier { get; } + NSString SpeedMultiplierKey { get; } [Field ("kCMSampleBufferAttachmentKey_Reverse")] NSString Reverse { get; } @@ -112,39 +112,127 @@ namespace XamCore.CoreMedia { NSString EndsPreviousSampleDuration { get; } [Field ("kCMSampleBufferAttachmentKey_SampleReferenceURL")] - NSString SampleReferenceURL { get; } + NSString SampleReferenceUrlKey { get; } [Field ("kCMSampleBufferAttachmentKey_SampleReferenceByteOffset")] - NSString SampleReferenceByteOffset { get; } + NSString SampleReferenceByteOffsetKey { get; } [Field ("kCMSampleBufferAttachmentKey_GradualDecoderRefresh")] - NSString GradualDecoderRefresh { get; } + NSString GradualDecoderRefreshKey { get; } -#if !MONOMAC - [iOS (6,0)] + [iOS (6,0)][NoMac] [Field ("kCMSampleBufferAttachmentKey_DroppedFrameReason")] NSString DroppedFrameReason { get; } - [iOS (9,0)] + [iOS (9,0)][NoMac] [Field ("kCMSampleBufferAttachmentKey_StillImageLensStabilizationInfo")] NSString StillImageLensStabilizationInfo { get; } - [iOS (9,0)] + [iOS (9,0)][NoMac] [Field ("kCMSampleBufferLensStabilizationInfo_Active")] NSString BufferLensStabilizationInfo_Active { get; } - [iOS (9,0)] + [iOS (9,0)][NoMac] [Field ("kCMSampleBufferLensStabilizationInfo_OutOfRange")] NSString BufferLensStabilizationInfo_OutOfRange { get; } - [iOS (9,0)] + [iOS (9,0)][NoMac] [Field ("kCMSampleBufferLensStabilizationInfo_Unavailable")] NSString BufferLensStabilizationInfo_Unavailable { get; } - [iOS (9,0)] + [iOS (9,0)][NoMac] [Field ("kCMSampleBufferLensStabilizationInfo_Off")] NSString BufferLensStabilizationInfo_Off { get; } -#endif + + [iOS (11,0), Mac (10,13), TV (11,0)] + [Field ("kCMSampleAttachmentKey_HEVCTemporalLevelInfo")] + NSString HevcTemporalLevelInfoKey { get; } + + [iOS (11,0), Mac (10,13), TV (11,0)] + [Field ("kCMSampleAttachmentKey_HEVCTemporalSubLayerAccess")] + NSString HevcTemporalSubLayerAccessKey { get; } + + [iOS (11,0), Mac (10,13), TV (11,0)] + [Field ("kCMSampleAttachmentKey_HEVCStepwiseTemporalSubLayerAccess")] + NSString HevcStepwiseTemporalSubLayerAccessKey { get; } + + [iOS (11,0), Mac (10,13), TV (11,0)] + [Field ("kCMSampleAttachmentKey_HEVCSyncSampleNALUnitType")] + NSString HevcSyncSampleNalUnitTypeKey { get; } + + [iOS (11,0), Mac (10,13), TV (11,0)] + [Field ("kCMSampleBufferAttachmentKey_CameraIntrinsicMatrix")] + NSString CameraIntrinsicMatrixKey { get; } + } + + [StrongDictionary ("CMSampleAttachmentKey")] + interface CMSampleBufferAttachmentSettings { + + NSDictionary PostNotificationWhenConsumed { get; set; } + bool ResumeOutput { get; set; } + int TransitionId { get; set; } + NSDictionary TrimDurationAtStart { get; set; } + NSDictionary TrimDurationAtEnd { get; set; } + float SpeedMultiplier { get; set; } + NSUrl SampleReferenceUrl { get; set; } + int SampleReferenceByteOffset { get; set; } + NSNumber GradualDecoderRefresh { get; set; } + + [iOS (11,0), Mac (10,13), TV (11,0)] + [StrongDictionary] + CMHevcTemporalLevelInfoSettings HevcTemporalLevelInfo { get; set; } + + [iOS (11,0), Mac (10,13), TV (11,0)] + bool HevcTemporalSubLayerAccess { get; set; } + + [iOS (11,0), Mac (10,13), TV (11,0)] + bool HevcStepwiseTemporalSubLayerAccess { get; set; } + + [iOS (11,0), Mac (10,13), TV (11,0)] + int HevcSyncSampleNalUnitType { get; set; } + + [iOS (11,0), Mac (10,13), TV (11,0)] + NSData CameraIntrinsicMatrix { get; set; } + } + + [Internal] + [iOS (11,0), Mac (10,13), TV (11,0)] + [Static] + interface CMHevcTemporalLevelInfoKeys { + + [Field ("kCMHEVCTemporalLevelInfoKey_TemporalLevel")] + NSString TemporalLevelKey { get; } + + [Field ("kCMHEVCTemporalLevelInfoKey_ProfileSpace")] + NSString ProfileSpaceKey { get; } + + [Field ("kCMHEVCTemporalLevelInfoKey_TierFlag")] + NSString TierFlagKey { get; } + + [Field ("kCMHEVCTemporalLevelInfoKey_ProfileIndex")] + NSString ProfileIndexKey { get; } + + [Field ("kCMHEVCTemporalLevelInfoKey_ProfileCompatibilityFlags")] + NSString ProfileCompatibilityFlagsKey { get; } + + [Field ("kCMHEVCTemporalLevelInfoKey_ConstraintIndicatorFlags")] + NSString ConstraintIndicatorFlagsKey { get; } + + [Field ("kCMHEVCTemporalLevelInfoKey_LevelIndex")] + NSString LevelIndexKey { get; } + } + + [iOS (11,0), Mac (10,13), TV (11,0)] + [StrongDictionary ("CMHevcTemporalLevelInfoKeys")] + interface CMHevcTemporalLevelInfoSettings { + + int TemporalLevel { get; set; } + int ProfileSpace { get; set; } + int TierFlag { get; set; } + int ProfileIndex { get; set; } + NSData ProfileCompatibilityFlags { get; set; } + NSData ConstraintIndicatorFlags { get; set; } + int LevelIndex { get; set; } } #if false diff --git a/tests/monotouch-test/CoreMedia/CMFormatDescriptionTest.cs b/tests/monotouch-test/CoreMedia/CMFormatDescriptionTest.cs index 209300edd1..3cb33a06a9 100644 --- a/tests/monotouch-test/CoreMedia/CMFormatDescriptionTest.cs +++ b/tests/monotouch-test/CoreMedia/CMFormatDescriptionTest.cs @@ -150,6 +150,46 @@ namespace MonoTouchFixtures.CoreMedia { Assert.That (arr1, Is.EqualTo (bytes), "H264ParameterSetsTest roundtrip"); } + [Test] + public void HevcParameterSetsTest () + { + TestRuntime.AssertXcodeVersion (9, 0); + + var arr0 = new byte [] { 0x40, 0x01, 0x0C, 0x06, 0xFF, 0xFF, 0x01, 0x60, 0x00, 0x00, 0x03, 0x00, 0xB0, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x5D, 0x00, 0x00, 0x15, 0xC0, 0x90 }; + var arr1 = new byte [] { 0x42, 0x01, 0x06, 0x01, 0x60, 0x00, 0x00, 0x03, 0x00, 0xB0, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x5D, 0x00, 0x00, 0xA0, 0x02, 0x80, 0x80, 0x2D, 0x16, 0x20, 0x57, 0xB9, 0x16, 0x41, 0x57, 0x20, 0x92, 0x7E, 0x84, 0x95, 0x4D, 0x69, 0x94, 0x92, 0x7E, 0x84, 0x95, 0x4D, 0x69, 0x9C, 0x92, 0x4B, 0x95, 0x4F, 0xA9, 0x49, 0x3E, 0x49, 0xD4, 0x93, 0xEA, 0x72, 0x49, 0x2B, 0x92, 0x5C, 0x97, 0xA9, 0xB8, 0x08, 0x08, 0x35, 0x20, 0x10 }; + var arr2 = new byte [] { 0x44, 0x01, 0xC0, 0x2C, 0xBC, 0x14, 0xC9 }; + + var props = new List { arr0, arr1, arr2 }; + CMFormatDescriptionError error; + var desc = CMVideoFormatDescription.FromHevcParameterSets (props, 4, null, out error); + + props = null; + Assert.That (error == CMFormatDescriptionError.None, "HevcParameterSetsTest 1"); + Assert.NotNull (desc, "HevcParameterSetsTest 2"); + Assert.That (desc.Dimensions.Height == 720 && desc.Dimensions.Width == 1280, "HevcParameterSetsTest 3"); + + CMFormatDescriptionError err; + nuint paramCount; + int nalCount; + var bytes = desc.GetHevcParameterSet (0, out paramCount, out nalCount, out err); + Assert.That (err == CMFormatDescriptionError.None, "HevcParameterSetsTest arr0 1"); + Assert.NotNull (bytes, "HevcParameterSetsTest arr0 2"); + Assert.True (nalCount == 4 && paramCount == 3); + Assert.That (arr0, Is.EqualTo (bytes), "HevcParameterSetsTest arr0 roundtrip"); + + bytes = desc.GetHevcParameterSet (1, out paramCount, out nalCount, out err); + Assert.That (err == CMFormatDescriptionError.None, "HevcParameterSetsTest arr1 1"); + Assert.NotNull (bytes, "HevcParameterSetsTest arr1 2"); + Assert.True (nalCount == 4 && paramCount == 3); + Assert.That (arr1, Is.EqualTo (bytes), "HevcParameterSetsTest arr1 roundtrip"); + + bytes = desc.GetHevcParameterSet (2, out paramCount, out nalCount, out err); + Assert.That (err == CMFormatDescriptionError.None, "HevcParameterSetsTest arr2 1"); + Assert.NotNull (bytes, "HevcParameterSetsTest arr2 2"); + Assert.True (nalCount == 4 && paramCount == 3); + Assert.That (arr2, Is.EqualTo (bytes), "HevcParameterSetsTest arr2 roundtrip"); + } + [Test] public void VideoFormatDescriptionConstructors () {