feat: Add VideoCodecInfo / AudioCodecInfo (#727)

* define AudioCodecInfo and VideoCodecInfo

* comment

* fix

* fix for failure of android test on CI

* fix bug

* test

.gitignore

@@ -332,10 +332,10 @@ healthchecksdb
 # Backup folder for Package Reference Convert tool in Visual Studio 2017
 MigrationBackup/
 
-# Jet Stream Rider 
+# Jet Stream Rider
 .idea/
 
-# Unity Folder 
+# Unity Folder
 [Ll]ibrary/
 [Tt]emp/
 [Oo]bj/
@@ -346,13 +346,17 @@ MigrationBackup/
 !BuildScripts~
 !Samples~
 
-# Exclude webrtc source 
+# Exclude webrtc source
 Plugin/webrtc
 
 # Exclude imported samples
 Assets/Samples.meta
 Assets/Samples/
 
+# Exclude upm-ci
+.bin/
+.Editor/
+
 # Exclude Webapp coverage result
 WebApp/**/coverage/*
 

com.unity.renderstreaming/Runtime/Scripts/AudioCodecInfo.cs

@@ -0,0 +1,51 @@
+using System;
+using Unity.WebRTC;
+
+namespace Unity.RenderStreaming
+{
+    /// <summary>
+    /// 
+    /// </summary>
+    public class AudioCodecInfo : IEquatable<AudioCodecInfo>
+    {
+        /// <summary>
+        /// 
+        /// </summary>
+        public string name { get { return capability.mimeType.Split('/')[1]; } }
+
+        /// <summary>
+        /// 
+        /// </summary>
+        public string mimeType { get { return capability.mimeType; } }
+
+        /// <summary>
+        /// 
+        /// </summary>
+        public int channelCount { get { return capability.channels.Value; } }
+
+        /// <summary>
+        /// 
+        /// </summary>
+        public int sampleRate { get { return capability.clockRate.Value; } }
+
+        internal RTCRtpCodecCapability capability;
+
+        /// <summary>
+        /// 
+        /// </summary>
+        /// <param name="other"></param>
+        /// <returns></returns>
+        public bool Equals(AudioCodecInfo other)
+        {
+            if (other == null)
+                return false;
+            return this.capability.mimeType == other.capability.mimeType
+                && this.capability.sdpFmtpLine == other.capability.sdpFmtpLine;
+        }
+
+        internal AudioCodecInfo(RTCRtpCodecCapability caps)
+        {
+            capability = caps;
+        }
+    }
+}

com.unity.renderstreaming/Runtime/Scripts/AudioCodecInfo.cs.meta

@@ -0,0 +1,11 @@
+fileFormatVersion: 2
+guid: 4d20d601e065b9147a6dd6d64fb43de6
+MonoImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  defaultReferences: []
+  executionOrder: 0
+  icon: {instanceID: 0}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

com.unity.renderstreaming/Runtime/Scripts/AudioStreamReceiver.cs

@@ -1,3 +1,5 @@
+using System.Linq;
+using System.Collections.Generic;
 using Unity.WebRTC;
 using UnityEngine;
 
@@ -31,6 +33,17 @@ namespace Unity.RenderStreaming
 
         private AudioSource m_source;
 
+        /// <summary>
+        /// 
+        /// </summary>
+        /// <returns></returns>
+        static public IEnumerable<AudioCodecInfo> GetAvailableCodecs()
+        {
+            var excludeCodecMimeType = new[] { "audio/CN", "audio/telephone-event" };
+            var capabilities = RTCRtpReceiver.GetCapabilities(TrackKind.Audio);
+            return capabilities.codecs.Where(codec => !excludeCodecMimeType.Contains(codec.mimeType)).Select(codec => new AudioCodecInfo(codec));
+        }
+
         /// <summary>
         ///
         /// </summary>

com.unity.renderstreaming/Runtime/Scripts/AudioStreamSender.cs

@@ -1,4 +1,6 @@
 using System;
+using System.Linq;
+using System.Collections.Generic;
 using Unity.Collections;
 using Unity.WebRTC;
 using UnityEngine;
@@ -24,6 +26,17 @@ namespace Unity.RenderStreaming
             get { return m_bitrate; }
         }
 
+        /// <summary>
+        /// 
+        /// </summary>
+        /// <returns></returns>
+        static public IEnumerable<AudioCodecInfo> GetAvailableCodecs()
+        {
+            var excludeCodecMimeType = new[] { "audio/CN", "audio/telephone-event" };
+            var capabilities = RTCRtpSender.GetCapabilities(TrackKind.Audio);
+            return capabilities.codecs.Where(codec => !excludeCodecMimeType.Contains(codec.mimeType)).Select(codec => new AudioCodecInfo(codec));
+        }
+
         /// <summary>
         /// 
         /// </summary>

com.unity.renderstreaming/Runtime/Scripts/RenderStreamingInternal.cs

@@ -158,6 +158,9 @@ namespace Unity.RenderStreaming
             _signaling.OnAnswer -= OnAnswer;
             _signaling.OnIceCandidate -= OnIceCandidate;
 
+            foreach(var pair in _mapConnectionIdAndPeer)
+                pair.Value.Dispose();
+
             this._disposed = true;
             GC.SuppressFinalize(this);
         }

com.unity.renderstreaming/Runtime/Scripts/RenderTextureBlitter.cs

@@ -1,4 +1,4 @@
-using UnityEngine;
+using UnityEngine;
 
 #if URS_USE_HDRP_RUNTIME
 #if UNITY_2019_1 || UNITY_2019_2 //HDRP 5.x, 6.x
@@ -14,7 +14,7 @@ namespace Unity.RenderStreaming
 #if URS_USE_HDRP_RUNTIME
     [RequireComponent(typeof(HDAdditionalCameraData))]
 #endif
-    public class RenderTextureBlitter : MonoBehaviour
+    internal class RenderTextureBlitter : MonoBehaviour
     {
         [SerializeField] Camera m_rtCamera = null;
 

com.unity.renderstreaming/Runtime/Scripts/ScreenStreamSender.cs

@@ -43,8 +43,10 @@ namespace Unity.RenderStreaming
             OnStoppedStream += id => connections.Remove(id);
         }
 
-        protected void OnDestroy()
+        override protected void OnDestroy()
         {
+            base.OnDestroy();
+
             if (m_sendTexture != null)
             {
                 DestroyImmediate(m_sendTexture);

com.unity.renderstreaming/Runtime/Scripts/StreamReceiverBase.cs

@@ -59,6 +59,12 @@ namespace Unity.RenderStreaming
                 OnStartedStream?.Invoke(connectionId);
         }
 
+        protected virtual void OnDestroy()
+        {
+            Track?.Dispose();
+            Track = null;
+        }
+
         private List<RTCRtpCodecCapability> m_receiverAudioCodecs = new List<RTCRtpCodecCapability>();
         private List<RTCRtpCodecCapability> m_receiverVideoCodecs = new List<RTCRtpCodecCapability>();
 

com.unity.renderstreaming/Runtime/Scripts/StreamSenderBase.cs

@@ -42,7 +42,7 @@ namespace Unity.RenderStreaming
         /// <summary>
         /// 
         /// </summary>
-        virtual protected void OnDestroy()
+        protected virtual void OnDestroy()
         {
             m_track?.Dispose();
             m_track = null;

com.unity.renderstreaming/Runtime/Scripts/VideoCodecInfo.cs

@@ -0,0 +1,168 @@
+using System;
+using System.Collections.Generic;
+using Unity.WebRTC;
+
+namespace Unity.RenderStreaming
+{
+    /// <summary>
+    /// 
+    /// </summary>
+    public class VideoCodecInfo : IEquatable<VideoCodecInfo>
+    {
+        const string KeyCodecImplementation = "implementation_name";
+
+        /// <summary>
+        /// 
+        /// </summary>
+        public string name { get { return capability.mimeType.Split('/')[1]; } }
+
+        /// <summary>
+        /// 
+        /// </summary>
+        public string mimeType { get { return capability.mimeType; } }
+
+        /// <summary>
+        /// 
+        /// </summary>
+        public string CodecImplementation { get { return parameters[KeyCodecImplementation]; } }
+
+        /// <summary>
+        /// 
+        /// </summary>
+        /// <param name="other"></param>
+        /// <returns></returns>
+        public bool Equals(VideoCodecInfo other)
+        {
+            if (other == null)
+                return false;
+            return this.capability.mimeType == other.capability.mimeType
+                && this.capability.sdpFmtpLine == other.capability.sdpFmtpLine;
+        }
+
+        internal RTCRtpCodecCapability capability;
+
+        protected readonly Dictionary<string, string> parameters = new Dictionary<string, string>();
+
+
+        static public VideoCodecInfo Create(RTCRtpCodecCapability caps)
+        {
+            switch(caps.mimeType)
+            {
+                case "video/H264":
+                    return new H264CodecInfo(caps);
+                case "video/VP9":
+                    return new VP9CodecInfo(caps);
+                default:
+                    return new VideoCodecInfo(caps);
+            }
+        }
+        protected VideoCodecInfo(RTCRtpCodecCapability caps)
+        {
+            capability = caps;
+
+            string[] subs = capability.sdpFmtpLine.Split(';');
+
+            foreach(string sub in subs)
+            {
+                string[] pair = sub.Split('=');
+                parameters.Add(pair[0], pair[1]);
+            }
+        }
+    }
+
+    /// <summary>
+    /// 
+    /// </summary>
+    public enum VP9Profile
+    {
+        /// <summary>
+        /// 
+        /// </summary>
+        Profile0 = 0,
+        /// <summary>
+        /// 
+        /// </summary>
+        Profile1 = 1,
+        /// <summary>
+        /// 
+        /// </summary>
+        Profile2 = 2,
+        /// <summary>
+        /// 
+        /// </summary>
+        Profile3 = 3,
+    }
+
+    /// <summary>
+    /// 
+    /// </summary>
+    public class VP9CodecInfo : VideoCodecInfo
+    {
+        const string KeyProfileId = "profile-id";
+
+        /// <summary>
+        /// 
+        /// </summary>
+        public VP9Profile profile
+        {
+            get { return (VP9Profile)Enum.ToObject(typeof(VP9Profile), Convert.ToInt32(parameters[KeyProfileId])); }
+        }
+
+        internal VP9CodecInfo(RTCRtpCodecCapability caps) : base(caps)
+        {
+        }
+    }
+
+    /// <summary>
+    /// 
+    /// </summary>
+    public enum H264Profile
+    {
+        /// <summary>
+        /// 
+        /// </summary>
+        ConstrainedBaseline = 0x42e0,
+        /// <summary>
+        /// 
+        /// </summary>
+        Baseline = 0x4200,
+        /// <summary>
+        /// 
+        /// </summary>
+        ProfileMain = 0x4d00,
+        /// <summary>
+        /// 
+        /// </summary>
+        ConstrainedHigh = 0x640c,
+        /// <summary>
+        /// 
+        /// </summary>
+        High = 0x6400,
+    }
+
+    /// <summary>
+    /// 
+    /// </summary>
+    public class H264CodecInfo : VideoCodecInfo
+    {
+        const string KeyProfileLevelId = "profile-level-id";
+
+        /// <summary>
+        /// 
+        /// </summary>
+        public H264Profile profile
+        {
+            get { return (H264Profile)Enum.ToObject(typeof(H264Profile), Convert.ToInt32(parameters[KeyProfileLevelId], 16) >> 8); }
+        }
+
+        /// <summary>
+        /// 
+        /// </summary>
+        public int level { get { return Convert.ToInt32(parameters[KeyProfileLevelId], 16) & 0xFF; } }
+
+        internal H264CodecInfo(RTCRtpCodecCapability caps) : base(caps)
+        {
+        }
+    }
+
+}

com.unity.renderstreaming/Runtime/Scripts/VideoCodecInfo.cs.meta

@@ -0,0 +1,11 @@
+fileFormatVersion: 2
+guid: ccaec4be995ff6a4a818bfb3726cd3be
+MonoImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  defaultReferences: []
+  executionOrder: 0
+  icon: {instanceID: 0}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

com.unity.renderstreaming/Runtime/Scripts/VideoStreamReceiver.cs

@@ -1,5 +1,8 @@
 using Unity.WebRTC;
 using UnityEngine;
+using System;
+using System.Linq;
+using System.Collections.Generic;
 
 namespace Unity.RenderStreaming
 {
@@ -34,6 +37,16 @@ namespace Unity.RenderStreaming
         /// </summary>
         public Texture ReceiveTexture => m_receiveTexture;
 
+        /// <summary>
+        /// 
+        /// </summary>
+        /// <returns></returns>
+        public static IEnumerable<VideoCodecInfo> GetAvailableCodecs()
+        {
+            string[] excludeCodecMimeType = { "video/red", "video/ulpfec", "video/rtx", "video/flexfec-03" };
+            var capabilities = RTCRtpReceiver.GetCapabilities(TrackKind.Video);
+            return capabilities.codecs.Where(codec => !excludeCodecMimeType.Contains(codec.mimeType)).Select(codec => VideoCodecInfo.Create(codec));
+        }
         private Texture m_receiveTexture;
 
         protected virtual void Start()

com.unity.renderstreaming/Runtime/Scripts/VideoStreamSender.cs

@@ -1,6 +1,7 @@
 using Unity.WebRTC;
 using UnityEngine;
 using System;
+using System.Collections.Generic;
 using System.Linq;
 
 namespace Unity.RenderStreaming
@@ -8,27 +9,27 @@ namespace Unity.RenderStreaming
     /// <summary>
     ///
     /// </summary>
-    public sealed class StreamingSizeAttribute : PropertyAttribute { }
+    internal sealed class StreamingSizeAttribute : PropertyAttribute { }
 
     /// <summary>
     ///
     /// </summary>
-    public sealed class FramerateAttribute : PropertyAttribute { }
+    internal sealed class FramerateAttribute : PropertyAttribute { }
 
     /// <summary>
     ///
     /// </summary>
-    public sealed class BitrateAttribute : PropertyAttribute { }
+    internal sealed class BitrateAttribute : PropertyAttribute { }
 
     /// <summary>
     ///
     /// </summary>
-    public sealed class RenderTextureAntiAliasingAttribute : PropertyAttribute { }
+    internal sealed class RenderTextureAntiAliasingAttribute : PropertyAttribute { }
 
     /// <summary>
     ///
     /// </summary>
-    public sealed class RenderTextureDepthBufferAttribute : PropertyAttribute { }
+    internal sealed class RenderTextureDepthBufferAttribute : PropertyAttribute { }
 
     internal static class RTCRtpSenderExtension
     {
@@ -131,6 +132,9 @@ namespace Unity.RenderStreaming
             get { return m_bitrate; }
         }
 
+        /// <summary>
+        /// 
+        /// </summary>
         public float scaleResolutionDown
         {
             get { return m_scaleFactor; }
@@ -158,6 +162,17 @@ namespace Unity.RenderStreaming
             transceiver.Sender.SetParameters(parameters);
         }
 
+        /// <summary>
+        /// 
+        /// </summary>
+        /// <returns></returns>
+        public static IEnumerable<VideoCodecInfo> GetAvailableCodecs()
+        {
+            string[] excludeCodecMimeType = { "video/red", "video/ulpfec", "video/rtx", "video/flexfec-03" };
+            var capabilities = RTCRtpSender.GetCapabilities(TrackKind.Video);
+            return capabilities.codecs.Where(codec => !excludeCodecMimeType.Contains(codec.mimeType)).Select(codec => VideoCodecInfo.Create(codec));
+        }
+
         /// <summary>
         ///
         /// </summary>
@@ -165,7 +180,7 @@ namespace Unity.RenderStreaming
         public void SetFrameRate(float frameRate)
         {
             if (frameRate < 0)
-                throw new ArgumentOutOfRangeException("framerate", frameRate, "The parameter must be greater than zero.");
+                throw new ArgumentOutOfRangeException("frameRate", frameRate, "The parameter must be greater than zero.");
             m_frameRate = frameRate;
             foreach (var transceiver in Transceivers.Values)
             {

com.unity.renderstreaming/Runtime/Scripts/WebCamStreamSender.cs

@@ -33,8 +33,10 @@ namespace Unity.RenderStreaming
         public override Texture SendTexture => m_webCamTexture;
         public IEnumerable<string> WebCamNameList => WebCamTexture.devices.Select(x => x.name);
 
-        protected virtual void OnDestroy()
+        protected override void OnDestroy()
         {
+            base.OnDestroy();
+
             if (m_webCamTexture != null)
             {
                 m_webCamTexture.Stop();

com.unity.renderstreaming/Tests/Runtime/StreamingComponentTest.cs

@@ -1,4 +1,6 @@
 using System;
+using System.Linq;
+using System.Collections.Generic;
 using NUnit.Framework;
 using UnityEngine;
 
@@ -6,6 +8,35 @@ namespace Unity.RenderStreaming.RuntimeTest
 {
     class VideoStreamSenderTest
     {
+        [Test]
+        public void GetAvailableCodec()
+        {
+            IEnumerable<VideoCodecInfo> codecs = VideoStreamSender.GetAvailableCodecs();
+            Assert.That(codecs, Is.Not.Empty);
+            foreach (var codec in codecs)
+            {
+                Assert.That(codec.name, Is.Not.Empty);
+                Assert.That(codec.mimeType, Is.Not.Empty);
+            }
+            Assert.That(codecs.Any(codec => codec.name == "VP8"));
+            Assert.That(codecs.Any(codec => codec.name == "VP9"));
+            Assert.That(codecs.Any(codec => codec.name == "AV1X"));
+
+            var codec1 = codecs.First(codec => codec.name == "VP9");
+            Assert.That(codec1, Is.TypeOf<VP9CodecInfo>());
+            VP9CodecInfo vp9Codec = codec1 as VP9CodecInfo;
+            Assert.That(vp9Codec.profile, Is.Not.Zero);
+
+            var codec2 = codecs.FirstOrDefault(codec => codec.name == "H264");
+            if(codec2 != null)
+            {
+                Assert.That(codec2, Is.TypeOf<H264CodecInfo>());
+                H264CodecInfo h264Codec = codec2 as H264CodecInfo;
+                Assert.That(h264Codec.level, Is.GreaterThan(0));
+                Assert.That(h264Codec.profile, Is.Not.Zero);
+            }
+        }
+
         [Test]
         public void SetEnabled()
         {
@@ -81,6 +112,20 @@ namespace Unity.RenderStreaming.RuntimeTest
 
     class AudioStreamSenderTest
     {
+        [Test]
+        public void GetAvailableCodec()
+        {
+            IEnumerable<AudioCodecInfo> codecs = AudioStreamSender.GetAvailableCodecs();
+            Assert.That(codecs, Is.Not.Empty);
+            foreach(var codec in codecs)
+            {
+                Assert.That(codec.name, Is.Not.Empty);
+                Assert.That(codec.mimeType, Is.Not.Empty);
+                Assert.That(codec.channelCount, Is.GreaterThan(0));
+                Assert.That(codec.sampleRate, Is.GreaterThan(0));
+            }
+        }
+
         [Test]
         public void SetEnabled()
         {