Bug 1766646 - Vendor libwebrtc from 4005e5abb8

Upstream commit: https://webrtc.googlesource.com/src/+/4005e5abb8bd160f237d89dd207c357cf56195cf
    Add av1 svc configuration for target bitrates

    This configuration mostly copies vp9 configuration for regular video,
    but is done separately to allow tune av1 svc bitrates independently of vp9.

    Bug: webrtc:12148
    Change-Id: Icd11817ada8f9b6135ee2da57204eadb50de3954
    Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/195329
    Reviewed-by: Philip Eliasson <philipel@webrtc.org>
    Commit-Queue: Danil Chapovalov <danilchap@webrtc.org>
    Cr-Commit-Position: refs/heads/master@{#32713}

third_party/libwebrtc/README.moz-ff-commit

@@ -1677,3 +1677,6 @@ cd982137df
 # MOZ_LIBWEBRTC_SRC=/home/mfroman/git-checkouts/trial-webrtc-builds/moz-libwebrtc-checkout/src MOZ_LIBWEBRTC_COMMIT=mjfdev bash dom/media/webrtc/third_party_build/fast-forward-libwebrtc.sh
 # base of lastest vendoring
 faaaa87960
+# MOZ_LIBWEBRTC_SRC=/home/mfroman/git-checkouts/trial-webrtc-builds/moz-libwebrtc-checkout/src MOZ_LIBWEBRTC_COMMIT=mjfdev bash dom/media/webrtc/third_party_build/fast-forward-libwebrtc.sh
+# base of lastest vendoring
+4005e5abb8

third_party/libwebrtc/README.mozilla

@@ -1120,3 +1120,5 @@ libwebrtc updated from /home/mfroman/git-checkouts/trial-webrtc-builds/moz-libwe
 libwebrtc updated from /home/mfroman/git-checkouts/trial-webrtc-builds/moz-libwebrtc-checkout/src commit mjfdev on 2022-04-16T15:27:52.737068.
 # python3 vendor-libwebrtc.py --from-local /home/mfroman/git-checkouts/trial-webrtc-builds/moz-libwebrtc-checkout/src --commit mjfdev libwebrtc
 libwebrtc updated from /home/mfroman/git-checkouts/trial-webrtc-builds/moz-libwebrtc-checkout/src commit mjfdev on 2022-04-16T15:28:33.344431.
+# python3 vendor-libwebrtc.py --from-local /home/mfroman/git-checkouts/trial-webrtc-builds/moz-libwebrtc-checkout/src --commit mjfdev libwebrtc
+libwebrtc updated from /home/mfroman/git-checkouts/trial-webrtc-builds/moz-libwebrtc-checkout/src commit mjfdev on 2022-04-16T15:29:10.064493.

third_party/libwebrtc/modules/video_coding/BUILD.gn

@@ -99,29 +99,6 @@ rtc_library("nack_module") {
 
 rtc_library("video_coding") {
   visibility = [ "*" ]
-  deps = [
-    "..:module_fec_api",
-    "../../api:array_view",
-    "../../api:scoped_refptr",
-    "../../api/video:encoded_image",
-    "../../api/video:video_adaptation",
-    "../../api/video:video_bitrate_allocation",
-    "../../api/video:video_bitrate_allocator_factory",
-    "../../rtc_base:deprecation",
-    "../../rtc_base/system:no_unique_address",
-    "../../rtc_base/task_utils:to_queued_task",
-    "../../system_wrappers:field_trial",
-    "../../system_wrappers:metrics",
-    "../rtp_rtcp:rtp_video_header",
-  ]
-  absl_deps = [
-    "//third_party/abseil-cpp/absl/base:core_headers",
-    "//third_party/abseil-cpp/absl/container:inlined_vector",
-    "//third_party/abseil-cpp/absl/memory",
-    "//third_party/abseil-cpp/absl/types:optional",
-    "//third_party/abseil-cpp/absl/types:variant",
-  ]
-
   sources = [
     "codec_timer.cc",
     "codec_timer.h",
@@ -169,7 +146,7 @@ rtc_library("video_coding") {
     "video_receiver2.h",
   ]
 
-  deps += [
+  deps = [
     ":codec_globals_headers",
     ":encoded_frame",
     ":video_codec_interface",
@@ -177,15 +154,22 @@ rtc_library("video_coding") {
     ":webrtc_vp9_helpers",
     "..:module_api",
     "..:module_api_public",
+    "..:module_fec_api",
+    "../../api:array_view",
     "../../api:fec_controller_api",
     "../../api:rtp_headers",
     "../../api:rtp_packet_info",
+    "../../api:scoped_refptr",
     "../../api/units:data_rate",
     "../../api/units:time_delta",
     "../../api/video:builtin_video_bitrate_allocator_factory",
     "../../api/video:encoded_frame",
+    "../../api/video:encoded_image",
     "../../api/video:video_adaptation",
+    "../../api/video:video_adaptation",
+    "../../api/video:video_bitrate_allocation",
     "../../api/video:video_bitrate_allocator",
+    "../../api/video:video_bitrate_allocator_factory",
     "../../api/video:video_frame",
     "../../api/video:video_frame_type",
     "../../api/video:video_rtp_headers",
@@ -193,6 +177,7 @@ rtc_library("video_coding") {
     "../../common_video",
     "../../rtc_base",
     "../../rtc_base:checks",
+    "../../rtc_base:deprecation",
     "../../rtc_base:rtc_base_approved",
     "../../rtc_base:rtc_numerics",
     "../../rtc_base:rtc_task_queue",
@@ -204,12 +189,25 @@ rtc_library("video_coding") {
     "../../rtc_base/experiments:rtt_mult_experiment",
     "../../rtc_base/synchronization:mutex",
     "../../rtc_base/synchronization:sequence_checker",
+    "../../rtc_base/system:no_unique_address",
     "../../rtc_base/task_utils:repeating_task",
+    "../../rtc_base/task_utils:to_queued_task",
     "../../rtc_base/third_party/base64",
     "../../rtc_base/time:timestamp_extrapolator",
     "../../system_wrappers",
+    "../../system_wrappers:field_trial",
+    "../../system_wrappers:metrics",
     "../rtp_rtcp",
     "../rtp_rtcp:rtp_rtcp_format",
+    "../rtp_rtcp:rtp_video_header",
+    "codecs/av1:av1_svc_config",
+  ]
+  absl_deps = [
+    "//third_party/abseil-cpp/absl/base:core_headers",
+    "//third_party/abseil-cpp/absl/container:inlined_vector",
+    "//third_party/abseil-cpp/absl/memory",
+    "//third_party/abseil-cpp/absl/types:optional",
+    "//third_party/abseil-cpp/absl/types:variant",
   ]
 }
 

third_party/libwebrtc/modules/video_coding/codecs/av1/BUILD.gn

@@ -9,6 +9,20 @@
 #import("//third_party/libaom/options.gni")
 import("../../../../webrtc.gni")
 
+rtc_library("av1_svc_config") {
+  sources = [
+    "av1_svc_config.cc",
+    "av1_svc_config.h",
+  ]
+  deps = [
+    "../../../../api/video_codecs:video_codecs_api",
+    "../../../../rtc_base:checks",
+    "../../../../rtc_base:logging",
+    "../../svc:scalability_structures",
+    "../../svc:scalable_video_controller",
+  ]
+}
+
 rtc_library("libaom_av1_decoder") {
   visibility = [ "*" ]
   poisonous = [ "software_video_codecs" ]
@@ -71,12 +85,18 @@ if (rtc_include_tests) {
   rtc_library("video_coding_codecs_av1_tests") {
     testonly = true
 
+    sources = [ "av1_svc_config_unittest.cc" ]
+    deps = [
+      ":av1_svc_config",
+      "../../../../api/video_codecs:video_codecs_api",
+    ]
+
     if (enable_libaom) {
-      sources = [
+      sources += [
         "libaom_av1_encoder_unittest.cc",
         "libaom_av1_unittest.cc",
       ]
-      deps = [
+      deps += [
         ":libaom_av1_decoder",
         ":libaom_av1_encoder",
         "../..:encoded_video_frame_producer",
@@ -85,7 +105,6 @@ if (rtc_include_tests) {
         "../../../../api/units:data_size",
         "../../../../api/units:time_delta",
         "../../../../api/video:video_frame",
-        "../../../../api/video_codecs:video_codecs_api",
         "../../../../test:test_support",
         "../../svc:scalability_structures",
         "../../svc:scalable_video_controller",

third_party/libwebrtc/modules/video_coding/codecs/av1/av1_svc_config.cc

@@ -0,0 +1,73 @@
+/*
+ *  Copyright (c) 2020 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "modules/video_coding/codecs/av1/av1_svc_config.h"
+
+#include <algorithm>
+#include <cmath>
+#include <memory>
+
+#include "modules/video_coding/svc/create_scalability_structure.h"
+#include "modules/video_coding/svc/scalable_video_controller.h"
+#include "rtc_base/checks.h"
+#include "rtc_base/logging.h"
+
+namespace webrtc {
+
+bool SetAv1SvcConfig(VideoCodec& video_codec) {
+  RTC_DCHECK_EQ(video_codec.codecType, kVideoCodecAV1);
+
+  if (video_codec.ScalabilityMode().empty()) {
+    RTC_LOG(LS_INFO) << "No scalability mode set.";
+    return false;
+  }
+  std::unique_ptr<ScalableVideoController> structure =
+      CreateScalabilityStructure(video_codec.ScalabilityMode());
+  if (structure == nullptr) {
+    RTC_LOG(LS_INFO) << "Failed to create structure "
+                     << video_codec.ScalabilityMode();
+    return false;
+  }
+  ScalableVideoController::StreamLayersConfig info = structure->StreamConfig();
+  for (int sl_idx = 0; sl_idx < info.num_spatial_layers; ++sl_idx) {
+    SpatialLayer& spatial_layer = video_codec.spatialLayers[sl_idx];
+    spatial_layer.width = video_codec.width * info.scaling_factor_num[sl_idx] /
+                          info.scaling_factor_den[sl_idx];
+    spatial_layer.height = video_codec.height *
+                           info.scaling_factor_num[sl_idx] /
+                           info.scaling_factor_den[sl_idx];
+    spatial_layer.maxFramerate = video_codec.maxFramerate;
+    spatial_layer.numberOfTemporalLayers = info.num_temporal_layers;
+    spatial_layer.active = true;
+  }
+
+  if (info.num_spatial_layers == 1) {
+    SpatialLayer& spatial_layer = video_codec.spatialLayers[0];
+    spatial_layer.minBitrate = video_codec.minBitrate;
+    spatial_layer.targetBitrate = video_codec.startBitrate;
+    spatial_layer.maxBitrate = video_codec.maxBitrate;
+    return true;
+  }
+
+  for (int sl_idx = 0; sl_idx < info.num_spatial_layers; ++sl_idx) {
+    SpatialLayer& spatial_layer = video_codec.spatialLayers[sl_idx];
+    // minBitrate and maxBitrate formulas are copied from vp9 settings and
+    // are not yet tuned for av1.
+    const int num_pixels = spatial_layer.width * spatial_layer.height;
+    int min_bitrate_kbps = (600.0 * std::sqrt(num_pixels) - 95'000.0) / 1000.0;
+    spatial_layer.minBitrate = std::max(min_bitrate_kbps, 20);
+    spatial_layer.maxBitrate = 50 + static_cast<int>(1.6 * num_pixels / 1000.0);
+    spatial_layer.targetBitrate =
+        (spatial_layer.minBitrate + spatial_layer.maxBitrate) / 2;
+  }
+  return true;
+}
+
+}  // namespace webrtc

third_party/libwebrtc/modules/video_coding/codecs/av1/av1_svc_config.h

@@ -0,0 +1,22 @@
+/* Copyright (c) 2020 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef MODULES_VIDEO_CODING_CODECS_AV1_AV1_SVC_CONFIG_H_
+#define MODULES_VIDEO_CODING_CODECS_AV1_AV1_SVC_CONFIG_H_
+
+#include "api/video_codecs/video_codec.h"
+
+namespace webrtc {
+
+// Fills `video_codec.spatialLayers` using other members.
+bool SetAv1SvcConfig(VideoCodec& video_codec);
+
+}  // namespace webrtc
+
+#endif  // MODULES_VIDEO_CODING_CODECS_AV1_AV1_SVC_CONFIG_H_

third_party/libwebrtc/modules/video_coding/codecs/av1/av1_svc_config_unittest.cc

@@ -0,0 +1,142 @@
+/*
+ *  Copyright (c) 2020 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "modules/video_coding/codecs/av1/av1_svc_config.h"
+
+#include "api/video_codecs/video_codec.h"
+#include "test/gmock.h"
+#include "test/gtest.h"
+
+namespace webrtc {
+namespace {
+
+TEST(Av1SvcConfigTest, RequireScalabilityMode) {
+  VideoCodec video_codec;
+  video_codec.codecType = kVideoCodecAV1;
+
+  video_codec.SetScalabilityMode("");
+  EXPECT_FALSE(SetAv1SvcConfig(video_codec));
+
+  video_codec.SetScalabilityMode("Unknown");
+  EXPECT_FALSE(SetAv1SvcConfig(video_codec));
+
+  video_codec.SetScalabilityMode("NONE");
+  EXPECT_TRUE(SetAv1SvcConfig(video_codec));
+}
+
+TEST(Av1SvcConfigTest, SetsActiveSpatialLayersFromScalabilityMode) {
+  VideoCodec video_codec;
+  video_codec.codecType = kVideoCodecAV1;
+  video_codec.SetScalabilityMode("L2T1");
+
+  EXPECT_TRUE(SetAv1SvcConfig(video_codec));
+
+  EXPECT_TRUE(video_codec.spatialLayers[0].active);
+  EXPECT_TRUE(video_codec.spatialLayers[1].active);
+  EXPECT_FALSE(video_codec.spatialLayers[2].active);
+}
+
+TEST(Av1SvcConfigTest, ConfiguresDobuleResolutionRatioFromScalabilityMode) {
+  VideoCodec video_codec;
+  video_codec.codecType = kVideoCodecAV1;
+  video_codec.SetScalabilityMode("L2T1");
+  video_codec.width = 1200;
+  video_codec.height = 800;
+
+  EXPECT_TRUE(SetAv1SvcConfig(video_codec));
+
+  EXPECT_EQ(video_codec.spatialLayers[0].width, 600);
+  EXPECT_EQ(video_codec.spatialLayers[0].height, 400);
+  EXPECT_EQ(video_codec.spatialLayers[1].width, 1200);
+  EXPECT_EQ(video_codec.spatialLayers[1].height, 800);
+}
+
+TEST(Av1SvcConfigTest, ConfiguresSmallResolutionRatioFromScalabilityMode) {
+  VideoCodec video_codec;
+  video_codec.codecType = kVideoCodecAV1;
+  // h mode uses 1.5:1 ratio
+  video_codec.SetScalabilityMode("L2T1h");
+  video_codec.width = 1500;
+  video_codec.height = 900;
+
+  EXPECT_TRUE(SetAv1SvcConfig(video_codec));
+
+  EXPECT_EQ(video_codec.spatialLayers[0].width, 1000);
+  EXPECT_EQ(video_codec.spatialLayers[0].height, 600);
+  EXPECT_EQ(video_codec.spatialLayers[1].width, 1500);
+  EXPECT_EQ(video_codec.spatialLayers[1].height, 900);
+}
+
+TEST(Av1SvcConfigTest, CopiesFramrate) {
+  VideoCodec video_codec;
+  video_codec.codecType = kVideoCodecAV1;
+  // h mode uses 1.5:1 ratio
+  video_codec.SetScalabilityMode("L2T1");
+  video_codec.maxFramerate = 27;
+
+  EXPECT_TRUE(SetAv1SvcConfig(video_codec));
+
+  EXPECT_EQ(video_codec.spatialLayers[0].maxFramerate, 27);
+  EXPECT_EQ(video_codec.spatialLayers[1].maxFramerate, 27);
+}
+
+TEST(Av1SvcConfigTest, SetsNumberOfTemporalLayers) {
+  VideoCodec video_codec;
+  video_codec.codecType = kVideoCodecAV1;
+  video_codec.SetScalabilityMode("L1T3");
+
+  EXPECT_TRUE(SetAv1SvcConfig(video_codec));
+
+  EXPECT_EQ(video_codec.spatialLayers[0].numberOfTemporalLayers, 3);
+}
+
+TEST(Av1SvcConfigTest, CopiesBitrateForSingleSpatialLayer) {
+  VideoCodec video_codec;
+  video_codec.codecType = kVideoCodecAV1;
+  video_codec.SetScalabilityMode("L1T3");
+  video_codec.minBitrate = 100;
+  video_codec.startBitrate = 200;
+  video_codec.maxBitrate = 500;
+
+  EXPECT_TRUE(SetAv1SvcConfig(video_codec));
+
+  EXPECT_EQ(video_codec.spatialLayers[0].minBitrate, 100u);
+  EXPECT_EQ(video_codec.spatialLayers[0].targetBitrate, 200u);
+  EXPECT_EQ(video_codec.spatialLayers[0].maxBitrate, 500u);
+}
+
+TEST(Av1SvcConfigTest, SetsBitratesForMultipleSpatialLayers) {
+  VideoCodec video_codec;
+  video_codec.codecType = kVideoCodecAV1;
+  video_codec.SetScalabilityMode("L3T3");
+
+  EXPECT_TRUE(SetAv1SvcConfig(video_codec));
+
+  EXPECT_GT(video_codec.spatialLayers[0].minBitrate, 0u);
+  EXPECT_LE(video_codec.spatialLayers[0].minBitrate,
+            video_codec.spatialLayers[0].targetBitrate);
+  EXPECT_LE(video_codec.spatialLayers[0].targetBitrate,
+            video_codec.spatialLayers[0].maxBitrate);
+
+  EXPECT_GT(video_codec.spatialLayers[1].minBitrate, 0u);
+  EXPECT_LE(video_codec.spatialLayers[1].minBitrate,
+            video_codec.spatialLayers[1].targetBitrate);
+  EXPECT_LE(video_codec.spatialLayers[1].targetBitrate,
+            video_codec.spatialLayers[1].maxBitrate);
+
+  EXPECT_GT(video_codec.spatialLayers[2].minBitrate, 0u);
+  EXPECT_LE(video_codec.spatialLayers[2].minBitrate,
+            video_codec.spatialLayers[2].targetBitrate);
+  EXPECT_LE(video_codec.spatialLayers[2].targetBitrate,
+            video_codec.spatialLayers[2].maxBitrate);
+}
+
+}  // namespace
+}  // namespace webrtc

third_party/libwebrtc/modules/video_coding/video_codec_initializer.cc

@@ -20,6 +20,7 @@
 #include "api/units/data_rate.h"
 #include "api/video/video_bitrate_allocation.h"
 #include "api/video_codecs/video_encoder.h"
+#include "modules/video_coding/codecs/av1/av1_svc_config.h"
 #include "modules/video_coding/codecs/vp9/svc_config.h"
 #include "modules/video_coding/include/video_coding_defines.h"
 #include "rtc_base/checks.h"
@@ -56,7 +57,6 @@ VideoCodec VideoCodecInitializer::VideoEncoderConfigToVideoCodec(
   RTC_DCHECK_GE(config.min_transmit_bitrate_bps, 0);
 
   VideoCodec video_codec;
-  memset(&video_codec, 0, sizeof(video_codec));
   video_codec.codecType = config.codec_type;
 
   switch (config.content_type) {
@@ -255,6 +255,11 @@ VideoCodec VideoCodecInitializer::VideoEncoderConfigToVideoCodec(
 
       break;
     }
+    case kVideoCodecAV1:
+      if (!SetAv1SvcConfig(video_codec)) {
+        RTC_LOG(LS_WARNING) << "Failed to configure svc bitrates for av1.";
+      }
+      break;
     case kVideoCodecH264: {
       if (!config.encoder_specific_settings)
         *video_codec.H264() = VideoEncoder::GetDefaultH264Settings();