Merge branch 'release/2022.6.0' into main

CHANGES.md

@@ -11,7 +11,12 @@
 
 ## develop
 
-## 2022.5.0 (2022/6/22)
+## 2022.6.0 (2022-06-30)
+
+- [ADD] Jetson の AV1 HWA に対応
+    - @melpon
+
+## 2022.5.0 (2022-06-22)
 
 - [ADD] bundle_id を追加
     - @melpon

README.md

@@ -34,7 +34,7 @@ https://github.com/shiguredo/sora-cpp-sdk/releases
 
 ## 対応 Sora
 
-- WebRTC SFU Sora 2022.1 以降
+- WebRTC SFU Sora 2022.1.0 以降
 
 ## 動作環境
 

VERSION

@@ -1,4 +1,4 @@
-SORA_CPP_SDK_VERSION=2022.5.0
+SORA_CPP_SDK_VERSION=2022.6.0
 WEBRTC_BUILD_VERSION=m102.5005.7.5
 BOOST_VERSION=1.79.0
 CMAKE_VERSION=3.23.1
@@ -7,4 +7,4 @@ ANDROID_NDK_VERSION=r23b
 ANDROID_NATIVE_API_LEVEL=24
 ANDROID_SDK_CMDLINE_TOOLS_VERSION=8092744
 LIBVA_VERSION=2.7.0
-MSDK_VERSION=intel-mediasdk-20.1.1
+MSDK_VERSION=intel-mediasdk-20.1.1

include/sora/hwenc_jetson/jetson_video_decoder.h

@@ -28,6 +28,9 @@ class JetsonVideoDecoder : public webrtc::VideoDecoder {
   JetsonVideoDecoder(webrtc::VideoCodecType codec);
   ~JetsonVideoDecoder() override;
 
+  static bool IsSupportedVP8();
+  static bool IsSupportedAV1();
+
   bool Configure(const Settings& settings) override;
 
   int32_t Decode(const webrtc::EncodedImage& input_image,

include/sora/hwenc_jetson/jetson_video_encoder.h

@@ -24,7 +24,8 @@
 #include <common_video/include/bitrate_adjuster.h>
 #include <modules/video_coding/codecs/h264/include/h264.h>
 #include <modules/video_coding/codecs/vp9/include/vp9_globals.h>
-#include "rtc_base/synchronization/mutex.h"
+#include <modules/video_coding/svc/scalable_video_controller.h>
+#include <rtc_base/synchronization/mutex.h>
 
 #include "jetson_jpeg_decoder.h"
 
@@ -45,6 +46,7 @@ class JetsonVideoEncoder : public webrtc::VideoEncoder {
 
   static bool IsSupportedVP8();
   static bool IsSupportedVP9();
+  static bool IsSupportedAV1();
 
   int32_t InitEncode(const webrtc::VideoCodec* codec_settings,
                      int32_t number_of_cores,
@@ -144,6 +146,8 @@ class JetsonVideoEncoder : public webrtc::VideoEncoder {
   webrtc::GofInfoVP9 gof_;
   size_t gof_idx_;
 
+  std::unique_ptr<webrtc::ScalableVideoController> svc_controller_;
+
   webrtc::Mutex frame_params_lock_;
   std::queue<std::unique_ptr<FrameParams>> frame_params_;
   std::mutex enc0_buffer_mtx_;

src/hwenc_jetson/jetson_video_decoder.cpp

@@ -53,6 +53,24 @@ JetsonVideoDecoder::~JetsonVideoDecoder() {
   Release();
 }
 
+bool JetsonVideoDecoder::IsSupportedVP8() {
+  //SuppressErrors sup;
+
+  auto decoder = NvVideoDecoder::createVideoDecoder("dec0");
+  auto ret = decoder->setOutputPlaneFormat(V4L2_PIX_FMT_VP8, CHUNK_SIZE);
+  delete decoder;
+  return ret >= 0;
+}
+
+bool JetsonVideoDecoder::IsSupportedAV1() {
+  //SuppressErrors sup;
+
+  auto decoder = NvVideoDecoder::createVideoDecoder("dec0");
+  auto ret = decoder->setOutputPlaneFormat(V4L2_PIX_FMT_AV1, CHUNK_SIZE);
+  delete decoder;
+  return ret >= 0;
+}
+
 bool JetsonVideoDecoder::Configure(const Settings& settings) {
   if (JetsonConfigure() != WEBRTC_VIDEO_CODEC_OK) {
     RTC_LOG(LS_ERROR) << __FUNCTION__ << "Failed to JetsonConfigure";

src/hwenc_jetson/jetson_video_encoder.cpp

@@ -16,6 +16,7 @@
 
 // WebRTC
 #include <common_video/libyuv/include/webrtc_libyuv.h>
+#include <modules/video_coding/svc/create_scalability_structure.h>
 #include <rtc_base/checks.h>
 #include <rtc_base/logging.h>
 #include <rtc_base/time_utils.h>
@@ -99,6 +100,17 @@ bool JetsonVideoEncoder::IsSupportedVP9() {
   return ret >= 0;
 }
 
+bool JetsonVideoEncoder::IsSupportedAV1() {
+  //SuppressErrors sup;
+
+  auto encoder = NvVideoEncoder::createVideoEncoder("enc0");
+  auto ret = encoder->setCapturePlaneFormat(V4L2_PIX_FMT_AV1, 1024, 768,
+                                            2 * 1024 * 1024);
+  delete encoder;
+
+  return ret >= 0;
+}
+
 int32_t JetsonVideoEncoder::InitEncode(const webrtc::VideoCodec* codec_settings,
                                        int32_t number_of_cores,
                                        size_t max_payload_size) {
@@ -137,6 +149,14 @@ int32_t JetsonVideoEncoder::InitEncode(const webrtc::VideoCodec* codec_settings,
                      << codec_settings->VP9().numberOfSpatialLayers;
     RTC_LOG(LS_INFO) << "interLayerPred: "
                      << codec_settings->VP9().interLayerPred;
+  } else if (codec_settings->codecType == webrtc::kVideoCodecAV1) {
+    absl::string_view scalability_mode = codec_settings->ScalabilityMode();
+    if (scalability_mode.empty()) {
+      RTC_LOG(LS_WARNING) << "Scalability mode is not set, using 'L1T1'.";
+      scalability_mode = "L1T1";
+    }
+    RTC_LOG(LS_INFO) << "InitEncode scalability_mode:" << scalability_mode;
+    svc_controller_ = webrtc::CreateScalabilityStructure(scalability_mode);
   }
   framerate_ = codec_settings->maxFramerate;
 
@@ -235,6 +255,9 @@ int32_t JetsonVideoEncoder::JetsonConfigure() {
   } else if (codec_.codecType == webrtc::kVideoCodecVP9) {
     ret = encoder_->setCapturePlaneFormat(V4L2_PIX_FMT_VP9, width_, height_,
                                           2 * 1024 * 1024);
+  } else if (codec_.codecType == webrtc::kVideoCodecAV1) {
+    ret = encoder_->setCapturePlaneFormat(V4L2_PIX_FMT_AV1, width_, height_,
+                                          2 * 1024 * 1024);
   }
   INIT_ERROR(ret < 0, "Failed to encoder setCapturePlaneFormat");
 
@@ -609,6 +632,10 @@ void JetsonVideoEncoder::SetRates(const RateControlParameters& parameters) {
 
   RTC_LOG(LS_INFO) << __FUNCTION__ << " framerate:" << parameters.framerate_fps
                    << " bitrate:" << parameters.bitrate.ToString();
+  if (svc_controller_) {
+    svc_controller_->OnRatesUpdated(parameters.bitrate);
+  }
+
   framerate_ = parameters.framerate_fps;
   target_bitrate_bps_ = parameters.bitrate.get_sum_bps();
 
@@ -671,6 +698,11 @@ webrtc::VideoEncoder::EncoderInfo JetsonVideoEncoder::GetEncoderInfo() const {
     static const int kHighVp9QpThreshold = 151;
     info.scaling_settings =
         VideoEncoder::ScalingSettings(kLowVp9QpThreshold, kHighVp9QpThreshold);
+  } else if (codec_.codecType == webrtc::kVideoCodecAV1) {
+    static const int kLowAv1QpThreshold = 145;
+    static const int kHighAv1QpThreshold = 205;
+    info.scaling_settings =
+        VideoEncoder::ScalingSettings(kLowAv1QpThreshold, kHighAv1QpThreshold);
   }
   return info;
 }
@@ -931,9 +963,10 @@ int32_t JetsonVideoEncoder::SendFrame(
 
     codec_specific.codecSpecific.H264.packetization_mode =
         webrtc::H264PacketizationMode::NonInterleaved;
-  } else if (codec_.codecType == webrtc::kVideoCodecVP9 ||
+  } else if (codec_.codecType == webrtc::kVideoCodecAV1 ||
+             codec_.codecType == webrtc::kVideoCodecVP9 ||
              codec_.codecType == webrtc::kVideoCodecVP8) {
-    // VP8, VP9 はIVFヘッダーがエンコードフレームについているので取り除く
+    // VP8, VP9, AV1 はIVFヘッダーがエンコードフレームについているので取り除く
     if ((buffer[0] == 'D') && (buffer[1] == 'K') && (buffer[2] == 'I') &&
         (buffer[3] == 'F')) {
       buffer += 32;
@@ -977,6 +1010,27 @@ int32_t JetsonVideoEncoder::SendFrame(
             encoded_image_._encodedHeight;
         codec_specific.codecSpecific.VP9.gof.CopyGofInfoVP9(gof_);
       }
+    } else if (codec_.codecType == webrtc::kVideoCodecAV1) {
+      bool is_key = buffer[2] == 0x0a;
+      // v4l2_ctrl_videoenc_outputbuf_metadata.KeyFrame が効いていない
+      // キーフレームの時には OBU_SEQUENCE_HEADER が入っているために 0x0a になるためこれを使う
+      // キーフレームではない時には OBU_FRAME が入っていて 0x32 になっている
+      if (is_key) {
+        encoded_image_._frameType = webrtc::VideoFrameType::kVideoFrameKey;
+      }
+
+      std::vector<webrtc::ScalableVideoController::LayerFrameConfig>
+          layer_frames = svc_controller_->NextFrameConfig(is_key);
+      codec_specific.end_of_picture = true;
+      codec_specific.generic_frame_info =
+          svc_controller_->OnEncodeDone(layer_frames[0]);
+      if (is_key && codec_specific.generic_frame_info) {
+        codec_specific.template_structure =
+            svc_controller_->DependencyStructure();
+        auto& resolutions = codec_specific.template_structure->resolutions;
+        resolutions = {webrtc::RenderResolution(encoded_image_._encodedWidth,
+                                                encoded_image_._encodedHeight)};
+      }
     }
   }
 

src/sora_video_decoder_factory.cpp

@@ -206,12 +206,22 @@ SoraVideoDecoderFactoryConfig GetDefaultVideoDecoderFactoryConfig(
 #endif
 
 #if USE_JETSON_ENCODER
-  config.decoders.insert(
-      config.decoders.begin(),
-      VideoDecoderConfig(webrtc::kVideoCodecVP8, [](auto format) {
-        return std::unique_ptr<webrtc::VideoDecoder>(
-            absl::make_unique<JetsonVideoDecoder>(webrtc::kVideoCodecVP8));
-      }));
+  if (JetsonVideoDecoder::IsSupportedVP8()) {
+    config.decoders.insert(
+        config.decoders.begin(),
+        VideoDecoderConfig(webrtc::kVideoCodecVP8, [](auto format) {
+          return std::unique_ptr<webrtc::VideoDecoder>(
+              absl::make_unique<JetsonVideoDecoder>(webrtc::kVideoCodecVP8));
+        }));
+  }
+  if (JetsonVideoDecoder::IsSupportedAV1()) {
+    config.decoders.insert(
+        config.decoders.begin(),
+        VideoDecoderConfig(webrtc::kVideoCodecAV1, [](auto format) {
+          return std::unique_ptr<webrtc::VideoDecoder>(
+              absl::make_unique<JetsonVideoDecoder>(webrtc::kVideoCodecAV1));
+        }));
+  }
   config.decoders.insert(
       config.decoders.begin(),
       VideoDecoderConfig(webrtc::kVideoCodecVP9, [](auto format) {

src/sora_video_encoder_factory.cpp

@@ -209,7 +209,7 @@ SoraVideoEncoderFactoryConfig GetDefaultVideoEncoderFactoryConfig(
                   cricket::VideoCodec(format)));
         }));
   }
-  if (JetsonVideoEncoder::IsSupportedVP8()) {
+  if (JetsonVideoEncoder::IsSupportedVP9()) {
     config.encoders.insert(
         config.encoders.begin(),
         VideoEncoderConfig(webrtc::kVideoCodecVP9, [](auto format) {
@@ -218,6 +218,15 @@ SoraVideoEncoderFactoryConfig GetDefaultVideoEncoderFactoryConfig(
                   cricket::VideoCodec(format)));
         }));
   }
+  if (JetsonVideoEncoder::IsSupportedAV1()) {
+    config.encoders.insert(
+        config.encoders.begin(),
+        VideoEncoderConfig(webrtc::kVideoCodecAV1, [](auto format) {
+          return std::unique_ptr<webrtc::VideoEncoder>(
+              absl::make_unique<JetsonVideoEncoder>(
+                  cricket::VideoCodec(format)));
+        }));
+  }
   config.encoders.insert(
       config.encoders.begin(),
       VideoEncoderConfig(webrtc::kVideoCodecH264, [](auto format) {