From 61d01840cce2948545fb23721857d62780b9ee3d Mon Sep 17 00:00:00 2001 From: voluntas Date: Wed, 22 Jun 2022 09:39:05 +0900 Subject: [PATCH 1/3] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index d94720e7..fd84bedb 100644 --- a/README.md +++ b/README.md @@ -34,7 +34,7 @@ https://github.com/shiguredo/sora-cpp-sdk/releases ## 対応 Sora -- WebRTC SFU Sora 2022.1 以降 +- WebRTC SFU Sora 2022.1.0 以降 ## 動作環境 From e73b6f9e736c30244d39706b66af47a4ff66ed50 Mon Sep 17 00:00:00 2001 From: melpon Date: Thu, 30 Jun 2022 03:02:16 +0900 Subject: [PATCH 2/3] =?UTF-8?q?Jetson=20=E3=81=AE=20AV1=20HWA=20=E3=81=AB?= =?UTF-8?q?=E5=AF=BE=E5=BF=9C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGES.md | 3 + .../sora/hwenc_jetson/jetson_video_decoder.h | 3 + .../sora/hwenc_jetson/jetson_video_encoder.h | 6 +- src/hwenc_jetson/jetson_video_decoder.cpp | 18 ++++++ src/hwenc_jetson/jetson_video_encoder.cpp | 58 ++++++++++++++++++- src/sora_video_decoder_factory.cpp | 22 +++++-- src/sora_video_encoder_factory.cpp | 11 +++- 7 files changed, 111 insertions(+), 10 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 9b9df11d..d579d3c8 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -11,6 +11,9 @@ ## develop +- [ADD] Jetson の AV1 HWA に対応 + - @melpon + ## 2022.5.0 (2022/6/22) - [ADD] bundle_id を追加 diff --git a/include/sora/hwenc_jetson/jetson_video_decoder.h b/include/sora/hwenc_jetson/jetson_video_decoder.h index 6a265317..b8cc9566 100644 --- a/include/sora/hwenc_jetson/jetson_video_decoder.h +++ b/include/sora/hwenc_jetson/jetson_video_decoder.h @@ -28,6 +28,9 @@ class JetsonVideoDecoder : public webrtc::VideoDecoder { JetsonVideoDecoder(webrtc::VideoCodecType codec); ~JetsonVideoDecoder() override; + static bool IsSupportedVP8(); + static bool IsSupportedAV1(); + bool Configure(const Settings& settings) override; int32_t Decode(const webrtc::EncodedImage& input_image, diff --git a/include/sora/hwenc_jetson/jetson_video_encoder.h b/include/sora/hwenc_jetson/jetson_video_encoder.h index 82c569ee..34ffa526 100644 --- a/include/sora/hwenc_jetson/jetson_video_encoder.h +++ b/include/sora/hwenc_jetson/jetson_video_encoder.h @@ -24,7 +24,8 @@ #include #include #include -#include "rtc_base/synchronization/mutex.h" +#include +#include #include "jetson_jpeg_decoder.h" @@ -45,6 +46,7 @@ class JetsonVideoEncoder : public webrtc::VideoEncoder { static bool IsSupportedVP8(); static bool IsSupportedVP9(); + static bool IsSupportedAV1(); int32_t InitEncode(const webrtc::VideoCodec* codec_settings, int32_t number_of_cores, @@ -144,6 +146,8 @@ class JetsonVideoEncoder : public webrtc::VideoEncoder { webrtc::GofInfoVP9 gof_; size_t gof_idx_; + std::unique_ptr svc_controller_; + webrtc::Mutex frame_params_lock_; std::queue> frame_params_; std::mutex enc0_buffer_mtx_; diff --git a/src/hwenc_jetson/jetson_video_decoder.cpp b/src/hwenc_jetson/jetson_video_decoder.cpp index e19cb98a..61f9fa07 100644 --- a/src/hwenc_jetson/jetson_video_decoder.cpp +++ b/src/hwenc_jetson/jetson_video_decoder.cpp @@ -53,6 +53,24 @@ JetsonVideoDecoder::~JetsonVideoDecoder() { Release(); } +bool JetsonVideoDecoder::IsSupportedVP8() { + //SuppressErrors sup; + + auto decoder = NvVideoDecoder::createVideoDecoder("dec0"); + auto ret = decoder->setOutputPlaneFormat(V4L2_PIX_FMT_VP8, CHUNK_SIZE); + delete decoder; + return ret >= 0; +} + +bool JetsonVideoDecoder::IsSupportedAV1() { + //SuppressErrors sup; + + auto decoder = NvVideoDecoder::createVideoDecoder("dec0"); + auto ret = decoder->setOutputPlaneFormat(V4L2_PIX_FMT_AV1, CHUNK_SIZE); + delete decoder; + return ret >= 0; +} + bool JetsonVideoDecoder::Configure(const Settings& settings) { if (JetsonConfigure() != WEBRTC_VIDEO_CODEC_OK) { RTC_LOG(LS_ERROR) << __FUNCTION__ << "Failed to JetsonConfigure"; diff --git a/src/hwenc_jetson/jetson_video_encoder.cpp b/src/hwenc_jetson/jetson_video_encoder.cpp index 4768a328..ca62cd7d 100644 --- a/src/hwenc_jetson/jetson_video_encoder.cpp +++ b/src/hwenc_jetson/jetson_video_encoder.cpp @@ -16,6 +16,7 @@ // WebRTC #include +#include #include #include #include @@ -99,6 +100,17 @@ bool JetsonVideoEncoder::IsSupportedVP9() { return ret >= 0; } +bool JetsonVideoEncoder::IsSupportedAV1() { + //SuppressErrors sup; + + auto encoder = NvVideoEncoder::createVideoEncoder("enc0"); + auto ret = encoder->setCapturePlaneFormat(V4L2_PIX_FMT_AV1, 1024, 768, + 2 * 1024 * 1024); + delete encoder; + + return ret >= 0; +} + int32_t JetsonVideoEncoder::InitEncode(const webrtc::VideoCodec* codec_settings, int32_t number_of_cores, size_t max_payload_size) { @@ -137,6 +149,14 @@ int32_t JetsonVideoEncoder::InitEncode(const webrtc::VideoCodec* codec_settings, << codec_settings->VP9().numberOfSpatialLayers; RTC_LOG(LS_INFO) << "interLayerPred: " << codec_settings->VP9().interLayerPred; + } else if (codec_settings->codecType == webrtc::kVideoCodecAV1) { + absl::string_view scalability_mode = codec_settings->ScalabilityMode(); + if (scalability_mode.empty()) { + RTC_LOG(LS_WARNING) << "Scalability mode is not set, using 'L1T1'."; + scalability_mode = "L1T1"; + } + RTC_LOG(LS_INFO) << "InitEncode scalability_mode:" << scalability_mode; + svc_controller_ = webrtc::CreateScalabilityStructure(scalability_mode); } framerate_ = codec_settings->maxFramerate; @@ -235,6 +255,9 @@ int32_t JetsonVideoEncoder::JetsonConfigure() { } else if (codec_.codecType == webrtc::kVideoCodecVP9) { ret = encoder_->setCapturePlaneFormat(V4L2_PIX_FMT_VP9, width_, height_, 2 * 1024 * 1024); + } else if (codec_.codecType == webrtc::kVideoCodecAV1) { + ret = encoder_->setCapturePlaneFormat(V4L2_PIX_FMT_AV1, width_, height_, + 2 * 1024 * 1024); } INIT_ERROR(ret < 0, "Failed to encoder setCapturePlaneFormat"); @@ -609,6 +632,10 @@ void JetsonVideoEncoder::SetRates(const RateControlParameters& parameters) { RTC_LOG(LS_INFO) << __FUNCTION__ << " framerate:" << parameters.framerate_fps << " bitrate:" << parameters.bitrate.ToString(); + if (svc_controller_) { + svc_controller_->OnRatesUpdated(parameters.bitrate); + } + framerate_ = parameters.framerate_fps; target_bitrate_bps_ = parameters.bitrate.get_sum_bps(); @@ -671,6 +698,11 @@ webrtc::VideoEncoder::EncoderInfo JetsonVideoEncoder::GetEncoderInfo() const { static const int kHighVp9QpThreshold = 151; info.scaling_settings = VideoEncoder::ScalingSettings(kLowVp9QpThreshold, kHighVp9QpThreshold); + } else if (codec_.codecType == webrtc::kVideoCodecAV1) { + static const int kLowAv1QpThreshold = 145; + static const int kHighAv1QpThreshold = 205; + info.scaling_settings = + VideoEncoder::ScalingSettings(kLowAv1QpThreshold, kHighAv1QpThreshold); } return info; } @@ -931,9 +963,10 @@ int32_t JetsonVideoEncoder::SendFrame( codec_specific.codecSpecific.H264.packetization_mode = webrtc::H264PacketizationMode::NonInterleaved; - } else if (codec_.codecType == webrtc::kVideoCodecVP9 || + } else if (codec_.codecType == webrtc::kVideoCodecAV1 || + codec_.codecType == webrtc::kVideoCodecVP9 || codec_.codecType == webrtc::kVideoCodecVP8) { - // VP8, VP9 はIVFヘッダーがエンコードフレームについているので取り除く + // VP8, VP9, AV1 はIVFヘッダーがエンコードフレームについているので取り除く if ((buffer[0] == 'D') && (buffer[1] == 'K') && (buffer[2] == 'I') && (buffer[3] == 'F')) { buffer += 32; @@ -977,6 +1010,27 @@ int32_t JetsonVideoEncoder::SendFrame( encoded_image_._encodedHeight; codec_specific.codecSpecific.VP9.gof.CopyGofInfoVP9(gof_); } + } else if (codec_.codecType == webrtc::kVideoCodecAV1) { + bool is_key = buffer[2] == 0x0a; + // v4l2_ctrl_videoenc_outputbuf_metadata.KeyFrame が効いていない + // キーフレームの時には OBU_SEQUENCE_HEADER が入っているために 0x0a になるためこれを使う + // キーフレームではない時には OBU_FRAME が入っていて 0x32 になっている + if (is_key) { + encoded_image_._frameType = webrtc::VideoFrameType::kVideoFrameKey; + } + + std::vector + layer_frames = svc_controller_->NextFrameConfig(is_key); + codec_specific.end_of_picture = true; + codec_specific.generic_frame_info = + svc_controller_->OnEncodeDone(layer_frames[0]); + if (is_key && codec_specific.generic_frame_info) { + codec_specific.template_structure = + svc_controller_->DependencyStructure(); + auto& resolutions = codec_specific.template_structure->resolutions; + resolutions = {webrtc::RenderResolution(encoded_image_._encodedWidth, + encoded_image_._encodedHeight)}; + } } } diff --git a/src/sora_video_decoder_factory.cpp b/src/sora_video_decoder_factory.cpp index 6941f19c..d8180177 100644 --- a/src/sora_video_decoder_factory.cpp +++ b/src/sora_video_decoder_factory.cpp @@ -206,12 +206,22 @@ SoraVideoDecoderFactoryConfig GetDefaultVideoDecoderFactoryConfig( #endif #if USE_JETSON_ENCODER - config.decoders.insert( - config.decoders.begin(), - VideoDecoderConfig(webrtc::kVideoCodecVP8, [](auto format) { - return std::unique_ptr( - absl::make_unique(webrtc::kVideoCodecVP8)); - })); + if (JetsonVideoDecoder::IsSupportedVP8()) { + config.decoders.insert( + config.decoders.begin(), + VideoDecoderConfig(webrtc::kVideoCodecVP8, [](auto format) { + return std::unique_ptr( + absl::make_unique(webrtc::kVideoCodecVP8)); + })); + } + if (JetsonVideoDecoder::IsSupportedAV1()) { + config.decoders.insert( + config.decoders.begin(), + VideoDecoderConfig(webrtc::kVideoCodecAV1, [](auto format) { + return std::unique_ptr( + absl::make_unique(webrtc::kVideoCodecAV1)); + })); + } config.decoders.insert( config.decoders.begin(), VideoDecoderConfig(webrtc::kVideoCodecVP9, [](auto format) { diff --git a/src/sora_video_encoder_factory.cpp b/src/sora_video_encoder_factory.cpp index c67d9349..8fef2f7c 100644 --- a/src/sora_video_encoder_factory.cpp +++ b/src/sora_video_encoder_factory.cpp @@ -209,7 +209,7 @@ SoraVideoEncoderFactoryConfig GetDefaultVideoEncoderFactoryConfig( cricket::VideoCodec(format))); })); } - if (JetsonVideoEncoder::IsSupportedVP8()) { + if (JetsonVideoEncoder::IsSupportedVP9()) { config.encoders.insert( config.encoders.begin(), VideoEncoderConfig(webrtc::kVideoCodecVP9, [](auto format) { @@ -218,6 +218,15 @@ SoraVideoEncoderFactoryConfig GetDefaultVideoEncoderFactoryConfig( cricket::VideoCodec(format))); })); } + if (JetsonVideoEncoder::IsSupportedAV1()) { + config.encoders.insert( + config.encoders.begin(), + VideoEncoderConfig(webrtc::kVideoCodecAV1, [](auto format) { + return std::unique_ptr( + absl::make_unique( + cricket::VideoCodec(format))); + })); + } config.encoders.insert( config.encoders.begin(), VideoEncoderConfig(webrtc::kVideoCodecH264, [](auto format) { From 7786783e5726938d7d7c6ad1845d099b1aac4142 Mon Sep 17 00:00:00 2001 From: melpon Date: Thu, 30 Jun 2022 19:57:48 +0900 Subject: [PATCH 3/3] 2022.6.0 --- CHANGES.md | 4 +++- VERSION | 4 ++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index d579d3c8..dbbaa82a 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -11,10 +11,12 @@ ## develop +## 2022.6.0 (2022-06-30) + - [ADD] Jetson の AV1 HWA に対応 - @melpon -## 2022.5.0 (2022/6/22) +## 2022.5.0 (2022-06-22) - [ADD] bundle_id を追加 - @melpon diff --git a/VERSION b/VERSION index eb4cb591..c592bcef 100644 --- a/VERSION +++ b/VERSION @@ -1,4 +1,4 @@ -SORA_CPP_SDK_VERSION=2022.5.0 +SORA_CPP_SDK_VERSION=2022.6.0 WEBRTC_BUILD_VERSION=m102.5005.7.5 BOOST_VERSION=1.79.0 CMAKE_VERSION=3.23.1 @@ -7,4 +7,4 @@ ANDROID_NDK_VERSION=r23b ANDROID_NATIVE_API_LEVEL=24 ANDROID_SDK_CMDLINE_TOOLS_VERSION=8092744 LIBVA_VERSION=2.7.0 -MSDK_VERSION=intel-mediasdk-20.1.1 \ No newline at end of file +MSDK_VERSION=intel-mediasdk-20.1.1