From 26dbba7de74f601c5fa998ebd534df8d0a1f1de9 Mon Sep 17 00:00:00 2001 From: Keukhan Date: Mon, 6 Jan 2025 11:17:42 +0900 Subject: [PATCH] Added WebP image codec for thumbnails --- misc/prerequisites.sh | 19 ++- src/projects/base/common_types.h | 2 + src/projects/base/info/media_track.cpp | 3 +- src/projects/base/mediarouter/media_type.h | 22 +++- .../mediarouter/mediarouter_stream.cpp | 9 +- .../modules/segment_writer/writer.cpp | 3 + .../thumbnail/thumbnail_interceptor.cpp | 10 +- .../thumbnail/thumbnail_publisher.cpp | 24 +++- .../thumbnail/thumbnail_publisher.h | 2 + .../publishers/thumbnail/thumbnail_stream.cpp | 9 +- .../transcoder/codec/encoder/encoder_jpeg.cpp | 2 +- .../transcoder/codec/encoder/encoder_png.cpp | 2 +- .../transcoder/codec/encoder/encoder_webp.cpp | 119 ++++++++++++++++++ .../transcoder/codec/encoder/encoder_webp.h | 42 +++++++ .../transcoder/transcoder_encoder.cpp | 12 ++ 15 files changed, 256 insertions(+), 24 deletions(-) create mode 100755 src/projects/transcoder/codec/encoder/encoder_webp.cpp create mode 100755 src/projects/transcoder/codec/encoder/encoder_webp.h diff --git a/misc/prerequisites.sh b/misc/prerequisites.sh index 36593c9cf..5d38aae83 100755 --- a/misc/prerequisites.sh +++ b/misc/prerequisites.sh @@ -20,7 +20,7 @@ OPENH264_VERSION=2.4.0 HIREDIS_VERSION=1.0.2 NVCC_HDR_VERSION=11.1.5.2 X264_VERSION=31e19f92 - +WEBP_VERSION=1.5.0 INTEL_QSV_HWACCELS=false NETINT_LOGAN_HWACCELS=false NETINT_LOGAN_PATCH_PATH="" @@ -173,6 +173,18 @@ install_libvpx() rm -rf ${DIR}) || fail_exit "vpx" } +install_libwebp() +{ + (DIR=${TEMP_PATH}/webp && \ + mkdir -p ${DIR} && \ + cd ${DIR} && \ + curl -sSLf https://storage.googleapis.com/downloads.webmproject.org/releases/webp/libwebp-${WEBP_VERSION}.tar.gz | tar -xz --strip-components=1 && \ + ./configure --prefix="${PREFIX}" --enable-shared --disable-static && \ + make -j$(nproc) && \ + sudo make install && \ + rm -rf ${DIR}) || fail_exit "webp" +} + install_fdk_aac() { (DIR=${TEMP_PATH}/aac && \ @@ -344,10 +356,10 @@ install_ffmpeg() --extra-libs=-ldl ${ADDI_EXTRA_LIBS} \ ${ADDI_LICENSE} \ --disable-everything --disable-programs --disable-avdevice --disable-dwt --disable-lsp --disable-lzo --disable-faan --disable-pixelutils \ - --enable-shared --enable-zlib --enable-libopus --enable-libvpx --enable-libfdk_aac --enable-libopenh264 --enable-openssl --enable-network --enable-libsrt --enable-dct --enable-rdft ${ADDI_LIBS} \ + --enable-shared --enable-zlib --enable-libopus --enable-libvpx --enable-libfdk_aac --enable-libopenh264 --enable-openssl --enable-network --enable-libsrt --enable-dct --enable-rdft --enable-libwebp ${ADDI_LIBS} \ ${ADDI_HWACCEL} \ --enable-ffmpeg \ - --enable-encoder=libvpx_vp8,libopus,libfdk_aac,libopenh264,mjpeg,png${ADDI_ENCODER} \ + --enable-encoder=libvpx_vp8,libopus,libfdk_aac,libopenh264,mjpeg,png,libwebp${ADDI_ENCODER} \ --enable-decoder=aac,aac_latm,aac_fixed,mp3float,mp3,h264,hevc,opus,vp8${ADDI_DECODER} \ --enable-parser=aac,aac_latm,aac_fixed,h264,hevc,opus,vp8 \ --enable-protocol=tcp,udp,rtp,file,rtmp,tls,rtmps,libsrt \ @@ -597,6 +609,7 @@ install_libopus install_libopenh264 install_libx264 install_libvpx +install_libwebp install_fdk_aac install_nvcc_hdr install_ffmpeg diff --git a/src/projects/base/common_types.h b/src/projects/base/common_types.h index 23b655157..eefcf0791 100644 --- a/src/projects/base/common_types.h +++ b/src/projects/base/common_types.h @@ -427,6 +427,8 @@ static ov::String StringFromMediaCodecId(const cmn::MediaCodecId &type) return "JPEG"; case cmn::MediaCodecId::Png: return "PNG"; + case cmn::MediaCodecId::Webp: + return "WEBP"; case cmn::MediaCodecId::None: default: return "Unknown"; diff --git a/src/projects/base/info/media_track.cpp b/src/projects/base/info/media_track.cpp index 32f8c6c98..83103ab0d 100644 --- a/src/projects/base/info/media_track.cpp +++ b/src/projects/base/info/media_track.cpp @@ -425,7 +425,8 @@ bool MediaTrack::IsValid() } break; case MediaCodecId::Jpeg: - case MediaCodecId::Png: { + case MediaCodecId::Png: + case MediaCodecId::Webp: { if (_width > 0 && _height > 0 && _time_base.GetNum() > 0 && diff --git a/src/projects/base/mediarouter/media_type.h b/src/projects/base/mediarouter/media_type.h index 4719fbd46..1ccb2ef16 100644 --- a/src/projects/base/mediarouter/media_type.h +++ b/src/projects/base/mediarouter/media_type.h @@ -40,6 +40,7 @@ namespace cmn OPUS_RTP_RFC_7587, JPEG, PNG, + WEBP, // For Data Track ID3v2, @@ -98,6 +99,7 @@ namespace cmn Opus, Jpeg, Png, + Webp }; enum class MediaCodecModuleId : uint8_t @@ -138,9 +140,11 @@ namespace cmn return false; } - static bool IsImageCodec(cmn::MediaCodecId codec_id) { - if (codec_id == cmn::MediaCodecId::Jpeg || - codec_id == cmn::MediaCodecId::Png) + static bool IsImageCodec(cmn::MediaCodecId codec_id) + { + if (codec_id == cmn::MediaCodecId::Jpeg || + codec_id == cmn::MediaCodecId::Png || + codec_id == cmn::MediaCodecId::Webp) { return true; } @@ -234,6 +238,8 @@ namespace cmn return "JPEG"; case cmn::BitstreamFormat::PNG: return "PNG"; + case cmn::BitstreamFormat::WEBP: + return "WEBP"; case cmn::BitstreamFormat::ID3v2: return "ID3v2"; case cmn::BitstreamFormat::OVEN_EVENT: @@ -360,10 +366,13 @@ namespace cmn return "VP9"; case cmn::MediaCodecId::Flv: return "FLV"; + // Image codecs case cmn::MediaCodecId::Jpeg: return "JPEG"; case cmn::MediaCodecId::Png: return "PNG"; + case cmn::MediaCodecId::Webp: + return "WEBP"; // Audio codecs case cmn::MediaCodecId::Aac: return "AAC"; @@ -403,6 +412,7 @@ namespace cmn { return cmn::MediaCodecId::Flv; } + // Image codecs else if (name.HasPrefix("JPEG")) { return cmn::MediaCodecId::Jpeg; @@ -411,8 +421,12 @@ namespace cmn { return cmn::MediaCodecId::Png; } + else if (name.HasPrefix("WEBP")) + { + return cmn::MediaCodecId::Webp; + } // Audio codecs - if (name.HasPrefix("AAC")) + else if (name.HasPrefix("AAC")) { return cmn::MediaCodecId::Aac; } diff --git a/src/projects/mediarouter/mediarouter_stream.cpp b/src/projects/mediarouter/mediarouter_stream.cpp index c0eb76038..d923517af 100644 --- a/src/projects/mediarouter/mediarouter_stream.cpp +++ b/src/projects/mediarouter/mediarouter_stream.cpp @@ -894,14 +894,12 @@ bool MediaRouteStream::NormalizeMediaPacket(std::shared_ptr &media_t break; case cmn::BitstreamFormat::JPEG: case cmn::BitstreamFormat::PNG: - { - if (GetInoutType() == MediaRouterStreamType::OUTBOUND) + case cmn::BitstreamFormat::WEBP: + if (GetInoutType() == MediaRouterStreamType::OUTBOUND) { result = true; } break; - } - case cmn::BitstreamFormat::AAC_LATM: case cmn::BitstreamFormat::Unknown: default: @@ -1055,11 +1053,12 @@ void MediaRouteStream::UpdateStatistics(std::shared_ptr &media_track ov::String stat_stream_str = ""; - stat_stream_str.AppendFormat("Stream. id: %10u, type: %s, name: %s/%s, uptime: %lldms, queue: %d, msid: %u, sync: %lldms", + stat_stream_str.AppendFormat("Stream. id: %10u, type: %s, name: %s/%s, status: %s, uptime: %lldms, queue: %d, msid: %u, sync: %lldms", _stream->GetId(), _inout_type == MediaRouterStreamType::INBOUND ? "Inbound" : "Outbound", _stream->GetApplicationInfo().GetVHostAppName().CStr(), _stream->GetName().CStr(), + IsStreamPrepared() ? "Started" : "Preapring", (int64_t)uptime, _packets_queue.Size(), _stream->GetMsid(), diff --git a/src/projects/modules/segment_writer/writer.cpp b/src/projects/modules/segment_writer/writer.cpp index 4779e754b..121d46063 100644 --- a/src/projects/modules/segment_writer/writer.cpp +++ b/src/projects/modules/segment_writer/writer.cpp @@ -151,6 +151,7 @@ static AVCodecID AvCodecIdFromMediaCodecId(cmn::MediaCodecId codec_id) WRITER_CASE(cmn::MediaCodecId::Opus, AV_CODEC_ID_OPUS) WRITER_CASE(cmn::MediaCodecId::Jpeg, AV_CODEC_ID_JPEG2000) WRITER_CASE(cmn::MediaCodecId::Png, AV_CODEC_ID_PNG) + WRITER_CASE(cmn::MediaCodecId::Webp, AV_CODEC_ID_WEBP) } return AV_CODEC_ID_NONE; @@ -862,6 +863,8 @@ bool Writer::WritePacket(const std::shared_ptr &packet) [[fallthrough]]; case cmn::BitstreamFormat::PNG: [[fallthrough]]; + case cmn::BitstreamFormat::WEBP: + [[fallthrough]]; case cmn::BitstreamFormat::ID3v2: [[fallthrough]]; case cmn::BitstreamFormat::MP3: diff --git a/src/projects/publishers/thumbnail/thumbnail_interceptor.cpp b/src/projects/publishers/thumbnail/thumbnail_interceptor.cpp index a156d6e84..425f026a2 100644 --- a/src/projects/publishers/thumbnail/thumbnail_interceptor.cpp +++ b/src/projects/publishers/thumbnail/thumbnail_interceptor.cpp @@ -2,7 +2,7 @@ // // OvenMediaEngine // -// Created by Keukhan +// Created by Kwon Keuk Han // Copyright (c) 2019 AirenSoft. All rights reserved. // //============================================================================== @@ -25,9 +25,11 @@ bool ThumbnailInterceptor::IsInterceptorForRequest(const std::shared_ptrGetRequestTarget().LowerCaseString().IndexOf(".jpg") >= 0) || - (request->GetRequestTarget().LowerCaseString().IndexOf(".png") >= 0)) + const auto target = request->GetRequestTarget().LowerCaseString(); + + if ((target.IndexOf(".jpg") >= 0) || + (target.IndexOf(".png") >= 0) || + (target.IndexOf(".webp") >= 0)) { return true; } diff --git a/src/projects/publishers/thumbnail/thumbnail_publisher.cpp b/src/projects/publishers/thumbnail/thumbnail_publisher.cpp index 5ddbf8ddd..c0f8a963e 100755 --- a/src/projects/publishers/thumbnail/thumbnail_publisher.cpp +++ b/src/projects/publishers/thumbnail/thumbnail_publisher.cpp @@ -206,7 +206,7 @@ bool ThumbnailPublisher::OnDeletePublisherApplication(const std::shared_ptr ThumbnailPublisher::CreateInterceptor() { - ov::String thumbnail_url_pattern = R"(.+thumb\.(jpg|png)$)"; + ov::String thumbnail_url_pattern = R"(.+thumb\.(jpg|png|webp)$)"; auto http_interceptor = std::make_shared(); @@ -355,6 +355,10 @@ std::shared_ptr ThumbnailPublisher::CreateInterceptor() { media_codec_id = cmn::MediaCodecId::Png; } + else if (request_url->File().LowerCaseString().IndexOf(".webp") >= 0) + { + media_codec_id = cmn::MediaCodecId::Webp; + } else { response->AppendString(ov::String::FormatString("Unsupported file extension")); @@ -377,7 +381,7 @@ std::shared_ptr ThumbnailPublisher::CreateInterceptor() return http::svr::NextHandler::DoNotCall; } - response->SetHeader("Content-Type", (media_codec_id == cmn::MediaCodecId::Jpeg) ? "image/jpeg" : "image/png"); + response->SetHeader("Content-Type", MimeTypeFromMediaCodecId(media_codec_id)); response->SetStatusCode(http::StatusCode::OK); response->AppendData(std::move(endcoded_video_frame->Clone())); auto sent_size = response->Response(); @@ -402,5 +406,19 @@ std::shared_ptr ThumbnailPublisher::CreateInterceptor() }); return http_interceptor; -} +} +ov::String ThumbnailPublisher::MimeTypeFromMediaCodecId(const cmn::MediaCodecId &type) +{ + switch (type) + { + case cmn::MediaCodecId::Jpeg: + return "image/jpeg"; + case cmn::MediaCodecId::Png: + return "image/png"; + case cmn::MediaCodecId::Webp: + return "image/webp"; + default: + return "application/octet-stream"; + } +} diff --git a/src/projects/publishers/thumbnail/thumbnail_publisher.h b/src/projects/publishers/thumbnail/thumbnail_publisher.h index de42f4371..8b78b8af7 100755 --- a/src/projects/publishers/thumbnail/thumbnail_publisher.h +++ b/src/projects/publishers/thumbnail/thumbnail_publisher.h @@ -48,6 +48,8 @@ class ThumbnailPublisher : public pub::Publisher std::shared_ptr OnCreatePublisherApplication(const info::Application &application_info) override; bool OnDeletePublisherApplication(const std::shared_ptr &application) override; + static ov::String MimeTypeFromMediaCodecId(const cmn::MediaCodecId &type); + private: std::shared_ptr CreateInterceptor(); diff --git a/src/projects/publishers/thumbnail/thumbnail_stream.cpp b/src/projects/publishers/thumbnail/thumbnail_stream.cpp index 444646223..b39b51006 100755 --- a/src/projects/publishers/thumbnail/thumbnail_stream.cpp +++ b/src/projects/publishers/thumbnail/thumbnail_stream.cpp @@ -34,10 +34,13 @@ bool ThumbnailStream::Start() return false; } + // Check if there is a supported codec bool found = false; for (const auto &[id, track] : _tracks) { - if ((track->GetCodecId() == cmn::MediaCodecId::Png || track->GetCodecId() == cmn::MediaCodecId::Jpeg)) + if ((track->GetCodecId() == cmn::MediaCodecId::Png || + track->GetCodecId() == cmn::MediaCodecId::Jpeg || + track->GetCodecId() == cmn::MediaCodecId::Webp)) { found = true; break; @@ -75,7 +78,9 @@ void ThumbnailStream::SendVideoFrame(const std::shared_ptr &media_p return; } - if (!(track->GetCodecId() == cmn::MediaCodecId::Png || track->GetCodecId() == cmn::MediaCodecId::Jpeg)) + if (!(track->GetCodecId() == cmn::MediaCodecId::Png || + track->GetCodecId() == cmn::MediaCodecId::Jpeg || + track->GetCodecId() == cmn::MediaCodecId::Webp)) { // Could not support codec for image return; diff --git a/src/projects/transcoder/codec/encoder/encoder_jpeg.cpp b/src/projects/transcoder/codec/encoder/encoder_jpeg.cpp index 64ac4224c..f9002b5f0 100755 --- a/src/projects/transcoder/codec/encoder/encoder_jpeg.cpp +++ b/src/projects/transcoder/codec/encoder/encoder_jpeg.cpp @@ -29,7 +29,7 @@ bool EncoderJPEG::SetCodecParams() _codec_context->color_range = AVCOL_RANGE_JPEG; _codec_context->strict_std_compliance = FF_COMPLIANCE_STRICT; - _bitstream_format = cmn::BitstreamFormat::JPEG; + _bitstream_format = GetBitstreamFormat(); _packet_type = cmn::PacketType::RAW; diff --git a/src/projects/transcoder/codec/encoder/encoder_png.cpp b/src/projects/transcoder/codec/encoder/encoder_png.cpp index 184a1b6e1..5da42a247 100755 --- a/src/projects/transcoder/codec/encoder/encoder_png.cpp +++ b/src/projects/transcoder/codec/encoder/encoder_png.cpp @@ -26,7 +26,7 @@ bool EncoderPNG::SetCodecParams() // Set the compression level _codec_context->compression_level = 1; - _bitstream_format = cmn::BitstreamFormat::PNG; + _bitstream_format = GetBitstreamFormat(); _packet_type = cmn::PacketType::RAW; diff --git a/src/projects/transcoder/codec/encoder/encoder_webp.cpp b/src/projects/transcoder/codec/encoder/encoder_webp.cpp new file mode 100755 index 000000000..85f5e9d5e --- /dev/null +++ b/src/projects/transcoder/codec/encoder/encoder_webp.cpp @@ -0,0 +1,119 @@ +//============================================================================== +// +// Transcode +// +// Created by Kwon Keuk Han +// Copyright (c) 2018 AirenSoft. All rights reserved. +// +//============================================================================== +#include "encoder_webp.h" + +#include + +#include + +#include "../../transcoder_private.h" + +bool EncoderWEBP::SetCodecParams() +{ + _codec_context->codec_type = AVMEDIA_TYPE_VIDEO; + _codec_context->framerate = ::av_d2q((GetRefTrack()->GetFrameRateByConfig() > 0) ? GetRefTrack()->GetFrameRateByConfig() : GetRefTrack()->GetEstimateFrameRate(), AV_TIME_BASE); + _codec_context->time_base = ffmpeg::Conv::TimebaseToAVRational(GetRefTrack()->GetTimeBase()); + _codec_context->pix_fmt = (AVPixelFormat)GetSupportedFormat(); + _codec_context->width = GetRefTrack()->GetWidth(); + _codec_context->height = GetRefTrack()->GetHeight(); + + // Set the compression level + _codec_context->compression_level = 1; + + // Set the preset + auto preset = GetRefTrack()->GetPreset(); + if (preset.IsEmpty()) + { + ::av_opt_set(_codec_context->priv_data, "preset", "default", 0); + } + else + { + if (preset == "none" || + preset == "default" || + preset == "picture" || + preset == "photo" || + preset == "drawing" || + preset == "icon" || + preset == "text") + { + ::av_opt_set(_codec_context->priv_data, "preset", preset.CStr(), 0); + } + } + + _bitstream_format = GetBitstreamFormat(); + + _packet_type = cmn::PacketType::RAW; + + return true; +} + +bool EncoderWEBP::InitCodec() +{ + auto codec_id = GetCodecID(); + + const AVCodec *codec = ::avcodec_find_encoder(codec_id); + + if (codec == nullptr) + { + logte("Could not find encoder: %d (%s)", codec_id, ::avcodec_get_name(codec_id)); + return false; + } + + _codec_context = ::avcodec_alloc_context3(codec); + if (_codec_context == nullptr) + { + logte("Could not allocate codec context for %s (%d)", ::avcodec_get_name(codec_id), codec_id); + return false; + } + + if (SetCodecParams() == false) + { + logte("Could not set codec parameters for %s (%d)", ::avcodec_get_name(codec_id), codec_id); + return false; + } + + if (::avcodec_open2(_codec_context, codec, nullptr) < 0) + { + logte("Could not open codec: %s (%d)", ::avcodec_get_name(codec_id), codec_id); + return false; + } + + return true; +} + +bool EncoderWEBP::Configure(std::shared_ptr context) +{ + if (TranscodeEncoder::Configure(context) == false) + { + return false; + } + + try + { + _kill_flag = false; + + _codec_thread = std::thread(&EncoderWEBP::CodecThread, this); + pthread_setname_np(_codec_thread.native_handle(), ov::String::FormatString("ENC-%s-t%d", avcodec_get_name(GetCodecID()), _track->GetId()).CStr()); + + // Initialize the codec and wait for completion. + if(_codec_init_event.Get() == false) + { + _kill_flag = true; + return false; + } + } + catch (const std::system_error &e) + { + _kill_flag = true; + return false; + } + + return true; +} + diff --git a/src/projects/transcoder/codec/encoder/encoder_webp.h b/src/projects/transcoder/codec/encoder/encoder_webp.h new file mode 100755 index 000000000..74a652d74 --- /dev/null +++ b/src/projects/transcoder/codec/encoder/encoder_webp.h @@ -0,0 +1,42 @@ +//============================================================================== +// +// Transcode +// +// Created by Kwon Keuk Han +// Copyright (c) 2018 AirenSoft. All rights reserved. +// +//============================================================================== +#pragma once + +#include "../../transcoder_encoder.h" + +class EncoderWEBP : public TranscodeEncoder +{ +public: + EncoderWEBP(const info::Stream &stream_info) + : TranscodeEncoder(stream_info) + { + } + + AVCodecID GetCodecID() const noexcept override + { + return AV_CODEC_ID_WEBP; + } + + int GetSupportedFormat() const noexcept override + { + return AV_PIX_FMT_YUV420P; + } + + cmn::BitstreamFormat GetBitstreamFormat() const noexcept override + { + return cmn::BitstreamFormat::WEBP; + } + + bool Configure(std::shared_ptr context) override; + + bool InitCodec() override; + +private: + bool SetCodecParams() override; +}; diff --git a/src/projects/transcoder/transcoder_encoder.cpp b/src/projects/transcoder/transcoder_encoder.cpp index 81be3f01f..1c01bfa22 100644 --- a/src/projects/transcoder/transcoder_encoder.cpp +++ b/src/projects/transcoder/transcoder_encoder.cpp @@ -26,6 +26,8 @@ #include "codec/encoder/encoder_opus.h" #include "codec/encoder/encoder_png.h" #include "codec/encoder/encoder_vp8.h" +#include "codec/encoder/encoder_webp.h" + #include "transcoder_gpu.h" #include "transcoder_private.h" @@ -263,6 +265,16 @@ std::shared_ptr TranscodeEncoder::Create( } break; } + else if (candidate->GetCodecId() == cmn::MediaCodecId::Webp) + { + switch (candidate->GetModuleId()) + { + default: + CASE_CREATE_CODEC_IFNEED(DEFAULT, EncoderWEBP); + break; + } + break; + } else { OV_ASSERT(false, "Not supported codec: %d", track->GetCodecId());