diff --git a/base/cvd/BUILD.nv_codec_headers.bazel b/base/cvd/BUILD.nv_codec_headers.bazel new file mode 100644 index 00000000000..045cab8b6b0 --- /dev/null +++ b/base/cvd/BUILD.nv_codec_headers.bazel @@ -0,0 +1,7 @@ +package(default_visibility = ["//visibility:public"]) + +cc_library( + name = "nv_headers", + hdrs = glob(["include/ffnvcodec/*.h"]), + includes = ["include/ffnvcodec"], +) diff --git a/base/cvd/MODULE.bazel b/base/cvd/MODULE.bazel index 4d602e0e636..cae63c3c3be 100644 --- a/base/cvd/MODULE.bazel +++ b/base/cvd/MODULE.bazel @@ -6,6 +6,26 @@ ############################################################################### bazel_dep(name = "abseil-cpp", version = "20260107.0") +bazel_dep(name = "hermetic_cc_toolchain", version = "4.0.1") + +# Hermetic C/C++ toolchain using zig cc for cross-compilation and GLIBC targeting. +# Use with: bazel build ... --extra_toolchains=@zig_sdk//toolchain:linux_amd64_gnu.2.36 +zig_toolchains = use_extension("@hermetic_cc_toolchain//toolchain:ext.bzl", "toolchains") +use_repo(zig_toolchains, "zig_sdk") + +# CUDA support for NVENC +bazel_dep(name = "rules_cuda", version = "0.2.5") +git_override( + module_name = "rules_cuda", + commit = "32414222a7997ee66035d51404fc5427df94c699", + remote = "https://github.com/bazel-contrib/rules_cuda.git", +) +cuda = use_extension("@rules_cuda//cuda:extensions.bzl", "toolchain") +cuda.local_toolchain( + name = "local_cuda", + toolkit_path = "", +) +use_repo(cuda, cuda = "local_cuda") bazel_dep(name = "aspect_bazel_lib", version = "2.19.3") bazel_dep(name = "aspect_rules_lint", version = "1.4.4", dev_dependency = True) bazel_dep(name = "bazel_skylib", version = "1.8.2", dev_dependency = True) @@ -65,3 +85,15 @@ include("//toolchain:bazel.MODULE.bazel") include("//toolchain:llvm.MODULE.bazel") include("//toolchain:python.MODULE.bazel") include("//toolchain:rust.MODULE.bazel") + +# NVENC video encoding headers (nv-codec-headers SDK 12.1) +http_archive = use_repo_rule("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") +http_archive( + name = "nv_codec_headers", + build_file = "@//:BUILD.nv_codec_headers.bazel", + strip_prefix = "nv-codec-headers-n12.1.14.0", + urls = [ + "https://github.com/FFmpeg/nv-codec-headers/archive/refs/tags/n12.1.14.0.tar.gz", + ], + sha256 = "2fefaa227d2a3b4170797796425a59d1dd2ed5fd231db9b4244468ba327acd0b", +) diff --git a/base/cvd/cuttlefish/host/frontend/webrtc/BUILD.bazel b/base/cvd/cuttlefish/host/frontend/webrtc/BUILD.bazel index c039d96d530..b1a4a5efd43 100644 --- a/base/cvd/cuttlefish/host/frontend/webrtc/BUILD.bazel +++ b/base/cvd/cuttlefish/host/frontend/webrtc/BUILD.bazel @@ -44,6 +44,8 @@ cf_cc_binary( "client_server.h", "connection_observer.cpp", "connection_observer.h", + "cvd_abgr_video_frame_buffer.cpp", + "cvd_abgr_video_frame_buffer.h", "cvd_video_frame_buffer.cpp", "cvd_video_frame_buffer.h", "display_handler.cpp", diff --git a/base/cvd/cuttlefish/host/frontend/webrtc/cvd_abgr_video_frame_buffer.cpp b/base/cvd/cuttlefish/host/frontend/webrtc/cvd_abgr_video_frame_buffer.cpp new file mode 100644 index 00000000000..713f552fddd --- /dev/null +++ b/base/cvd/cuttlefish/host/frontend/webrtc/cvd_abgr_video_frame_buffer.cpp @@ -0,0 +1,39 @@ +/* + * Copyright (C) 2025 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "cuttlefish/host/frontend/webrtc/cvd_abgr_video_frame_buffer.h" + +#include + +namespace cuttlefish { + +CvdAbgrVideoFrameBuffer::CvdAbgrVideoFrameBuffer(int width, int height, + uint32_t format, int stride, + const uint8_t* data) + : width_(width), height_(height), format_(format), stride_(stride) { + // Allocate buffer: height * stride + size_t size = height * stride; + data_.resize(size); + if (data) { + std::memcpy(data_.data(), data, size); + } +} + +std::unique_ptr CvdAbgrVideoFrameBuffer::Clone() const { + return std::make_unique(*this); +} + +} // namespace cuttlefish diff --git a/base/cvd/cuttlefish/host/frontend/webrtc/cvd_abgr_video_frame_buffer.h b/base/cvd/cuttlefish/host/frontend/webrtc/cvd_abgr_video_frame_buffer.h new file mode 100644 index 00000000000..2d90e78d88e --- /dev/null +++ b/base/cvd/cuttlefish/host/frontend/webrtc/cvd_abgr_video_frame_buffer.h @@ -0,0 +1,50 @@ +/* + * Copyright (C) 2025 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include + +#include "cuttlefish/host/libs/screen_connector/video_frame_buffer.h" + +namespace cuttlefish { + +class CvdAbgrVideoFrameBuffer : public VideoFrameBuffer { + public: + CvdAbgrVideoFrameBuffer(int width, int height, uint32_t format, int stride, + const uint8_t* data); + ~CvdAbgrVideoFrameBuffer() override = default; + + int width() const override { return width_; } + int height() const override { return height_; } + + uint8_t* Data() const override { return const_cast(data_.data()); } + int Stride() const override { return stride_; } + std::size_t DataSize() const override { return data_.size(); } + uint32_t PixelFormat() const override { return format_; } + + std::unique_ptr Clone() const override; + + private: + int width_; + int height_; + uint32_t format_; + int stride_; + std::vector data_; +}; + +} // namespace cuttlefish diff --git a/base/cvd/cuttlefish/host/frontend/webrtc/cvd_video_frame_buffer.cpp b/base/cvd/cuttlefish/host/frontend/webrtc/cvd_video_frame_buffer.cpp index fd591e2641c..09c4e0b97ee 100644 --- a/base/cvd/cuttlefish/host/frontend/webrtc/cvd_video_frame_buffer.cpp +++ b/base/cvd/cuttlefish/host/frontend/webrtc/cvd_video_frame_buffer.cpp @@ -79,4 +79,8 @@ int CvdVideoFrameBuffer::StrideV() const { return AlignStride((width_ + 1) / 2); } +std::unique_ptr CvdVideoFrameBuffer::Clone() const { + return std::make_unique(*this); +} + } // namespace cuttlefish diff --git a/base/cvd/cuttlefish/host/frontend/webrtc/cvd_video_frame_buffer.h b/base/cvd/cuttlefish/host/frontend/webrtc/cvd_video_frame_buffer.h index 285d22bffa9..c34c498d61e 100644 --- a/base/cvd/cuttlefish/host/frontend/webrtc/cvd_video_frame_buffer.h +++ b/base/cvd/cuttlefish/host/frontend/webrtc/cvd_video_frame_buffer.h @@ -52,6 +52,8 @@ class CvdVideoFrameBuffer : public VideoFrameBuffer { size_t DataSizeU() const override { return u_.size(); } size_t DataSizeV() const override { return v_.size(); } + std::unique_ptr Clone() const override; + private: const int width_; const int height_; diff --git a/base/cvd/cuttlefish/host/frontend/webrtc/display_handler.cpp b/base/cvd/cuttlefish/host/frontend/webrtc/display_handler.cpp index 91d8175bfb2..3156ae34c28 100644 --- a/base/cvd/cuttlefish/host/frontend/webrtc/display_handler.cpp +++ b/base/cvd/cuttlefish/host/frontend/webrtc/display_handler.cpp @@ -27,6 +27,7 @@ #include #include "absl/log/log.h" +#include "cuttlefish/host/frontend/webrtc/cvd_abgr_video_frame_buffer.h" #include "cuttlefish/host/frontend/webrtc/libdevice/streamer.h" #include "cuttlefish/host/libs/screen_connector/composition_manager.h" #include "cuttlefish/host/libs/screen_connector/video_frame_buffer.h" @@ -104,28 +105,18 @@ DisplayHandler::GetScreenConnectorCallback() { uint32_t frame_fourcc_format, uint32_t frame_stride_bytes, uint8_t* frame_pixels, WebRtcScProcessedFrame& processed_frame) { processed_frame.display_number_ = display_number; - processed_frame.buf_ = - std::make_unique(frame_width, frame_height); if (composition_manager.has_value()) { composition_manager.value()->OnFrame( display_number, frame_width, frame_height, frame_fourcc_format, frame_stride_bytes, frame_pixels); } if (frame_fourcc_format == DRM_FORMAT_ARGB8888 || - frame_fourcc_format == DRM_FORMAT_XRGB8888) { - libyuv::ARGBToI420( - frame_pixels, frame_stride_bytes, processed_frame.buf_->DataY(), - processed_frame.buf_->StrideY(), processed_frame.buf_->DataU(), - processed_frame.buf_->StrideU(), processed_frame.buf_->DataV(), - processed_frame.buf_->StrideV(), frame_width, frame_height); - processed_frame.is_success_ = true; - } else if (frame_fourcc_format == DRM_FORMAT_ABGR8888 || - frame_fourcc_format == DRM_FORMAT_XBGR8888) { - libyuv::ABGRToI420( - frame_pixels, frame_stride_bytes, processed_frame.buf_->DataY(), - processed_frame.buf_->StrideY(), processed_frame.buf_->DataU(), - processed_frame.buf_->StrideU(), processed_frame.buf_->DataV(), - processed_frame.buf_->StrideV(), frame_width, frame_height); + frame_fourcc_format == DRM_FORMAT_XRGB8888 || + frame_fourcc_format == DRM_FORMAT_ABGR8888 || + frame_fourcc_format == DRM_FORMAT_XBGR8888) { + processed_frame.buf_ = std::make_unique( + frame_width, frame_height, frame_fourcc_format, frame_stride_bytes, + frame_pixels); processed_frame.is_success_ = true; } else { processed_frame.is_success_ = false; @@ -138,7 +129,7 @@ DisplayHandler::GetScreenConnectorCallback() { for (;;) { auto processed_frame = screen_connector_.OnNextFrame(); - std::shared_ptr buffer = + std::shared_ptr buffer = std::move(processed_frame.buf_); const uint32_t display_number = processed_frame.display_number_; @@ -147,7 +138,7 @@ DisplayHandler::GetScreenConnectorCallback() { display_last_buffers_[display_number] = std::make_shared(BufferInfo{ .last_sent_time_stamp = std::chrono::system_clock::now(), - .buffer = std::static_pointer_cast(buffer), + .buffer = buffer, }); } if (processed_frame.is_success_) { @@ -255,8 +246,7 @@ void DisplayHandler::RepeatFramesPeriodically() { buffer_info->last_sent_time_stamp + kRepeatingInterval) { if (composition_manager_.has_value()) { composition_manager_.value()->ComposeFrame( - display_number, std::static_pointer_cast( - buffer_info->buffer)); + display_number, buffer_info->buffer); } buffers[display_number] = buffer_info; } diff --git a/base/cvd/cuttlefish/host/frontend/webrtc/display_handler.h b/base/cvd/cuttlefish/host/frontend/webrtc/display_handler.h index b762b2dea10..c8f7235d97d 100644 --- a/base/cvd/cuttlefish/host/frontend/webrtc/display_handler.h +++ b/base/cvd/cuttlefish/host/frontend/webrtc/display_handler.h @@ -42,12 +42,13 @@ class CompositionManager; */ struct WebRtcScProcessedFrame : public ScreenConnectorFrameInfo { // must support move semantic - std::unique_ptr buf_; + std::unique_ptr buf_; std::unique_ptr Clone() { // copy internal buffer, not move - CvdVideoFrameBuffer* new_buffer = new CvdVideoFrameBuffer(*(buf_.get())); auto cloned_frame = std::make_unique(); - cloned_frame->buf_ = std::unique_ptr(new_buffer); + if (buf_) { + cloned_frame->buf_ = buf_->Clone(); + } return cloned_frame; } }; diff --git a/base/cvd/cuttlefish/host/frontend/webrtc/libcommon/BUILD.bazel b/base/cvd/cuttlefish/host/frontend/webrtc/libcommon/BUILD.bazel index 80bdb03358b..3ad3cdcd6b5 100644 --- a/base/cvd/cuttlefish/host/frontend/webrtc/libcommon/BUILD.bazel +++ b/base/cvd/cuttlefish/host/frontend/webrtc/libcommon/BUILD.bazel @@ -4,6 +4,217 @@ package( default_visibility = ["//:android_cuttlefish"], ) +# ============================================================================= +# Plugin Framework (Upstream - Codec Agnostic) +# ============================================================================= + +cf_cc_library( + name = "encoder_provider", + hdrs = ["encoder_provider.h"], + deps = ["@libwebrtc"], + visibility = ["//visibility:public"], +) + +cf_cc_library( + name = "encoder_provider_registry", + srcs = ["encoder_provider_registry.cpp"], + hdrs = ["encoder_provider_registry.h"], + deps = [ + ":encoder_provider", + "@libwebrtc", + ], + visibility = ["//visibility:public"], +) + +cf_cc_library( + name = "composite_encoder_factory", + srcs = ["composite_encoder_factory.cpp"], + hdrs = ["composite_encoder_factory.h"], + deps = [ + ":encoder_provider_registry", + "@libwebrtc", + ], + visibility = ["//visibility:public"], +) + +cf_cc_library( + name = "builtin_encoder_provider", + srcs = ["builtin_encoder_provider.cpp"], + deps = [ + ":encoder_provider", + ":encoder_provider_registry", + "@libwebrtc", + ], + alwayslink = True, # Static initializer must be linked +) + +cf_cc_library( + name = "decoder_provider", + hdrs = ["decoder_provider.h"], + deps = ["@libwebrtc"], + visibility = ["//visibility:public"], +) + +cf_cc_library( + name = "decoder_provider_registry", + srcs = ["decoder_provider_registry.cpp"], + hdrs = ["decoder_provider_registry.h"], + deps = [ + ":decoder_provider", + "@libwebrtc", + ], + visibility = ["//visibility:public"], +) + +cf_cc_library( + name = "composite_decoder_factory", + srcs = ["composite_decoder_factory.cpp"], + hdrs = ["composite_decoder_factory.h"], + deps = [ + ":decoder_provider_registry", + "@libwebrtc", + ], + visibility = ["//visibility:public"], +) + +cf_cc_library( + name = "builtin_decoder_provider", + srcs = ["builtin_decoder_provider.cpp"], + deps = [ + ":decoder_provider", + ":decoder_provider_registry", + "@libwebrtc", + ], + alwayslink = True, # Static initializer must be linked +) + +# ============================================================================= +# NVENC/CUDA Support +# ============================================================================= + +cf_cc_library( + name = "cuda_context", + srcs = ["cuda_context.cpp"], + hdrs = ["cuda_context.h"], + deps = [ + "@cuda//:cuda_headers", + "@libwebrtc", + ], + linkopts = [ + "-lcuda", + ], +) + +cf_cc_library( + name = "nvenc_encoder_config", + hdrs = ["nvenc_encoder_config.h"], + deps = [ + "@libwebrtc", + "@nv_codec_headers//:nv_headers", + ], +) + +cf_cc_library( + name = "nvenc_capabilities", + srcs = ["nvenc_capabilities.cpp"], + hdrs = ["nvenc_capabilities.h"], + deps = [ + ":cuda_context", + "@libwebrtc", + "@nv_codec_headers//:nv_headers", + ], + linkopts = [ + "-lnvidia-encode", + ], +) + +cf_cc_library( + name = "stub_decoder", + hdrs = ["stub_decoder.h"], + deps = [ + "@libwebrtc", + ], +) + +cf_cc_library( + name = "nvenc_video_encoder", + srcs = ["nvenc_video_encoder.cpp"], + hdrs = ["nvenc_video_encoder.h"], + deps = [ + ":abgr_buffer", + ":cuda_context", + ":nvenc_encoder_config", + "@libdrm//:libdrm_fourcc", + "@libwebrtc", + "@nv_codec_headers//:nv_headers", + "@cuda//:cuda_headers", + "@cuda//:cuda_runtime", + "@libyuv", + ], + linkopts = [ + "-lnvidia-encode", + "-lcuda", + "-lcudart", + ], +) + +# ============================================================================= +# Per-Codec NVENC Providers +# ============================================================================= + +cf_cc_library( + name = "av1_nvenc_encoder_provider", + srcs = ["av1_nvenc_encoder_provider.cpp"], + deps = [ + ":encoder_provider", + ":encoder_provider_registry", + ":nvenc_capabilities", + ":nvenc_encoder_config", + ":nvenc_video_encoder", + "@abseil-cpp//absl/strings", + "@libwebrtc", + "@nv_codec_headers//:nv_headers", + ], + alwayslink = True, +) + +cf_cc_library( + name = "av1_nvenc_decoder_provider", + srcs = ["av1_nvenc_decoder_provider.cpp"], + deps = [ + ":decoder_provider", + ":decoder_provider_registry", + ":nvenc_capabilities", + ":stub_decoder", + "@abseil-cpp//absl/strings", + "@libwebrtc", + "@nv_codec_headers//:nv_headers", + ], + alwayslink = True, +) + +# Injection point for future private repo providers. +# Currently a no-op cc_library. When the public/private repo split +# happens, replace with a label_flag that points to private repo +# providers. +cc_library(name = "extra_codec_providers") + +# ============================================================================= +# Common Components +# ============================================================================= + +cf_cc_library( + name = "abgr_buffer", + srcs = ["abgr_buffer.cpp"], + hdrs = ["abgr_buffer.h"], + deps = [ + "//cuttlefish/host/libs/screen_connector:video_frame_buffer", + "@libdrm//:libdrm_fourcc", + "@libwebrtc", + "@libyuv", + ], +) + cf_cc_library( name = "audio_device", srcs = ["audio_device.cpp"], @@ -43,8 +254,17 @@ cf_cc_library( srcs = ["peer_connection_utils.cpp"], hdrs = ["peer_connection_utils.h"], deps = [ - "//cuttlefish/host/frontend/webrtc/libcommon:audio_device", - "//cuttlefish/host/frontend/webrtc/libcommon:vp8only_encoder_factory", + ":audio_device", + ":composite_encoder_factory", + ":composite_decoder_factory", + ":encoder_provider_registry", + ":decoder_provider_registry", + ":builtin_encoder_provider", + ":builtin_decoder_provider", + # Per-codec NVENC providers + ":av1_nvenc_encoder_provider", + ":av1_nvenc_decoder_provider", + ":extra_codec_providers", "//cuttlefish/result", "@libwebrtc", ], diff --git a/base/cvd/cuttlefish/host/frontend/webrtc/libcommon/abgr_buffer.cpp b/base/cvd/cuttlefish/host/frontend/webrtc/libcommon/abgr_buffer.cpp new file mode 100644 index 00000000000..14f6748bdb6 --- /dev/null +++ b/base/cvd/cuttlefish/host/frontend/webrtc/libcommon/abgr_buffer.cpp @@ -0,0 +1,73 @@ +/* + * Copyright (C) 2026 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "cuttlefish/host/frontend/webrtc/libcommon/abgr_buffer.h" + +#include +#include +#include + +namespace cuttlefish { + +AbgrBuffer::AbgrBuffer(std::shared_ptr buffer) + : buffer_(std::move(buffer)) {} + +int AbgrBuffer::width() const { return buffer_->width(); } +int AbgrBuffer::height() const { return buffer_->height(); } + +const uint8_t* AbgrBuffer::Data() const { return buffer_->Data(); } +int AbgrBuffer::Stride() const { return buffer_->Stride(); } + +rtc::scoped_refptr AbgrBuffer::ToI420() { + rtc::scoped_refptr i420_buffer = + webrtc::I420Buffer::Create(width(), height()); + + uint32_t format = buffer_->PixelFormat(); + + // Default to ABGR if format is unknown or matches ABGR + // Note: CvdAbgrVideoFrameBuffer sets PixelFormat. + // Assuming input is ABGR if checking DRM_FORMAT_ABGR8888. + + // Note: libyuv::ABGRToI420 expects ABGR input. + // If format is ARGB, we should use ARGBToI420. + + int res = -1; + if (format == DRM_FORMAT_ARGB8888 || format == DRM_FORMAT_XRGB8888) { + res = libyuv::ARGBToI420( + buffer_->Data(), buffer_->Stride(), + i420_buffer->MutableDataY(), i420_buffer->StrideY(), + i420_buffer->MutableDataU(), i420_buffer->StrideU(), + i420_buffer->MutableDataV(), i420_buffer->StrideV(), + width(), height()); + } else { + // Default to ABGR (most common for Cuttlefish/Wayland) + res = libyuv::ABGRToI420( + buffer_->Data(), buffer_->Stride(), + i420_buffer->MutableDataY(), i420_buffer->StrideY(), + i420_buffer->MutableDataU(), i420_buffer->StrideU(), + i420_buffer->MutableDataV(), i420_buffer->StrideV(), + width(), height()); + } + + if (res != 0) { + // Fallback or error? + return i420_buffer; + } + + return i420_buffer; +} + +} // namespace cuttlefish diff --git a/base/cvd/cuttlefish/host/frontend/webrtc/libcommon/abgr_buffer.h b/base/cvd/cuttlefish/host/frontend/webrtc/libcommon/abgr_buffer.h new file mode 100644 index 00000000000..26014334146 --- /dev/null +++ b/base/cvd/cuttlefish/host/frontend/webrtc/libcommon/abgr_buffer.h @@ -0,0 +1,44 @@ +/* + * Copyright (C) 2026 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include + +#include "cuttlefish/host/libs/screen_connector/video_frame_buffer.h" + +namespace cuttlefish { + +class AbgrBuffer : public webrtc::VideoFrameBuffer { + public: + explicit AbgrBuffer(std::shared_ptr buffer); + + Type type() const override { return Type::kNative; } + int width() const override; + int height() const override; + + rtc::scoped_refptr ToI420() override; + + const uint8_t* Data() const; + int Stride() const; + uint32_t PixelFormat() const { return buffer_->PixelFormat(); } + + private: + std::shared_ptr buffer_; +}; + +} // namespace cuttlefish diff --git a/base/cvd/cuttlefish/host/frontend/webrtc/libcommon/av1_nvenc_decoder_provider.cpp b/base/cvd/cuttlefish/host/frontend/webrtc/libcommon/av1_nvenc_decoder_provider.cpp new file mode 100644 index 00000000000..ac716ddf5e2 --- /dev/null +++ b/base/cvd/cuttlefish/host/frontend/webrtc/libcommon/av1_nvenc_decoder_provider.cpp @@ -0,0 +1,69 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include +#include + +#include + +#include "absl/strings/match.h" +#include "cuttlefish/host/frontend/webrtc/libcommon/decoder_provider.h" +#include "cuttlefish/host/frontend/webrtc/libcommon/decoder_provider_registry.h" +#include "cuttlefish/host/frontend/webrtc/libcommon/nvenc_capabilities.h" +#include "cuttlefish/host/frontend/webrtc/libcommon/stub_decoder.h" + +namespace cuttlefish { +namespace webrtc_streaming { +namespace { + +// AV1 NVENC decoder stub provider. +// +// Advertises AV1 decoding support so WebRTC's SDP negotiation succeeds. +// Cuttlefish never decodes video (the browser does), so this returns a +// StubDecoder. Availability is gated on the GPU supporting AV1 NVENC +// encoding — there's no point offering AV1 in SDP if we can't encode it. +class Av1NvencDecoderProvider : public DecoderProvider { + public: + Av1NvencDecoderProvider() { + available_ = IsNvencCodecSupported(NV_ENC_CODEC_AV1_GUID); + } + + std::string GetName() const override { return "nvenc_av1_stub"; } + int GetPriority() const override { return 101; } + bool IsAvailable() const override { return available_; } + + std::vector GetSupportedFormats() + const override { + if (!available_) return {}; + return {{"AV1", {{"profile", "0"}}}}; + } + + std::unique_ptr CreateDecoder( + const webrtc::SdpVideoFormat& format) const override { + if (!available_) return nullptr; + if (!absl::EqualsIgnoreCase(format.name, "AV1")) return nullptr; + return std::make_unique(); + } + + private: + bool available_ = false; +}; + +} // namespace + +REGISTER_DECODER_PROVIDER(Av1NvencDecoderProvider); + +} // namespace webrtc_streaming +} // namespace cuttlefish diff --git a/base/cvd/cuttlefish/host/frontend/webrtc/libcommon/av1_nvenc_encoder_provider.cpp b/base/cvd/cuttlefish/host/frontend/webrtc/libcommon/av1_nvenc_encoder_provider.cpp new file mode 100644 index 00000000000..a9eb941820a --- /dev/null +++ b/base/cvd/cuttlefish/host/frontend/webrtc/libcommon/av1_nvenc_encoder_provider.cpp @@ -0,0 +1,122 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include +#include + +#include + +#include "absl/strings/match.h" +#include "cuttlefish/host/frontend/webrtc/libcommon/encoder_provider.h" +#include "cuttlefish/host/frontend/webrtc/libcommon/encoder_provider_registry.h" +#include "cuttlefish/host/frontend/webrtc/libcommon/nvenc_capabilities.h" +#include "cuttlefish/host/frontend/webrtc/libcommon/nvenc_encoder_config.h" +#include "cuttlefish/host/frontend/webrtc/libcommon/nvenc_video_encoder.h" +#include "rtc_base/logging.h" + +namespace cuttlefish { +namespace webrtc_streaming { +namespace { + +// Codec-specific NVENC configuration for AV1. + +void Av1ApplyDefaults(NV_ENC_CONFIG* config) { + auto& av1 = config->encodeCodecConfig.av1Config; + av1.outputAnnexBFormat = 0; // OBU format required for WebRTC. + av1.disableSeqHdr = 0; + av1.repeatSeqHdr = 1; // Keyframes must be self-contained. + av1.chromaFormatIDC = 1; // YUV 4:2:0. + av1.inputPixelBitDepthMinus8 = 0; // 8-bit input. + av1.pixelBitDepthMinus8 = 0; // 8-bit output. + av1.numTileColumns = 1; + av1.numTileRows = 1; + av1.tier = NV_ENC_TIER_AV1_0; + av1.level = NV_ENC_LEVEL_AV1_AUTOSELECT; + av1.idrPeriod = config->gopLength; +} + +void Av1SetPicParams(NV_ENC_PIC_PARAMS* params) { + // AV1 requires tile configuration on every frame. + params->codecPicParams.av1PicParams.numTileColumns = 1; + params->codecPicParams.av1PicParams.numTileRows = 1; +} + +// AV1 achieves similar quality at ~60-70% of H.264 bitrate. +// These are initial estimates and should be tuned with real +// encoding measurements on target hardware (L4/L40). +const webrtc::VideoEncoder::ResolutionBitrateLimits + kAv1BitrateLimits[] = { + {320 * 240, 80000, 40000, 800000}, + {640 * 480, 200000, 80000, 2000000}, + {1280 * 720, 500000, 200000, 4000000}, + {1600 * 900, 900000, 350000, 7000000}, + {1920 * 1080, 1200000, 500000, 10000000}, + {2560 * 1440, 2500000, 1000000, 18000000}, + {3840 * 2160, 5000000, 2000000, 30000000}, +}; + +const NvencEncoderConfig kAv1Config = { + .codec_guid = NV_ENC_CODEC_AV1_GUID, + .profile_guid = NV_ENC_AV1_PROFILE_MAIN_GUID, + .webrtc_codec_type = webrtc::kVideoCodecAV1, + .implementation_name = "NvencAv1", + .bitrate_limits = kAv1BitrateLimits, + .bitrate_limits_count = std::size(kAv1BitrateLimits), + .min_bitrate_bps = 50000, // 50 kbps technical minimum + .max_bitrate_bps = 12000000, // 12 Mbps + .apply_defaults = Av1ApplyDefaults, + .set_pic_params = Av1SetPicParams, +}; + +// AV1 NVENC encoder provider (public repo). +// +// Priority 101 ensures AV1 is preferred over other codec providers in +// SDP offers when multiple are available. AV1 offers better compression +// efficiency at comparable quality. +class Av1NvencEncoderProvider : public EncoderProvider { + public: + Av1NvencEncoderProvider() { + available_ = IsNvencCodecSupported(NV_ENC_CODEC_AV1_GUID); + RTC_LOG(LS_INFO) << "Av1NvencEncoderProvider: " + << (available_ ? "available" : "not available"); + } + + std::string GetName() const override { return "nvenc_av1"; } + int GetPriority() const override { return 101; } + bool IsAvailable() const override { return available_; } + + std::vector GetSupportedFormats() + const override { + if (!available_) return {}; + return {{"AV1", {{"profile", "0"}}}}; + } + + std::unique_ptr CreateEncoder( + const webrtc::SdpVideoFormat& format) const override { + if (!available_) return nullptr; + if (!absl::EqualsIgnoreCase(format.name, "AV1")) return nullptr; + return std::make_unique(kAv1Config, format); + } + + private: + bool available_ = false; +}; + +} // namespace + +REGISTER_ENCODER_PROVIDER(Av1NvencEncoderProvider); + +} // namespace webrtc_streaming +} // namespace cuttlefish diff --git a/base/cvd/cuttlefish/host/frontend/webrtc/libcommon/builtin_decoder_provider.cpp b/base/cvd/cuttlefish/host/frontend/webrtc/libcommon/builtin_decoder_provider.cpp new file mode 100644 index 00000000000..fe7542168c5 --- /dev/null +++ b/base/cvd/cuttlefish/host/frontend/webrtc/libcommon/builtin_decoder_provider.cpp @@ -0,0 +1,69 @@ +/* + * Copyright (C) 2026 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include + +#include "api/video_codecs/builtin_video_decoder_factory.h" +#include "api/video_codecs/sdp_video_format.h" +#include "api/video_codecs/video_decoder.h" +#include "api/video_codecs/video_decoder_factory.h" +#include "cuttlefish/host/frontend/webrtc/libcommon/decoder_provider.h" +#include "cuttlefish/host/frontend/webrtc/libcommon/decoder_provider_registry.h" + +namespace cuttlefish { +namespace webrtc_streaming { + +namespace { + +// Decoder provider that wraps WebRTC's builtin decoder factory. +// +// Provides software codecs (VP8, VP9, AV1) as a fallback when hardware +// or stub decoders are not available. Priority 0 ensures this is used only +// when no higher-priority provider can handle the format. +class BuiltinDecoderProvider : public DecoderProvider { + public: + BuiltinDecoderProvider() + : factory_(webrtc::CreateBuiltinVideoDecoderFactory()) {} + + std::string GetName() const override { return "builtin"; } + + int GetPriority() const override { return 0; } + + bool IsAvailable() const override { return true; } + + std::vector GetSupportedFormats() const override { + return factory_->GetSupportedFormats(); + } + + std::unique_ptr CreateDecoder( + const webrtc::SdpVideoFormat& format) const override { + return factory_->CreateVideoDecoder(format); + } + + private: + // Cached factory instance - created once at construction time. + std::unique_ptr factory_; +}; + +} // namespace + +// Static registration - provider is registered during static initialization. +REGISTER_DECODER_PROVIDER(BuiltinDecoderProvider); + +} // namespace webrtc_streaming +} // namespace cuttlefish diff --git a/base/cvd/cuttlefish/host/frontend/webrtc/libcommon/builtin_encoder_provider.cpp b/base/cvd/cuttlefish/host/frontend/webrtc/libcommon/builtin_encoder_provider.cpp new file mode 100644 index 00000000000..14be817698e --- /dev/null +++ b/base/cvd/cuttlefish/host/frontend/webrtc/libcommon/builtin_encoder_provider.cpp @@ -0,0 +1,70 @@ +/* + * Copyright (C) 2026 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include + +#include "api/video_codecs/builtin_video_encoder_factory.h" +#include "api/video_codecs/sdp_video_format.h" +#include "api/video_codecs/video_encoder.h" +#include "api/video_codecs/video_encoder_factory.h" +#include "cuttlefish/host/frontend/webrtc/libcommon/encoder_provider.h" +#include "cuttlefish/host/frontend/webrtc/libcommon/encoder_provider_registry.h" + +namespace cuttlefish { +namespace webrtc_streaming { + +namespace { + +// Encoder provider that wraps WebRTC's builtin encoder factory. +// +// Provides software codecs (VP8, VP9, AV1) as a fallback when hardware +// encoders are not available. Priority 0 ensures this is used only when +// no higher-priority provider can handle the format. +class BuiltinEncoderProvider : public EncoderProvider { + public: + BuiltinEncoderProvider() + : factory_(webrtc::CreateBuiltinVideoEncoderFactory()) {} + + std::string GetName() const override { return "builtin"; } + + int GetPriority() const override { return 0; } + + bool IsAvailable() const override { return true; } + + std::vector GetSupportedFormats() const override { + return factory_->GetSupportedFormats(); + } + + std::unique_ptr CreateEncoder( + const webrtc::SdpVideoFormat& format) const override { + return factory_->CreateVideoEncoder(format); + } + + private: + // Cached factory instance - created once at construction time. + // This avoids repeated factory creation overhead. + std::unique_ptr factory_; +}; + +} // namespace + +// Static registration - provider is registered during static initialization. +REGISTER_ENCODER_PROVIDER(BuiltinEncoderProvider); + +} // namespace webrtc_streaming +} // namespace cuttlefish diff --git a/base/cvd/cuttlefish/host/frontend/webrtc/libcommon/composite_decoder_factory.cpp b/base/cvd/cuttlefish/host/frontend/webrtc/libcommon/composite_decoder_factory.cpp new file mode 100644 index 00000000000..d1f4451a8e1 --- /dev/null +++ b/base/cvd/cuttlefish/host/frontend/webrtc/libcommon/composite_decoder_factory.cpp @@ -0,0 +1,95 @@ +/* + * Copyright (C) 2026 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "cuttlefish/host/frontend/webrtc/libcommon/composite_decoder_factory.h" + +#include +#include + +#include "rtc_base/logging.h" + +namespace cuttlefish { +namespace webrtc_streaming { + +namespace { + +class CompositeDecoderFactory : public webrtc::VideoDecoderFactory { + public: + explicit CompositeDecoderFactory(const DecoderProviderRegistry* registry) + : registry_(registry ? registry : &DecoderProviderRegistry::Get()) {} + + std::vector GetSupportedFormats() const override { + std::vector all_formats; + std::set seen; + + // Collect formats from all available providers, deduplicating by string + // representation. Higher-priority providers are iterated first. + for (const DecoderProvider* provider : registry_->GetAvailableProviders()) { + for (const auto& format : provider->GetSupportedFormats()) { + std::string key = format.ToString(); + if (seen.insert(key).second) { + all_formats.push_back(format); + } + } + } + + return all_formats; + } + + std::unique_ptr CreateVideoDecoder( + const webrtc::SdpVideoFormat& format) override { + // Try providers in priority order until one returns a decoder + for (DecoderProvider* provider : registry_->GetAvailableProviders()) { + auto decoder = provider->CreateDecoder(format); + if (decoder) { + RTC_LOG(LS_INFO) << "Created decoder for " << format.ToString() + << " using provider: " << provider->GetName(); + return decoder; + } + } + + RTC_LOG(LS_WARNING) << "No decoder provider could create decoder for: " + << format.ToString(); + return nullptr; + } + + CodecSupport QueryCodecSupport( + const webrtc::SdpVideoFormat& format, + bool reference_scaling) const override { + // Check if any provider can create this codec + for (const DecoderProvider* provider : registry_->GetAvailableProviders()) { + for (const auto& supported : provider->GetSupportedFormats()) { + if (format.IsSameCodec(supported)) { + return {.is_supported = true}; + } + } + } + return {.is_supported = false}; + } + + private: + const DecoderProviderRegistry* registry_; +}; + +} // namespace + +std::unique_ptr CreateCompositeDecoderFactory( + const DecoderProviderRegistry* registry) { + return std::make_unique(registry); +} + +} // namespace webrtc_streaming +} // namespace cuttlefish diff --git a/base/cvd/cuttlefish/host/frontend/webrtc/libcommon/composite_decoder_factory.h b/base/cvd/cuttlefish/host/frontend/webrtc/libcommon/composite_decoder_factory.h new file mode 100644 index 00000000000..ac6416e2ec7 --- /dev/null +++ b/base/cvd/cuttlefish/host/frontend/webrtc/libcommon/composite_decoder_factory.h @@ -0,0 +1,43 @@ +/* + * Copyright (C) 2026 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef CUTTLEFISH_HOST_FRONTEND_WEBRTC_LIBCOMMON_COMPOSITE_DECODER_FACTORY_H_ +#define CUTTLEFISH_HOST_FRONTEND_WEBRTC_LIBCOMMON_COMPOSITE_DECODER_FACTORY_H_ + +#include + +#include "api/video_codecs/video_decoder_factory.h" +#include "cuttlefish/host/frontend/webrtc/libcommon/decoder_provider_registry.h" + +namespace cuttlefish { +namespace webrtc_streaming { + +// Creates a VideoDecoderFactory that delegates to registered +// DecoderProviders. +// +// The factory merges supported formats from all available providers and +// creates decoders by trying providers in priority order until one +// succeeds. +// +// Uses the global DecoderProviderRegistry by default, but accepts an +// optional registry pointer for dependency injection in tests. +std::unique_ptr CreateCompositeDecoderFactory( + const DecoderProviderRegistry* registry = nullptr); + +} // namespace webrtc_streaming +} // namespace cuttlefish + +#endif // CUTTLEFISH_HOST_FRONTEND_WEBRTC_LIBCOMMON_COMPOSITE_DECODER_FACTORY_H_ diff --git a/base/cvd/cuttlefish/host/frontend/webrtc/libcommon/composite_encoder_factory.cpp b/base/cvd/cuttlefish/host/frontend/webrtc/libcommon/composite_encoder_factory.cpp new file mode 100644 index 00000000000..80cc3b8cfb2 --- /dev/null +++ b/base/cvd/cuttlefish/host/frontend/webrtc/libcommon/composite_encoder_factory.cpp @@ -0,0 +1,96 @@ +/* + * Copyright (C) 2026 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "cuttlefish/host/frontend/webrtc/libcommon/composite_encoder_factory.h" + +#include +#include + +#include "rtc_base/logging.h" + +namespace cuttlefish { +namespace webrtc_streaming { + +namespace { + +class CompositeEncoderFactory : public webrtc::VideoEncoderFactory { + public: + explicit CompositeEncoderFactory(const EncoderProviderRegistry* registry) + : registry_(registry ? registry : &EncoderProviderRegistry::Get()) {} + + std::vector GetSupportedFormats() const override { + std::vector all_formats; + std::set seen; + + // Collect formats from all available providers, deduplicating by string + // representation. Higher-priority providers are iterated first, so their + // format objects are preferred when duplicates exist. + for (const EncoderProvider* provider : registry_->GetAvailableProviders()) { + for (const auto& format : provider->GetSupportedFormats()) { + std::string key = format.ToString(); + if (seen.insert(key).second) { + all_formats.push_back(format); + } + } + } + + return all_formats; + } + + std::unique_ptr CreateVideoEncoder( + const webrtc::SdpVideoFormat& format) override { + // Try providers in priority order until one returns an encoder + for (EncoderProvider* provider : registry_->GetAvailableProviders()) { + auto encoder = provider->CreateEncoder(format); + if (encoder) { + RTC_LOG(LS_INFO) << "Created encoder for " << format.ToString() + << " using provider: " << provider->GetName(); + return encoder; + } + } + + RTC_LOG(LS_WARNING) << "No encoder provider could create encoder for: " + << format.ToString(); + return nullptr; + } + + CodecSupport QueryCodecSupport( + const webrtc::SdpVideoFormat& format, + absl::optional scalability_mode) const override { + // Check if any provider can create this codec + for (const EncoderProvider* provider : registry_->GetAvailableProviders()) { + for (const auto& supported : provider->GetSupportedFormats()) { + if (format.IsSameCodec(supported)) { + return {.is_supported = true}; + } + } + } + return {.is_supported = false}; + } + + private: + const EncoderProviderRegistry* registry_; +}; + +} // namespace + +std::unique_ptr CreateCompositeEncoderFactory( + const EncoderProviderRegistry* registry) { + return std::make_unique(registry); +} + +} // namespace webrtc_streaming +} // namespace cuttlefish diff --git a/base/cvd/cuttlefish/host/frontend/webrtc/libcommon/composite_encoder_factory.h b/base/cvd/cuttlefish/host/frontend/webrtc/libcommon/composite_encoder_factory.h new file mode 100644 index 00000000000..f40fac59097 --- /dev/null +++ b/base/cvd/cuttlefish/host/frontend/webrtc/libcommon/composite_encoder_factory.h @@ -0,0 +1,43 @@ +/* + * Copyright (C) 2026 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef CUTTLEFISH_HOST_FRONTEND_WEBRTC_LIBCOMMON_COMPOSITE_ENCODER_FACTORY_H_ +#define CUTTLEFISH_HOST_FRONTEND_WEBRTC_LIBCOMMON_COMPOSITE_ENCODER_FACTORY_H_ + +#include + +#include "api/video_codecs/video_encoder_factory.h" +#include "cuttlefish/host/frontend/webrtc/libcommon/encoder_provider_registry.h" + +namespace cuttlefish { +namespace webrtc_streaming { + +// Creates a VideoEncoderFactory that delegates to registered +// EncoderProviders. +// +// The factory merges supported formats from all available providers and +// creates encoders by trying providers in priority order until one +// succeeds. +// +// Uses the global EncoderProviderRegistry by default, but accepts an +// optional registry pointer for dependency injection in tests. +std::unique_ptr CreateCompositeEncoderFactory( + const EncoderProviderRegistry* registry = nullptr); + +} // namespace webrtc_streaming +} // namespace cuttlefish + +#endif // CUTTLEFISH_HOST_FRONTEND_WEBRTC_LIBCOMMON_COMPOSITE_ENCODER_FACTORY_H_ diff --git a/base/cvd/cuttlefish/host/frontend/webrtc/libcommon/connection_controller.cpp b/base/cvd/cuttlefish/host/frontend/webrtc/libcommon/connection_controller.cpp index 27a8c07381f..63aa891943c 100644 --- a/base/cvd/cuttlefish/host/frontend/webrtc/libcommon/connection_controller.cpp +++ b/base/cvd/cuttlefish/host/frontend/webrtc/libcommon/connection_controller.cpp @@ -44,7 +44,7 @@ class CreateSessionDescriptionObserverIntermediate void OnSuccess(webrtc::SessionDescriptionInterface* desc) override { auto res = controller_.OnCreateSDPSuccess(desc); if (!res.ok()) { - LOG(ERROR) << res.error(); + LOG(ERROR) << res.error().Message(); } } void OnFailure(webrtc::RTCError error) override { @@ -126,7 +126,7 @@ void ConnectionController::FailConnection(const std::string& message) { reply["error"] = message; auto res = sig_handler_.SendMessage(reply); if (!res.ok()) { - LOG(ERROR) << res.error(); + LOG(ERROR) << res.error().Message(); } observer_.OnConnectionStateChange(CF_ERR(message)); } @@ -277,7 +277,7 @@ ConnectionController::ThisAsSetRemoteSDPObserver() { void ConnectionController::HandleSignalingMessage(const Json::Value& msg) { auto result = HandleSignalingMessageInner(msg); if (!result.ok()) { - LOG(ERROR) << result.error(); + LOG(ERROR) << result.error().Message(); FailConnection(result.error().Message()); } } @@ -419,7 +419,7 @@ void ConnectionController::OnIceCandidate( auto res = sig_handler_.SendMessage(reply); if (!res.ok()) { - LOG(ERROR) << res.error(); + LOG(ERROR) << res.error().Message(); } } diff --git a/base/cvd/cuttlefish/host/frontend/webrtc/libcommon/cuda_context.cpp b/base/cvd/cuttlefish/host/frontend/webrtc/libcommon/cuda_context.cpp new file mode 100644 index 00000000000..8a9bc35730a --- /dev/null +++ b/base/cvd/cuttlefish/host/frontend/webrtc/libcommon/cuda_context.cpp @@ -0,0 +1,126 @@ +/* + * Copyright (C) 2026 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "cuttlefish/host/frontend/webrtc/libcommon/cuda_context.h" + +#include "rtc_base/logging.h" + +namespace cuttlefish { + +std::shared_ptr CudaContext::Get(int device_id) { + static std::mutex mutex; + static std::map> instances; + + RTC_LOG(LS_INFO) << "CudaContext::Get(device_id=" << device_id << ") called"; + + std::lock_guard lock(mutex); + + // Check if we already have a context for this device + auto it = instances.find(device_id); + if (it != instances.end()) { + auto shared = it->second.lock(); + if (shared) { + RTC_LOG(LS_INFO) << "Reusing existing CUDA context for device " << device_id; + return shared; + } + // Weak pointer expired, remove stale entry + RTC_LOG(LS_INFO) << "Previous context expired, creating new one for device " << device_id; + instances.erase(it); + } + + // Initialize CUDA if needed + // Note: cuInit can be called multiple times safely + RTC_LOG(LS_INFO) << "Initializing CUDA driver (cuInit)..."; + CUresult res = cuInit(0); + if (res != CUDA_SUCCESS) { + RTC_LOG(LS_ERROR) << "cuInit failed with error code: " << static_cast(res); + return nullptr; + } + RTC_LOG(LS_INFO) << "cuInit succeeded"; + + CUdevice device; + res = cuDeviceGet(&device, device_id); + if (res != CUDA_SUCCESS) { + RTC_LOG(LS_ERROR) << "cuDeviceGet failed for device " << device_id + << " with error code: " << static_cast(res); + return nullptr; + } + RTC_LOG(LS_INFO) << "cuDeviceGet succeeded for device " << device_id; + + // Log device name for debugging + char device_name[256]; + if (cuDeviceGetName(device_name, sizeof(device_name), device) == CUDA_SUCCESS) { + RTC_LOG(LS_INFO) << "CUDA Device " << device_id << ": " << device_name; + } + + // Use Primary Context for sharing across threads + RTC_LOG(LS_INFO) << "Retaining primary context for device " << device_id << "..."; + CUcontext ctx; + res = cuDevicePrimaryCtxRetain(&ctx, device); + if (res != CUDA_SUCCESS) { + RTC_LOG(LS_ERROR) << "cuDevicePrimaryCtxRetain failed for device " + << device_id << " with error code: " << static_cast(res); + return nullptr; + } + RTC_LOG(LS_INFO) << "cuDevicePrimaryCtxRetain succeeded, context=" << ctx; + + auto shared = std::shared_ptr(new CudaContext(ctx, device_id)); + instances[device_id] = shared; + + RTC_LOG(LS_INFO) << "Created and cached CUDA context for device " << device_id; + return shared; +} + +CudaContext::CudaContext(CUcontext ctx, int device_id) + : ctx_(ctx), device_id_(device_id) {} + +CudaContext::~CudaContext() { + if (ctx_) { + CUdevice device; + CUresult res = cuDeviceGet(&device, device_id_); + if (res == CUDA_SUCCESS) { + cuDevicePrimaryCtxRelease(device); + } + // If cuDeviceGet failed, the device is already gone - nothing to release + } +} + +ScopedCudaContext::ScopedCudaContext(CUcontext ctx) : push_succeeded_(false) { + if (ctx == nullptr) { + RTC_LOG(LS_ERROR) << "ScopedCudaContext: null context provided"; + return; + } + + CUresult res = cuCtxPushCurrent(ctx); + if (res != CUDA_SUCCESS) { + RTC_LOG(LS_ERROR) << "cuCtxPushCurrent failed with error: " << res; + return; + } + + push_succeeded_ = true; +} + +ScopedCudaContext::~ScopedCudaContext() { + if (push_succeeded_) { + CUcontext popped_ctx; + CUresult res = cuCtxPopCurrent(&popped_ctx); + if (res != CUDA_SUCCESS) { + RTC_LOG(LS_ERROR) << "cuCtxPopCurrent failed with error: " << res; + } + } +} + +} // namespace cuttlefish diff --git a/base/cvd/cuttlefish/host/frontend/webrtc/libcommon/cuda_context.h b/base/cvd/cuttlefish/host/frontend/webrtc/libcommon/cuda_context.h new file mode 100644 index 00000000000..63d2ba59da0 --- /dev/null +++ b/base/cvd/cuttlefish/host/frontend/webrtc/libcommon/cuda_context.h @@ -0,0 +1,85 @@ +/* + * Copyright (C) 2026 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef CUTTLEFISH_HOST_FRONTEND_WEBRTC_LIBCOMMON_CUDA_CONTEXT_H_ +#define CUTTLEFISH_HOST_FRONTEND_WEBRTC_LIBCOMMON_CUDA_CONTEXT_H_ + +#include +#include +#include +#include + +namespace cuttlefish { + +// Manages a shared CUDA context for a GPU device. +// +// Uses CUDA's primary context mechanism to share a single context across +// multiple users (e.g., multiple encoder instances). The context is +// automatically retained when obtained and released on destruction. +// +// Thread safety: Get() is thread-safe. The returned context may be used +// from any thread, but must be pushed onto the thread's context stack +// using ScopedCudaContext before making CUDA API calls. +class CudaContext { + public: + // Gets the shared context for the given device. + // Returns nullptr if the device is not available or CUDA initialization + // fails. + static std::shared_ptr Get(int device_id = 0); + + ~CudaContext(); + + // Returns the underlying CUDA context handle. + CUcontext get() const { return ctx_; } + + private: + CudaContext(CUcontext ctx, int device_id); + + CUcontext ctx_; + int device_id_; +}; + +// RAII helper to push/pop a CUDA context on the current thread's stack. +// +// CUDA requires a context to be active on the calling thread before making +// API calls. This class pushes the context in its constructor and pops it +// in the destructor, ensuring balanced operations even if exceptions occur. +// +// Usage: +// ScopedCudaContext scope(cuda_context); +// if (!scope.ok()) { +// // Handle error +// } +// // CUDA calls are now valid... +class ScopedCudaContext { + public: + explicit ScopedCudaContext(CUcontext ctx); + ~ScopedCudaContext(); + + // Returns true if the context was successfully pushed. + bool ok() const { return push_succeeded_; } + + // Non-copyable, non-movable + ScopedCudaContext(const ScopedCudaContext&) = delete; + ScopedCudaContext& operator=(const ScopedCudaContext&) = delete; + + private: + bool push_succeeded_ = false; +}; + +} // namespace cuttlefish + +#endif // CUTTLEFISH_HOST_FRONTEND_WEBRTC_LIBCOMMON_CUDA_CONTEXT_H_ diff --git a/base/cvd/cuttlefish/host/frontend/webrtc/libcommon/decoder_provider.h b/base/cvd/cuttlefish/host/frontend/webrtc/libcommon/decoder_provider.h new file mode 100644 index 00000000000..5af16817966 --- /dev/null +++ b/base/cvd/cuttlefish/host/frontend/webrtc/libcommon/decoder_provider.h @@ -0,0 +1,68 @@ +/* + * Copyright (C) 2026 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef CUTTLEFISH_HOST_FRONTEND_WEBRTC_LIBCOMMON_DECODER_PROVIDER_H_ +#define CUTTLEFISH_HOST_FRONTEND_WEBRTC_LIBCOMMON_DECODER_PROVIDER_H_ + +#include +#include +#include + +#include "api/video_codecs/sdp_video_format.h" +#include "api/video_codecs/video_decoder.h" + +namespace cuttlefish { +namespace webrtc_streaming { + +// Abstract interface for video decoder providers. +// +// Mirrors EncoderProvider for codec negotiation symmetry. WebRTC requires +// matching encoder and decoder format support for successful SDP negotiation. +// Even if the Cuttlefish host never actually decodes video (browser does), +// it must advertise decoder support for the codec negotiation to succeed. +// +// Priority values: +// 0 = Software fallback (builtin WebRTC codecs) +// 100 = Hardware/stub decoders (e.g., H.264 stub for NVENC) +// +// Thread safety: All methods must be safe to call from any thread. +class DecoderProvider { + public: + virtual ~DecoderProvider() = default; + + // Returns a human-readable name for this provider (e.g., "h264_stub", + // "builtin"). + virtual std::string GetName() const = 0; + + // Returns the priority of this provider. Higher values are preferred. + virtual int GetPriority() const = 0; + + // Returns true if this provider can create decoders on the current system. + virtual bool IsAvailable() const = 0; + + // Returns the list of video formats this provider can decode. + virtual std::vector GetSupportedFormats() const = 0; + + // Creates a decoder for the given format. + // Returns nullptr if the format is not supported or creation fails. + virtual std::unique_ptr CreateDecoder( + const webrtc::SdpVideoFormat& format) const = 0; +}; + +} // namespace webrtc_streaming +} // namespace cuttlefish + +#endif // CUTTLEFISH_HOST_FRONTEND_WEBRTC_LIBCOMMON_DECODER_PROVIDER_H_ diff --git a/base/cvd/cuttlefish/host/frontend/webrtc/libcommon/decoder_provider_registry.cpp b/base/cvd/cuttlefish/host/frontend/webrtc/libcommon/decoder_provider_registry.cpp new file mode 100644 index 00000000000..35b39fe9664 --- /dev/null +++ b/base/cvd/cuttlefish/host/frontend/webrtc/libcommon/decoder_provider_registry.cpp @@ -0,0 +1,71 @@ +/* + * Copyright (C) 2026 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "cuttlefish/host/frontend/webrtc/libcommon/decoder_provider_registry.h" + +#include "rtc_base/logging.h" + +namespace cuttlefish { +namespace webrtc_streaming { + +DecoderProviderRegistry& DecoderProviderRegistry::Get() { + // Leaked pointer singleton - intentionally never deleted. + static DecoderProviderRegistry* instance = new DecoderProviderRegistry(); + return *instance; +} + +void DecoderProviderRegistry::Register(std::unique_ptr provider) { + std::lock_guard lock(mutex_); + RTC_LOG(LS_INFO) << "Registering decoder provider: " << provider->GetName() + << " (priority=" << provider->GetPriority() << ")"; + providers_.push_back(std::move(provider)); +} + +std::vector DecoderProviderRegistry::GetAvailableProviders() const { + std::lock_guard lock(mutex_); + + // Build filtered list of available providers + std::vector result; + result.reserve(providers_.size()); + for (const auto& provider : providers_) { + if (provider->IsAvailable()) { + result.push_back(provider.get()); + } + } + + // Sort the COPY by priority (highest first), not the source vector. + std::sort(result.begin(), result.end(), + [](const DecoderProvider* a, const DecoderProvider* b) { + return a->GetPriority() > b->GetPriority(); + }); + + return result; +} + +std::vector DecoderProviderRegistry::GetAllProviders() const { + std::lock_guard lock(mutex_); + + std::vector result; + result.reserve(providers_.size()); + for (const auto& provider : providers_) { + result.push_back(provider.get()); + } + + return result; +} + +} // namespace webrtc_streaming +} // namespace cuttlefish diff --git a/base/cvd/cuttlefish/host/frontend/webrtc/libcommon/decoder_provider_registry.h b/base/cvd/cuttlefish/host/frontend/webrtc/libcommon/decoder_provider_registry.h new file mode 100644 index 00000000000..dc4b24842c0 --- /dev/null +++ b/base/cvd/cuttlefish/host/frontend/webrtc/libcommon/decoder_provider_registry.h @@ -0,0 +1,84 @@ +/* + * Copyright (C) 2026 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef CUTTLEFISH_HOST_FRONTEND_WEBRTC_LIBCOMMON_DECODER_PROVIDER_REGISTRY_H_ +#define CUTTLEFISH_HOST_FRONTEND_WEBRTC_LIBCOMMON_DECODER_PROVIDER_REGISTRY_H_ + +#include +#include +#include +#include + +#include "cuttlefish/host/frontend/webrtc/libcommon/decoder_provider.h" + +namespace cuttlefish { +namespace webrtc_streaming { + +// Thread-safe registry for DecoderProvider instances. +// +// Mirrors EncoderProviderRegistry for codec negotiation symmetry. +// Uses the "leaked pointer" singleton pattern recommended by the Google C++ +// style guide. +class DecoderProviderRegistry { + public: + // Returns the global registry instance. + static DecoderProviderRegistry& Get(); + + // Registers a provider. Takes ownership of the provider. + // Thread-safe. Typically called during static initialization. + void Register(std::unique_ptr provider); + + // Returns all available providers, sorted by priority (highest first). + // Thread-safe. Returns a copy to avoid holding the lock. + std::vector GetAvailableProviders() const; + + // Returns all registered providers (including unavailable ones). + // Thread-safe. Useful for diagnostics. + std::vector GetAllProviders() const; + + private: + DecoderProviderRegistry() = default; + ~DecoderProviderRegistry() = default; + + // Non-copyable + DecoderProviderRegistry(const DecoderProviderRegistry&) = delete; + DecoderProviderRegistry& operator=(const DecoderProviderRegistry&) = delete; + + mutable std::mutex mutex_; + std::vector> providers_; +}; + +// Helper class for static registration of decoder providers. +// Used by the REGISTER_DECODER_PROVIDER macro. +template +class DecoderProviderRegistrar { + public: + DecoderProviderRegistrar() { + DecoderProviderRegistry::Get().Register(std::make_unique()); + } +}; + +// Registers a DecoderProvider class at static initialization time. +// Usage: REGISTER_DECODER_PROVIDER(MyDecoderProvider); +#define REGISTER_DECODER_PROVIDER(ProviderClass) \ + static ::cuttlefish::webrtc_streaming::DecoderProviderRegistrar< \ + ProviderClass> \ + g_decoder_provider_registrar_##ProviderClass + +} // namespace webrtc_streaming +} // namespace cuttlefish + +#endif // CUTTLEFISH_HOST_FRONTEND_WEBRTC_LIBCOMMON_DECODER_PROVIDER_REGISTRY_H_ diff --git a/base/cvd/cuttlefish/host/frontend/webrtc/libcommon/encoder_provider.h b/base/cvd/cuttlefish/host/frontend/webrtc/libcommon/encoder_provider.h new file mode 100644 index 00000000000..1bf22100c8e --- /dev/null +++ b/base/cvd/cuttlefish/host/frontend/webrtc/libcommon/encoder_provider.h @@ -0,0 +1,67 @@ +/* + * Copyright (C) 2026 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef CUTTLEFISH_HOST_FRONTEND_WEBRTC_LIBCOMMON_ENCODER_PROVIDER_H_ +#define CUTTLEFISH_HOST_FRONTEND_WEBRTC_LIBCOMMON_ENCODER_PROVIDER_H_ + +#include +#include +#include + +#include "api/video_codecs/sdp_video_format.h" +#include "api/video_codecs/video_encoder.h" + +namespace cuttlefish { +namespace webrtc_streaming { + +// Abstract interface for video encoder providers. +// +// Implementations register themselves with EncoderProviderRegistry at static +// initialization time. The CompositeEncoderFactory queries all registered +// providers and selects encoders based on priority and availability. +// +// Priority values: +// 0 = Software fallback (builtin WebRTC codecs) +// 100 = Hardware acceleration (e.g., NVENC, VAAPI) +// +// Thread safety: All methods must be safe to call from any thread. +class EncoderProvider { + public: + virtual ~EncoderProvider() = default; + + // Returns a human-readable name for this provider (e.g., "nvenc", "builtin"). + virtual std::string GetName() const = 0; + + // Returns the priority of this provider. Higher values are preferred. + virtual int GetPriority() const = 0; + + // Returns true if this provider can create encoders on the current system. + // May check for hardware availability, driver versions, etc. + virtual bool IsAvailable() const = 0; + + // Returns the list of video formats this provider can encode. + virtual std::vector GetSupportedFormats() const = 0; + + // Creates an encoder for the given format. + // Returns nullptr if the format is not supported or creation fails. + virtual std::unique_ptr CreateEncoder( + const webrtc::SdpVideoFormat& format) const = 0; +}; + +} // namespace webrtc_streaming +} // namespace cuttlefish + +#endif // CUTTLEFISH_HOST_FRONTEND_WEBRTC_LIBCOMMON_ENCODER_PROVIDER_H_ diff --git a/base/cvd/cuttlefish/host/frontend/webrtc/libcommon/encoder_provider_registry.cpp b/base/cvd/cuttlefish/host/frontend/webrtc/libcommon/encoder_provider_registry.cpp new file mode 100644 index 00000000000..e990679b744 --- /dev/null +++ b/base/cvd/cuttlefish/host/frontend/webrtc/libcommon/encoder_provider_registry.cpp @@ -0,0 +1,76 @@ +/* + * Copyright (C) 2026 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "cuttlefish/host/frontend/webrtc/libcommon/encoder_provider_registry.h" + +#include "rtc_base/logging.h" + +namespace cuttlefish { +namespace webrtc_streaming { + +EncoderProviderRegistry& EncoderProviderRegistry::Get() { + // Leaked pointer singleton and intentionally never deleted. + // This pattern is recommended for singletons + // that must survive until program termination, avoiding static destruction + // order issues with providers registered during static initialization. + static EncoderProviderRegistry* instance = new EncoderProviderRegistry(); + return *instance; +} + +void EncoderProviderRegistry::Register(std::unique_ptr provider) { + std::lock_guard lock(mutex_); + RTC_LOG(LS_INFO) << "Registering encoder provider: " << provider->GetName() + << " (priority=" << provider->GetPriority() << ")"; + providers_.push_back(std::move(provider)); +} + +std::vector EncoderProviderRegistry::GetAvailableProviders() const { + std::lock_guard lock(mutex_); + + // Build filtered list of available providers + std::vector result; + result.reserve(providers_.size()); + for (const auto& provider : providers_) { + if (provider->IsAvailable()) { + result.push_back(provider.get()); + } + } + + // Sort the COPY by priority (highest first), not the source vector. + // This is a thread-safety fix: sorting the source would violate const + // correctness and could cause data races if another thread is iterating. + std::sort(result.begin(), result.end(), + [](const EncoderProvider* a, const EncoderProvider* b) { + return a->GetPriority() > b->GetPriority(); + }); + + return result; +} + +std::vector EncoderProviderRegistry::GetAllProviders() const { + std::lock_guard lock(mutex_); + + std::vector result; + result.reserve(providers_.size()); + for (const auto& provider : providers_) { + result.push_back(provider.get()); + } + + return result; +} + +} // namespace webrtc_streaming +} // namespace cuttlefish diff --git a/base/cvd/cuttlefish/host/frontend/webrtc/libcommon/encoder_provider_registry.h b/base/cvd/cuttlefish/host/frontend/webrtc/libcommon/encoder_provider_registry.h new file mode 100644 index 00000000000..e44550ea4bb --- /dev/null +++ b/base/cvd/cuttlefish/host/frontend/webrtc/libcommon/encoder_provider_registry.h @@ -0,0 +1,93 @@ +/* + * Copyright (C) 2026 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef CUTTLEFISH_HOST_FRONTEND_WEBRTC_LIBCOMMON_ENCODER_PROVIDER_REGISTRY_H_ +#define CUTTLEFISH_HOST_FRONTEND_WEBRTC_LIBCOMMON_ENCODER_PROVIDER_REGISTRY_H_ + +#include +#include +#include +#include + +#include "cuttlefish/host/frontend/webrtc/libcommon/encoder_provider.h" + +namespace cuttlefish { +namespace webrtc_streaming { + +// Thread-safe registry for EncoderProvider instances. +// +// Uses the "leaked pointer" singleton pattern. +// +// Example usage: +// // At static init time (via REGISTER_ENCODER_PROVIDER macro) +// EncoderProviderRegistry::Get().Register( +// std::make_unique()); +// +// // At runtime +// for (auto* provider : +// EncoderProviderRegistry::Get().GetAvailableProviders()) { +// // Use provider... +// } +class EncoderProviderRegistry { + public: + // Returns the global registry instance. + static EncoderProviderRegistry& Get(); + + // Registers a provider. Takes ownership of the provider. + // Thread-safe. Typically called during static initialization. + void Register(std::unique_ptr provider); + + // Returns all available providers, sorted by priority (highest first). + // Thread-safe. Returns a copy to avoid holding the lock. + std::vector GetAvailableProviders() const; + + // Returns all registered providers (including unavailable ones). + // Thread-safe. Useful for diagnostics. + std::vector GetAllProviders() const; + + private: + EncoderProviderRegistry() = default; + ~EncoderProviderRegistry() = default; + + // Non-copyable + EncoderProviderRegistry(const EncoderProviderRegistry&) = delete; + EncoderProviderRegistry& operator=(const EncoderProviderRegistry&) = delete; + + mutable std::mutex mutex_; + std::vector> providers_; +}; + +// Helper class for static registration of encoder providers. +// Used by the REGISTER_ENCODER_PROVIDER macro. +template +class EncoderProviderRegistrar { + public: + EncoderProviderRegistrar() { + EncoderProviderRegistry::Get().Register(std::make_unique()); + } +}; + +// Registers an EncoderProvider class at static initialization time. +// Usage: REGISTER_ENCODER_PROVIDER(MyEncoderProvider); +#define REGISTER_ENCODER_PROVIDER(ProviderClass) \ + static ::cuttlefish::webrtc_streaming::EncoderProviderRegistrar< \ + ProviderClass> \ + g_encoder_provider_registrar_##ProviderClass + +} // namespace webrtc_streaming +} // namespace cuttlefish + +#endif // CUTTLEFISH_HOST_FRONTEND_WEBRTC_LIBCOMMON_ENCODER_PROVIDER_REGISTRY_H_ diff --git a/base/cvd/cuttlefish/host/frontend/webrtc/libcommon/nvenc_capabilities.cpp b/base/cvd/cuttlefish/host/frontend/webrtc/libcommon/nvenc_capabilities.cpp new file mode 100644 index 00000000000..c5e7a38f12d --- /dev/null +++ b/base/cvd/cuttlefish/host/frontend/webrtc/libcommon/nvenc_capabilities.cpp @@ -0,0 +1,96 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Queries NVENC codec support from the GPU at startup and caches the +// result. The temporary encode session is opened once via std::call_once +// and the supported codec GUIDs are stored for the lifetime of the +// process. + +#include "cuttlefish/host/frontend/webrtc/libcommon/nvenc_capabilities.h" + +#include +#include +#include + +#include + +#include "cuttlefish/host/frontend/webrtc/libcommon/cuda_context.h" +#include "rtc_base/logging.h" + +namespace cuttlefish { +namespace { + +const std::vector& QuerySupportedCodecGuids() { + static std::once_flag flag; + static std::vector guids; + + std::call_once(flag, [] { + // Load NVENC API. + NV_ENCODE_API_FUNCTION_LIST funcs = {NV_ENCODE_API_FUNCTION_LIST_VER}; + if (NvEncodeAPICreateInstance(&funcs) != NV_ENC_SUCCESS) { + RTC_LOG(LS_WARNING) << "NVENC capabilities: " + << "NvEncodeAPICreateInstance failed"; + return; + } + + auto context = CudaContext::Get(0); + if (!context) { + RTC_LOG(LS_WARNING) << "NVENC capabilities: " + << "failed to get CUDA context for device 0"; + return; + } + + NV_ENC_OPEN_ENCODE_SESSION_EX_PARAMS open_params = + {NV_ENC_OPEN_ENCODE_SESSION_EX_PARAMS_VER}; + open_params.device = context->get(); + open_params.deviceType = NV_ENC_DEVICE_TYPE_CUDA; + open_params.apiVersion = NVENCAPI_VERSION; + + void* encoder = nullptr; + NVENCSTATUS status = + funcs.nvEncOpenEncodeSessionEx(&open_params, &encoder); + if (status != NV_ENC_SUCCESS) { + RTC_LOG(LS_WARNING) << "NVENC capabilities: " + << "nvEncOpenEncodeSessionEx failed: " + << status; + return; + } + + uint32_t guid_count = 0; + funcs.nvEncGetEncodeGUIDCount(encoder, &guid_count); + guids.resize(guid_count); + funcs.nvEncGetEncodeGUIDs(encoder, guids.data(), guid_count, + &guid_count); + funcs.nvEncDestroyEncoder(encoder); + + RTC_LOG(LS_INFO) << "NVENC capabilities: " << guids.size() + << " codec GUID(s) supported by GPU"; + }); + + return guids; +} + +} // namespace + +bool IsNvencCodecSupported(GUID codec_guid) { + const auto& guids = QuerySupportedCodecGuids(); + for (const auto& guid : guids) { + if (memcmp(&guid, &codec_guid, sizeof(GUID)) == 0) { + return true; + } + } + return false; +} + +} // namespace cuttlefish diff --git a/base/cvd/cuttlefish/host/frontend/webrtc/libcommon/nvenc_capabilities.h b/base/cvd/cuttlefish/host/frontend/webrtc/libcommon/nvenc_capabilities.h new file mode 100644 index 00000000000..785ec05bceb --- /dev/null +++ b/base/cvd/cuttlefish/host/frontend/webrtc/libcommon/nvenc_capabilities.h @@ -0,0 +1,30 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef CUTTLEFISH_HOST_FRONTEND_WEBRTC_LIBCOMMON_NVENC_CAPABILITIES_H_ +#define CUTTLEFISH_HOST_FRONTEND_WEBRTC_LIBCOMMON_NVENC_CAPABILITIES_H_ + +#include + +namespace cuttlefish { + +// Returns true if the GPU at device 0 supports encoding with the +// given NVENC codec GUID. Opens a temporary encode session on first +// call, queries all supported codec GUIDs, and caches the result. +// Subsequent calls use the cache. Thread-safe. +bool IsNvencCodecSupported(GUID codec_guid); + +} // namespace cuttlefish + +#endif // CUTTLEFISH_HOST_FRONTEND_WEBRTC_LIBCOMMON_NVENC_CAPABILITIES_H_ diff --git a/base/cvd/cuttlefish/host/frontend/webrtc/libcommon/nvenc_encoder_config.h b/base/cvd/cuttlefish/host/frontend/webrtc/libcommon/nvenc_encoder_config.h new file mode 100644 index 00000000000..1b9be23f3ff --- /dev/null +++ b/base/cvd/cuttlefish/host/frontend/webrtc/libcommon/nvenc_encoder_config.h @@ -0,0 +1,90 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Codec-specific configuration for NvencVideoEncoder. +// Each NVENC provider constructs one of these with the appropriate +// values for its codec. The encoder uses it without knowing which +// codec it is encoding. +// +// This struct uses only trivially destructible types so that config +// constants can be declared at namespace scope without violating the +// Google C++ Style Guide's prohibition on dynamic initialization of +// global variables. + +#ifndef CUTTLEFISH_HOST_FRONTEND_WEBRTC_LIBCOMMON_NVENC_ENCODER_CONFIG_H_ +#define CUTTLEFISH_HOST_FRONTEND_WEBRTC_LIBCOMMON_NVENC_ENCODER_CONFIG_H_ + +#include +#include + +#include + +#include "api/video/video_codec_type.h" +#include "api/video_codecs/video_encoder.h" + +namespace cuttlefish { + +struct NvencEncoderConfig { + // NVENC codec GUID (e.g., NV_ENC_CODEC_H264_GUID). + GUID codec_guid; + + // NVENC profile GUID (e.g., NV_ENC_H264_PROFILE_HIGH_GUID). + GUID profile_guid; + + // WebRTC codec type for EncodedImage metadata. + webrtc::VideoCodecType webrtc_codec_type; + + // Human-readable name for EncoderInfo (e.g., "NvencAv1"). + // Must point to a string literal or other storage with static + // lifetime. + const char* implementation_name; + + // Resolution-based bitrate limits for WebRTC rate control. + // Points to a static array defined in the provider file. + // Format per entry: {frame_size_pixels, min_start_bps, min_bps, + // max_bps}. + const webrtc::VideoEncoder::ResolutionBitrateLimits* bitrate_limits; + size_t bitrate_limits_count; + + // Technical minimum/maximum bitrate for SetRates() clamping. + // min_bitrate_bps should be set to the lowest bitrate at which + // NVENC will accept the configuration (~50 kbps), NOT a quality + // floor. WebRTC's bandwidth estimator must be free to drop the + // bitrate as low as needed to keep the stream alive on constrained + // networks. Quality management is handled by resolution_bitrate_ + // limits (above), which tell WebRTC to downscale the video rather + // than starve the encoder. + int32_t min_bitrate_bps; + int32_t max_bitrate_bps; + + // Called once during InitNvenc, after preset loading and common + // rate control config. Sets codec-specific encode config fields + // (e.g., repeatSPSPPS for H.264, tile config for AV1). + void (*apply_defaults)(NV_ENC_CONFIG* config); + + // Called on every frame before the keyframe check and before + // nvEncEncodePicture. Sets per-frame codec-specific picture + // parameters (e.g., tile counts for AV1). Must not clobber + // encodePicFlags. May be a no-op for codecs that need no + // per-frame params. + void (*set_pic_params)(NV_ENC_PIC_PARAMS* params); +}; + +static_assert(std::is_trivially_destructible_v, + "NvencEncoderConfig must be trivially destructible so " + "it can be used as a namespace-scope constant"); + +} // namespace cuttlefish + +#endif // CUTTLEFISH_HOST_FRONTEND_WEBRTC_LIBCOMMON_NVENC_ENCODER_CONFIG_H_ diff --git a/base/cvd/cuttlefish/host/frontend/webrtc/libcommon/nvenc_video_encoder.cpp b/base/cvd/cuttlefish/host/frontend/webrtc/libcommon/nvenc_video_encoder.cpp new file mode 100644 index 00000000000..b695aed6b87 --- /dev/null +++ b/base/cvd/cuttlefish/host/frontend/webrtc/libcommon/nvenc_video_encoder.cpp @@ -0,0 +1,669 @@ +/* + * Copyright (C) 2026 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "cuttlefish/host/frontend/webrtc/libcommon/nvenc_video_encoder.h" + +#include +#include +#include + +#include +#include +#include "cuttlefish/host/frontend/webrtc/libcommon/abgr_buffer.h" + +#include "modules/video_coding/include/video_error_codes.h" +#include "rtc_base/logging.h" + +namespace cuttlefish { + +NvencVideoEncoder::NvencVideoEncoder(const NvencEncoderConfig& config, + const webrtc::SdpVideoFormat& format) + : config_(config), format_(format) { + RTC_LOG(LS_INFO) << "Creating NvencVideoEncoder (" + << config_.implementation_name + << ") for format: " << format.ToString(); +} + +NvencVideoEncoder::~NvencVideoEncoder() { + Release(); +} + +int32_t NvencVideoEncoder::Release() { + DestroyEncoder(); + return WEBRTC_VIDEO_CODEC_OK; +} + +void NvencVideoEncoder::DestroyEncoder() { + if (encoder_) { + if (nvenc_input_buffer_) { + nvenc_funcs_.nvEncUnregisterResource(encoder_, nvenc_input_buffer_); + nvenc_input_buffer_ = nullptr; + } + if (nvenc_output_bitstream_) { + nvenc_funcs_.nvEncDestroyBitstreamBuffer(encoder_, nvenc_output_bitstream_); + nvenc_output_bitstream_ = nullptr; + } + nvenc_funcs_.nvEncDestroyEncoder(encoder_); + encoder_ = nullptr; + } + + if (cuda_context_) { + ScopedCudaContext scope(cuda_context_); + + if (d_rgba_buffer_) { + cudaFree(d_rgba_buffer_); + d_rgba_buffer_ = nullptr; + } + if (cuda_stream_) { + cudaStreamDestroy(cuda_stream_); + cuda_stream_ = nullptr; + } + } + + shared_context_.reset(); + cuda_context_ = nullptr; + input_resource_registered_ = false; + registered_pixel_format_ = 0; + nvenc_input_format_ = NV_ENC_BUFFER_FORMAT_UNDEFINED; + + inited_ = false; +} + +int NvencVideoEncoder::InitEncode( + const webrtc::VideoCodec* codec_settings, + const webrtc::VideoEncoder::Settings& settings) { + + RTC_LOG(LS_INFO) << "=== NvencVideoEncoder::InitEncode (" + << config_.implementation_name << ") ==="; + + if (!codec_settings) { + RTC_LOG(LS_ERROR) << "InitEncode: codec_settings is null"; + return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; + } + + width_ = codec_settings->width; + height_ = codec_settings->height; + bitrate_bps_ = codec_settings->startBitrate * 1000; + framerate_ = codec_settings->maxFramerate; + codec_settings_ = *codec_settings; + + RTC_LOG(LS_INFO) << "InitEncode parameters:"; + RTC_LOG(LS_INFO) << " Resolution: " << width_ << "x" << height_; + RTC_LOG(LS_INFO) << " Framerate: " << framerate_ << " fps"; + RTC_LOG(LS_INFO) << " Start Bitrate: " << bitrate_bps_ << " bps"; + RTC_LOG(LS_INFO) << " Max Bitrate from codec: " + << codec_settings->maxBitrate << " kbps"; + + // Enforce minimum bitrate + if (bitrate_bps_ < config_.min_bitrate_bps) { + bitrate_bps_ = config_.min_bitrate_bps; + } + + RTC_LOG(LS_INFO) << "Initializing CUDA..."; + if (!InitCuda()) { + RTC_LOG(LS_ERROR) << "InitCuda() FAILED"; + return WEBRTC_VIDEO_CODEC_ERROR; + } + RTC_LOG(LS_INFO) << "InitCuda() succeeded"; + + RTC_LOG(LS_INFO) << "Initializing NVENC..."; + if (!InitNvenc()) { + RTC_LOG(LS_ERROR) << "InitNvenc() FAILED"; + return WEBRTC_VIDEO_CODEC_ERROR; + } + RTC_LOG(LS_INFO) << "InitNvenc() succeeded"; + + inited_ = true; + RTC_LOG(LS_INFO) << "=== NvencVideoEncoder initialized successfully ==="; + RTC_LOG(LS_INFO) << "Note: Input resource will be registered on first " + << "frame (lazy init)"; + return WEBRTC_VIDEO_CODEC_OK; +} + +bool NvencVideoEncoder::InitCuda() { + RTC_LOG(LS_INFO) << "InitCuda: Getting shared CUDA context for " + << "device 0..."; + shared_context_ = CudaContext::Get(0); + if (!shared_context_) { + RTC_LOG(LS_ERROR) << "InitCuda: Failed to get CUDA context"; + return false; + } + cuda_context_ = shared_context_->get(); + RTC_LOG(LS_INFO) << "InitCuda: Got CUDA context: " << cuda_context_; + + ScopedCudaContext scope(cuda_context_); + if (!scope.ok()) { + RTC_LOG(LS_ERROR) << "InitCuda: Failed to push CUDA context"; + return false; + } + + RTC_LOG(LS_INFO) << "InitCuda: Creating CUDA stream..."; + cudaError_t err = cudaStreamCreate(&cuda_stream_); + if (err != cudaSuccess) { + RTC_LOG(LS_ERROR) << "cudaStreamCreate failed: " + << cudaGetErrorString(err); + return false; + } + RTC_LOG(LS_INFO) << "InitCuda: CUDA stream created"; + + // Allocate RGBA buffer (4 bytes per pixel) + // NVENC will handle RGB->YUV conversion internally + RTC_LOG(LS_INFO) << "InitCuda: Allocating RGBA GPU buffer (" + << width_ << "x" << height_ + << " @ 4 bytes/pixel)..."; + err = cudaMallocPitch(&d_rgba_buffer_, &rgba_pitch_, + width_ * 4, height_); + if (err != cudaSuccess) { + RTC_LOG(LS_ERROR) << "cudaMallocPitch (RGBA) failed: " + << cudaGetErrorString(err); + return false; + } + RTC_LOG(LS_INFO) << "InitCuda: RGBA buffer allocated, pitch=" + << rgba_pitch_; + + RTC_LOG(LS_INFO) << "InitCuda: CUDA resources allocated successfully"; + return true; +} + +bool NvencVideoEncoder::InitNvenc() { + RTC_LOG(LS_INFO) << "InitNvenc: Loading NVENC API (version " + << NVENCAPI_MAJOR_VERSION << "." + << NVENCAPI_MINOR_VERSION << ")..."; + + // Load NVENC API + nvenc_funcs_ = {NV_ENCODE_API_FUNCTION_LIST_VER}; + NVENCSTATUS status = NvEncodeAPICreateInstance(&nvenc_funcs_); + if (status != NV_ENC_SUCCESS) { + RTC_LOG(LS_ERROR) << "NvEncodeAPICreateInstance failed: " + << status; + RTC_LOG(LS_ERROR) << "Check that libnvidia-encode.so is " + << "installed and matches driver version"; + return false; + } + RTC_LOG(LS_INFO) << "InitNvenc: NVENC API loaded successfully"; + + // Open Encode Session + RTC_LOG(LS_INFO) << "InitNvenc: Opening encode session with " + << "CUDA context " << cuda_context_; + NV_ENC_OPEN_ENCODE_SESSION_EX_PARAMS open_params = + {NV_ENC_OPEN_ENCODE_SESSION_EX_PARAMS_VER}; + open_params.device = cuda_context_; + open_params.deviceType = NV_ENC_DEVICE_TYPE_CUDA; + open_params.apiVersion = NVENCAPI_VERSION; + + status = nvenc_funcs_.nvEncOpenEncodeSessionEx( + &open_params, &encoder_); + if (status != NV_ENC_SUCCESS) { + RTC_LOG(LS_ERROR) << "nvEncOpenEncodeSessionEx failed: " + << status; + RTC_LOG(LS_ERROR) << "Possible causes: GPU doesn't support " + << "NVENC, or encoder sessions exhausted"; + return false; + } + RTC_LOG(LS_INFO) << "InitNvenc: Encode session opened, encoder=" + << encoder_; + + // Initialize stored config struct + stored_encode_config_ = {NV_ENC_CONFIG_VER}; + + // Configure Encoder + RTC_LOG(LS_INFO) << "InitNvenc: Configuring " + << config_.implementation_name << " encoder..."; + RTC_LOG(LS_INFO) << "InitNvenc: Preset: P4 (Balanced)"; + RTC_LOG(LS_INFO) << "InitNvenc: Tuning: Ultra Low Latency"; + RTC_LOG(LS_INFO) << "InitNvenc: Resolution: " << width_ << "x" + << height_; + RTC_LOG(LS_INFO) << "InitNvenc: Framerate: " << framerate_ + << " fps"; + + stored_init_params_ = {NV_ENC_INITIALIZE_PARAMS_VER}; + stored_init_params_.encodeConfig = &stored_encode_config_; + stored_init_params_.encodeGUID = config_.codec_guid; + stored_init_params_.presetGUID = NV_ENC_PRESET_P4_GUID; + stored_init_params_.tuningInfo = + NV_ENC_TUNING_INFO_ULTRA_LOW_LATENCY; + stored_init_params_.encodeWidth = width_; + stored_init_params_.encodeHeight = height_; + stored_init_params_.darWidth = width_; + stored_init_params_.darHeight = height_; + stored_init_params_.frameRateNum = framerate_; + stored_init_params_.frameRateDen = 1; + stored_init_params_.enablePTD = 1; + stored_init_params_.enableEncodeAsync = 0; + + // Load preset defaults + RTC_LOG(LS_INFO) << "InitNvenc: Loading preset configuration..."; + NV_ENC_PRESET_CONFIG preset_config = {NV_ENC_PRESET_CONFIG_VER}; + preset_config.presetCfg = {NV_ENC_CONFIG_VER}; + + status = nvenc_funcs_.nvEncGetEncodePresetConfigEx( + encoder_, stored_init_params_.encodeGUID, + stored_init_params_.presetGUID, + stored_init_params_.tuningInfo, &preset_config); + + if (status != NV_ENC_SUCCESS) { + RTC_LOG(LS_ERROR) << "nvEncGetEncodePresetConfigEx failed: " + << status; + return false; + } + RTC_LOG(LS_INFO) << "InitNvenc: Preset configuration loaded"; + + // Copy preset config to stored config + stored_encode_config_ = preset_config.presetCfg; + stored_encode_config_.version = NV_ENC_CONFIG_VER; + + // Custom overrides for ultra-low latency WebRTC streaming + stored_encode_config_.gopLength = NVENC_INFINITE_GOPLENGTH; + stored_encode_config_.frameIntervalP = 1; // No B-frames + stored_encode_config_.rcParams.rateControlMode = + NV_ENC_PARAMS_RC_CBR; + stored_encode_config_.rcParams.averageBitRate = bitrate_bps_; + stored_encode_config_.rcParams.vbvBufferSize = bitrate_bps_; + stored_encode_config_.rcParams.vbvInitialDelay = bitrate_bps_; + stored_encode_config_.rcParams.zeroReorderDelay = 1; + + // Apply codec-specific configuration + config_.apply_defaults(&stored_encode_config_); + + // Ensure encodeConfig points to stored config + stored_init_params_.encodeConfig = &stored_encode_config_; + + // Initialize encoder + RTC_LOG(LS_INFO) << "InitNvenc: Initializing encoder..."; + status = nvenc_funcs_.nvEncInitializeEncoder( + encoder_, &stored_init_params_); + if (status != NV_ENC_SUCCESS) { + RTC_LOG(LS_ERROR) << "nvEncInitializeEncoder failed: " + << status; + RTC_LOG(LS_ERROR) << "Check that resolution/framerate are " + << "supported by the GPU"; + return false; + } + + RTC_LOG(LS_INFO) << "InitNvenc: Encoder initialized successfully"; + RTC_LOG(LS_INFO) << "InitNvenc: " << width_ << "x" << height_ + << " @" << framerate_ << "fps, " + << bitrate_bps_ << "bps"; + RTC_LOG(LS_INFO) << "InitNvenc: Rate control: CBR, " + << "GOP: infinite, B-frames: 0"; + + // NOTE: Input resource registration is deferred to first Encode() + // call to detect the actual pixel format (ARGB vs ABGR) + + // Create Output Bitstream Buffer + RTC_LOG(LS_INFO) << "InitNvenc: Creating output bitstream " + << "buffer..."; + NV_ENC_CREATE_BITSTREAM_BUFFER create_bs = + {NV_ENC_CREATE_BITSTREAM_BUFFER_VER}; + status = nvenc_funcs_.nvEncCreateBitstreamBuffer( + encoder_, &create_bs); + if (status != NV_ENC_SUCCESS) { + RTC_LOG(LS_ERROR) << "nvEncCreateBitstreamBuffer failed: " + << status; + return false; + } + nvenc_output_bitstream_ = create_bs.bitstreamBuffer; + RTC_LOG(LS_INFO) << "InitNvenc: Output bitstream buffer created: " + << nvenc_output_bitstream_; + + RTC_LOG(LS_INFO) << "=== NVENC initialization complete " + << "(input resource pending) ==="; + return true; +} + +bool NvencVideoEncoder::RegisterInputResource(uint32_t pixel_format) { + // Map DRM format to NVENC buffer format + // DRM_FORMAT_ARGB8888: Memory [B][G][R][A] = NV_ENC_BUFFER_FORMAT_ARGB + // DRM_FORMAT_ABGR8888: Memory [R][G][B][A] = NV_ENC_BUFFER_FORMAT_ABGR + NV_ENC_BUFFER_FORMAT nvenc_fmt; + const char* fmt_name; + + if (pixel_format == DRM_FORMAT_ARGB8888 || + pixel_format == DRM_FORMAT_XRGB8888) { + nvenc_fmt = NV_ENC_BUFFER_FORMAT_ARGB; + fmt_name = "ARGB"; + } else if (pixel_format == DRM_FORMAT_ABGR8888 || + pixel_format == DRM_FORMAT_XBGR8888) { + nvenc_fmt = NV_ENC_BUFFER_FORMAT_ABGR; + fmt_name = "ABGR"; + } else { + char hex_buf[16]; + snprintf(hex_buf, sizeof(hex_buf), "0x%08X", pixel_format); + RTC_LOG(LS_ERROR) << "RegisterInputResource: Unsupported " + << "pixel format: " << hex_buf; + return false; + } + + char hex_buf[16]; + snprintf(hex_buf, sizeof(hex_buf), "0x%08X", pixel_format); + RTC_LOG(LS_INFO) << "RegisterInputResource: Detected format " + << fmt_name << " (DRM " << hex_buf << ")"; + RTC_LOG(LS_INFO) << "RegisterInputResource: Using NVENC native " + << fmt_name + << " input (GPU handles RGB->YUV conversion)"; + + // Register RGBA GPU buffer with NVENC + RTC_LOG(LS_INFO) << "RegisterInputResource: Registering RGBA " + << "GPU buffer..."; + RTC_LOG(LS_INFO) << "RegisterInputResource: Buffer: " + << d_rgba_buffer_ << ", pitch=" << rgba_pitch_; + + NV_ENC_REGISTER_RESOURCE reg_res = + {NV_ENC_REGISTER_RESOURCE_VER}; + reg_res.resourceType = NV_ENC_INPUT_RESOURCE_TYPE_CUDADEVICEPTR; + reg_res.resourceToRegister = d_rgba_buffer_; + reg_res.width = width_; + reg_res.height = height_; + reg_res.pitch = rgba_pitch_; + reg_res.bufferFormat = nvenc_fmt; + + NVENCSTATUS status = nvenc_funcs_.nvEncRegisterResource( + encoder_, ®_res); + if (status != NV_ENC_SUCCESS) { + RTC_LOG(LS_ERROR) << "nvEncRegisterResource failed: " + << status; + return false; + } + + nvenc_input_buffer_ = reg_res.registeredResource; + nvenc_input_format_ = nvenc_fmt; + registered_pixel_format_ = pixel_format; + input_resource_registered_ = true; + + RTC_LOG(LS_INFO) << "RegisterInputResource: Input resource " + << "registered successfully"; + RTC_LOG(LS_INFO) << "RegisterInputResource: Handle: " + << nvenc_input_buffer_; + RTC_LOG(LS_INFO) << "RegisterInputResource: NVENC will use " + << "CUDA internally for RGB->YUV conversion"; + + return true; +} + +int32_t NvencVideoEncoder::RegisterEncodeCompleteCallback( + webrtc::EncodedImageCallback* callback) { + callback_ = callback; + return WEBRTC_VIDEO_CODEC_OK; +} + +void NvencVideoEncoder::SetRates( + const RateControlParameters& parameters) { + if (!inited_) return; + + if (parameters.bitrate.get_sum_bps() != 0) { + int32_t requested_bitrate = parameters.bitrate.get_sum_bps(); + + // Clamp bitrate to configured min/max range + bitrate_bps_ = std::max(requested_bitrate, + config_.min_bitrate_bps); + bitrate_bps_ = std::min(bitrate_bps_, + config_.max_bitrate_bps); + + Reconfigure(parameters); + } + framerate_ = parameters.framerate_fps; +} + +void NvencVideoEncoder::Reconfigure( + const RateControlParameters& parameters) { + if (!encoder_) { + RTC_LOG(LS_WARNING) << "Reconfigure called but encoder not " + << "initialized"; + return; + } + + // Update stored config with new bitrate + stored_encode_config_.rcParams.averageBitRate = bitrate_bps_; + stored_encode_config_.rcParams.vbvBufferSize = bitrate_bps_; + stored_encode_config_.rcParams.vbvInitialDelay = bitrate_bps_; + + // Update framerate if changed + if (parameters.framerate_fps > 0) { + stored_init_params_.frameRateNum = + static_cast(parameters.framerate_fps); + stored_init_params_.frameRateDen = 1; + } + + // Ensure encodeConfig points to our stored config + stored_init_params_.encodeConfig = &stored_encode_config_; + + // Build reconfigure params using stored init params + NV_ENC_RECONFIGURE_PARAMS reconfig = + {NV_ENC_RECONFIGURE_PARAMS_VER}; + reconfig.reInitEncodeParams = stored_init_params_; + reconfig.resetEncoder = 0; + reconfig.forceIDR = 0; + + NVENCSTATUS status = nvenc_funcs_.nvEncReconfigureEncoder( + encoder_, &reconfig); + if (status != NV_ENC_SUCCESS) { + RTC_LOG(LS_ERROR) << "nvEncReconfigureEncoder failed: " + << status; + } else { + RTC_LOG(LS_INFO) << "NVENC reconfigured: bitrate=" + << bitrate_bps_ << ", framerate=" + << stored_init_params_.frameRateNum; + } +} + +int32_t NvencVideoEncoder::Encode( + const webrtc::VideoFrame& frame, + const std::vector* frame_types) { + + if (++frame_count_ % 300 == 1) { + RTC_LOG(LS_INFO) << "NVENC Encode() called, frame #" + << frame_count_; + } + + if (!inited_ || !callback_) { + return WEBRTC_VIDEO_CODEC_UNINITIALIZED; + } + + // Detect dynamic resolution changes. WebRTC's VideoStreamAdapter + // can downscale frames without calling InitEncode() again. + if (frame.width() != width_ || frame.height() != height_) { + RTC_LOG(LS_WARNING) << "Resolution changed from " + << width_ << "x" << height_ << " to " + << frame.width() << "x" << frame.height(); + return WEBRTC_VIDEO_CODEC_ERROR; + } + + // Push CUDA context for this thread + ScopedCudaContext cuda_scope(cuda_context_); + if (!cuda_scope.ok()) { + RTC_LOG(LS_ERROR) << "Failed to push CUDA context in Encode()"; + return WEBRTC_VIDEO_CODEC_ERROR; + } + + rtc::scoped_refptr buffer = + frame.video_frame_buffer(); + cudaError_t cuda_err; + + // Check for RGBA input (native buffer from gfxstream/Wayland) + if (buffer->type() != webrtc::VideoFrameBuffer::Type::kNative) { + static int non_native_count = 0; + if (++non_native_count <= 5) { + RTC_LOG(LS_ERROR) << "NVENC: Non-native buffer type: " + << static_cast(buffer->type()) + << " (count=" << non_native_count << ")"; + } + return WEBRTC_VIDEO_CODEC_ERROR; + } + + AbgrBuffer* rgba_buf = static_cast(buffer.get()); + uint32_t pixel_format = rgba_buf->PixelFormat(); + + // Validate pixel format + bool is_argb = (pixel_format == DRM_FORMAT_ARGB8888 || + pixel_format == DRM_FORMAT_XRGB8888); + bool is_abgr = (pixel_format == DRM_FORMAT_ABGR8888 || + pixel_format == DRM_FORMAT_XBGR8888); + + if (!is_argb && !is_abgr) { + char hex_buf[16]; + snprintf(hex_buf, sizeof(hex_buf), "0x%08X", pixel_format); + RTC_LOG(LS_ERROR) << "Unsupported pixel format: " << hex_buf; + return WEBRTC_VIDEO_CODEC_ERROR; + } + + // Lazy registration of input resource on first frame + if (!input_resource_registered_) { + if (!RegisterInputResource(pixel_format)) { + RTC_LOG(LS_ERROR) << "Failed to register input resource"; + return WEBRTC_VIDEO_CODEC_ERROR; + } + } + + // Verify format hasn't changed (shouldn't happen in normal + // operation) + if (pixel_format != registered_pixel_format_) { + char old_fmt[16], new_fmt[16]; + snprintf(old_fmt, sizeof(old_fmt), "0x%08X", + registered_pixel_format_); + snprintf(new_fmt, sizeof(new_fmt), "0x%08X", pixel_format); + RTC_LOG(LS_WARNING) << "Pixel format changed from " << old_fmt + << " to " << new_fmt + << " - this is unexpected"; + return WEBRTC_VIDEO_CODEC_ERROR; + } + + // Upload RGBA data directly to GPU + const uint8_t* src_data = rgba_buf->Data(); + int src_stride = rgba_buf->Stride(); + + cuda_err = cudaMemcpy2DAsync(d_rgba_buffer_, rgba_pitch_, + src_data, src_stride, + width_ * 4, height_, + cudaMemcpyHostToDevice, cuda_stream_); + if (cuda_err != cudaSuccess) { + RTC_LOG(LS_ERROR) << "cudaMemcpy2DAsync (RGBA) failed: " + << cudaGetErrorString(cuda_err); + return WEBRTC_VIDEO_CODEC_ERROR; + } + + // Synchronize before encoding + cuda_err = cudaStreamSynchronize(cuda_stream_); + if (cuda_err != cudaSuccess) { + RTC_LOG(LS_ERROR) << "cudaStreamSynchronize failed: " + << cudaGetErrorString(cuda_err); + return WEBRTC_VIDEO_CODEC_ERROR; + } + + // Map Input Resource + NV_ENC_MAP_INPUT_RESOURCE map_input = + {NV_ENC_MAP_INPUT_RESOURCE_VER}; + map_input.registeredResource = nvenc_input_buffer_; + NVENCSTATUS status = nvenc_funcs_.nvEncMapInputResource( + encoder_, &map_input); + if (status != NV_ENC_SUCCESS) { + RTC_LOG(LS_ERROR) << "nvEncMapInputResource failed: " << status; + return WEBRTC_VIDEO_CODEC_ERROR; + } + + // Encode Picture - NVENC handles RGB->YUV conversion internally + NV_ENC_PIC_PARAMS pic_params = {NV_ENC_PIC_PARAMS_VER}; + pic_params.pictureStruct = NV_ENC_PIC_STRUCT_FRAME; + pic_params.inputBuffer = map_input.mappedResource; + pic_params.bufferFmt = nvenc_input_format_; // ARGB or ABGR + pic_params.inputWidth = width_; + pic_params.inputHeight = height_; + pic_params.outputBitstream = nvenc_output_bitstream_; + pic_params.inputPitch = rgba_pitch_; + + // Set per-frame codec params FIRST. + config_.set_pic_params(&pic_params); + + // THEN check for keyframe requests. Use |= to preserve flags. + if (frame_types && !frame_types->empty() && + (*frame_types)[0] == webrtc::VideoFrameType::kVideoFrameKey) { + pic_params.encodePicFlags |= NV_ENC_PIC_FLAG_FORCEIDR; + } + + status = nvenc_funcs_.nvEncEncodePicture(encoder_, &pic_params); + if (status != NV_ENC_SUCCESS) { + RTC_LOG(LS_ERROR) << "nvEncEncodePicture failed: " << status; + nvenc_funcs_.nvEncUnmapInputResource( + encoder_, map_input.mappedResource); + return WEBRTC_VIDEO_CODEC_ERROR; + } + + // Unmap Input + nvenc_funcs_.nvEncUnmapInputResource( + encoder_, map_input.mappedResource); + + // Retrieve Bitstream + NV_ENC_LOCK_BITSTREAM lock_bs = {NV_ENC_LOCK_BITSTREAM_VER}; + lock_bs.outputBitstream = nvenc_output_bitstream_; + lock_bs.doNotWait = 0; + + status = nvenc_funcs_.nvEncLockBitstream(encoder_, &lock_bs); + if (status != NV_ENC_SUCCESS) { + RTC_LOG(LS_ERROR) << "nvEncLockBitstream failed: " << status; + return WEBRTC_VIDEO_CODEC_ERROR; + } + + // Send to WebRTC + webrtc::EncodedImage encoded_image; + encoded_image.SetEncodedData(webrtc::EncodedImageBuffer::Create( + static_cast(lock_bs.bitstreamBufferPtr), + lock_bs.bitstreamSizeInBytes)); + encoded_image._encodedWidth = width_; + encoded_image._encodedHeight = height_; + encoded_image.SetTimestamp(frame.timestamp()); + encoded_image.ntp_time_ms_ = frame.ntp_time_ms(); + // H.264 signals keyframes as IDR; AV1 uses I-frames (no IDR concept). + // Check both so this works for any codec. + encoded_image._frameType = + (lock_bs.pictureType == NV_ENC_PIC_TYPE_IDR || + lock_bs.pictureType == NV_ENC_PIC_TYPE_I) + ? webrtc::VideoFrameType::kVideoFrameKey + : webrtc::VideoFrameType::kVideoFrameDelta; + + webrtc::CodecSpecificInfo codec_specific = {}; + codec_specific.codecType = config_.webrtc_codec_type; + + callback_->OnEncodedImage(encoded_image, &codec_specific); + + nvenc_funcs_.nvEncUnlockBitstream(encoder_, nvenc_output_bitstream_); + + return WEBRTC_VIDEO_CODEC_OK; +} + +webrtc::VideoEncoder::EncoderInfo NvencVideoEncoder::GetEncoderInfo() + const { + EncoderInfo info; + info.supports_native_handle = true; + info.implementation_name = config_.implementation_name; + info.has_trusted_rate_controller = true; + info.is_hardware_accelerated = true; + + // CRITICAL: Tell WebRTC we prefer native RGBA buffers, NOT I420. + // Without this, WebRTC defaults to kI420 and does expensive + // CPU-based RGBA->I420 conversion before calling Encode(), wasting + // CPU cycles. NVENC handles ARGB/ABGR natively and does RGB->YUV + // conversion on GPU. + info.preferred_pixel_formats = + {webrtc::VideoFrameBuffer::Type::kNative}; + + // Set resolution-based bitrate limits from config. + info.resolution_bitrate_limits.assign( + config_.bitrate_limits, + config_.bitrate_limits + config_.bitrate_limits_count); + + return info; +} + +} // namespace cuttlefish diff --git a/base/cvd/cuttlefish/host/frontend/webrtc/libcommon/nvenc_video_encoder.h b/base/cvd/cuttlefish/host/frontend/webrtc/libcommon/nvenc_video_encoder.h new file mode 100644 index 00000000000..cebd67a17aa --- /dev/null +++ b/base/cvd/cuttlefish/host/frontend/webrtc/libcommon/nvenc_video_encoder.h @@ -0,0 +1,119 @@ +/* + * Copyright (C) 2026 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef CUTTLEFISH_HOST_FRONTEND_WEBRTC_LIBCOMMON_NVENC_VIDEO_ENCODER_H_ +#define CUTTLEFISH_HOST_FRONTEND_WEBRTC_LIBCOMMON_NVENC_VIDEO_ENCODER_H_ + +#include +#include + +#include "api/video/video_codec_type.h" +#include "api/video/video_frame.h" +#include "api/video_codecs/sdp_video_format.h" +#include "api/video_codecs/video_encoder.h" +#include "modules/video_coding/include/video_codec_interface.h" + +#include +#include +#include + +#include "cuttlefish/host/frontend/webrtc/libcommon/cuda_context.h" +#include "cuttlefish/host/frontend/webrtc/libcommon/nvenc_encoder_config.h" + +namespace cuttlefish { + +// Hardware-accelerated video encoder using NVIDIA NVENC. +// +// This encoder accepts native ARGB/ABGR input buffers from the display +// compositor and uses NVENC's internal RGB-to-YUV conversion for optimal +// performance. No custom CUDA kernels are needed for color space conversion. +// +// Key features: +// - Native ARGB/ABGR input (no CPU-side color conversion) +// - CBR rate control optimized for low-latency WebRTC streaming +// - Ultra-low latency tuning (no B-frames, infinite GOP) +// - Resolution-based bitrate limits for high-quality streaming +// - Lazy input resource registration (detects pixel format on first frame) +// +// Thread safety: InitEncode() must be called before Encode(). After +// initialization, Encode() and SetRates() may be called from any thread. +class NvencVideoEncoder : public webrtc::VideoEncoder { + public: + NvencVideoEncoder(const NvencEncoderConfig& config, + const webrtc::SdpVideoFormat& format); + ~NvencVideoEncoder() override; + + int InitEncode(const webrtc::VideoCodec* codec_settings, + const webrtc::VideoEncoder::Settings& settings) override; + int32_t RegisterEncodeCompleteCallback( + webrtc::EncodedImageCallback* callback) override; + int32_t Release() override; + int32_t Encode(const webrtc::VideoFrame& frame, + const std::vector* frame_types) override; + void SetRates(const RateControlParameters& parameters) override; + EncoderInfo GetEncoderInfo() const override; + + private: + bool InitCuda(); + bool InitNvenc(); + bool RegisterInputResource(uint32_t pixel_format); + void DestroyEncoder(); + void Reconfigure(const RateControlParameters& parameters); + + NvencEncoderConfig config_; + webrtc::SdpVideoFormat format_; + webrtc::VideoCodec codec_settings_; + int32_t bitrate_bps_ = 0; + uint32_t framerate_ = 30; + bool inited_ = false; + + // CUDA context + std::shared_ptr shared_context_; + CUcontext cuda_context_ = nullptr; + cudaStream_t cuda_stream_ = nullptr; + + // GPU buffer for RGBA input (NVENC handles RGB->YUV conversion internally) + void* d_rgba_buffer_ = nullptr; + NV_ENC_INPUT_PTR nvenc_input_buffer_ = nullptr; + NV_ENC_OUTPUT_PTR nvenc_output_bitstream_ = nullptr; + + // Input resource registration state (lazy registration on first frame) + bool input_resource_registered_ = false; + uint32_t registered_pixel_format_ = 0; + NV_ENC_BUFFER_FORMAT nvenc_input_format_ = NV_ENC_BUFFER_FORMAT_UNDEFINED; + + // NVENC encoder + void* encoder_ = nullptr; + NV_ENCODE_API_FUNCTION_LIST nvenc_funcs_{}; + + // Stored initialization params for reconfiguration + NV_ENC_INITIALIZE_PARAMS stored_init_params_{}; + NV_ENC_CONFIG stored_encode_config_{}; + + // Frame dimensions + int width_ = 0; + int height_ = 0; + size_t rgba_pitch_ = 0; + + // Frame counter for periodic logging (per-instance, not static) + int frame_count_ = 0; + + webrtc::EncodedImageCallback* callback_ = nullptr; +}; + +} // namespace cuttlefish + +#endif // CUTTLEFISH_HOST_FRONTEND_WEBRTC_LIBCOMMON_NVENC_VIDEO_ENCODER_H_ \ No newline at end of file diff --git a/base/cvd/cuttlefish/host/frontend/webrtc/libcommon/peer_connection_utils.cpp b/base/cvd/cuttlefish/host/frontend/webrtc/libcommon/peer_connection_utils.cpp index cfea39696df..ac0f440cd71 100644 --- a/base/cvd/cuttlefish/host/frontend/webrtc/libcommon/peer_connection_utils.cpp +++ b/base/cvd/cuttlefish/host/frontend/webrtc/libcommon/peer_connection_utils.cpp @@ -20,11 +20,13 @@ #include #include #include -#include -#include #include "cuttlefish/host/frontend/webrtc/libcommon/audio_device.h" -#include "cuttlefish/host/frontend/webrtc/libcommon/vp8only_encoder_factory.h" +#include "cuttlefish/host/frontend/webrtc/libcommon/composite_decoder_factory.h" +#include "cuttlefish/host/frontend/webrtc/libcommon/composite_encoder_factory.h" +#include "cuttlefish/host/frontend/webrtc/libcommon/encoder_provider_registry.h" +#include "cuttlefish/host/frontend/webrtc/libcommon/decoder_provider_registry.h" +#include "rtc_base/logging.h" namespace cuttlefish { namespace webrtc_streaming { @@ -43,14 +45,52 @@ CreatePeerConnectionFactory( rtc::Thread* network_thread, rtc::Thread* worker_thread, rtc::Thread* signal_thread, rtc::scoped_refptr audio_device_module) { + + RTC_LOG(LS_INFO) << "=== Creating PeerConnectionFactory with plugin framework ==="; + + // Log available encoder providers + RTC_LOG(LS_INFO) << "Available encoder providers:"; + for (auto* provider : EncoderProviderRegistry::Get().GetAvailableProviders()) { + RTC_LOG(LS_INFO) << " - " << provider->GetName() + << " (priority=" << provider->GetPriority() << ")"; + for (const auto& fmt : provider->GetSupportedFormats()) { + RTC_LOG(LS_INFO) << " " << fmt.ToString(); + } + } + + // Log available decoder providers + RTC_LOG(LS_INFO) << "Available decoder providers:"; + for (auto* provider : DecoderProviderRegistry::Get().GetAvailableProviders()) { + RTC_LOG(LS_INFO) << " - " << provider->GetName() + << " (priority=" << provider->GetPriority() << ")"; + for (const auto& fmt : provider->GetSupportedFormats()) { + RTC_LOG(LS_INFO) << " " << fmt.ToString(); + } + } + + // Create composite factories that use all registered providers + auto video_encoder_factory = CreateCompositeEncoderFactory(); + auto video_decoder_factory = CreateCompositeDecoderFactory(); + + // Log final supported formats + auto encoder_formats = video_encoder_factory->GetSupportedFormats(); + RTC_LOG(LS_INFO) << "Composite encoder factory supports " << encoder_formats.size() << " format(s)"; + for (const auto& fmt : encoder_formats) { + RTC_LOG(LS_INFO) << " " << fmt.ToString(); + } + + auto decoder_formats = video_decoder_factory->GetSupportedFormats(); + RTC_LOG(LS_INFO) << "Composite decoder factory supports " << decoder_formats.size() << " format(s)"; + for (const auto& fmt : decoder_formats) { + RTC_LOG(LS_INFO) << " " << fmt.ToString(); + } + auto peer_connection_factory = webrtc::CreatePeerConnectionFactory( network_thread, worker_thread, signal_thread, audio_device_module, webrtc::CreateBuiltinAudioEncoderFactory(), webrtc::CreateBuiltinAudioDecoderFactory(), - // Only VP8 is supported - std::make_unique( - webrtc::CreateBuiltinVideoEncoderFactory()), - webrtc::CreateBuiltinVideoDecoderFactory(), nullptr /* audio_mixer */, + std::move(video_encoder_factory), + std::move(video_decoder_factory), nullptr /* audio_mixer */, nullptr /* audio_processing */); CF_EXPECT(peer_connection_factory.get(), "Failed to create peer connection factory"); diff --git a/base/cvd/cuttlefish/host/frontend/webrtc/libcommon/stub_decoder.h b/base/cvd/cuttlefish/host/frontend/webrtc/libcommon/stub_decoder.h new file mode 100644 index 00000000000..5a5eab6f6ac --- /dev/null +++ b/base/cvd/cuttlefish/host/frontend/webrtc/libcommon/stub_decoder.h @@ -0,0 +1,57 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef CUTTLEFISH_HOST_FRONTEND_WEBRTC_LIBCOMMON_STUB_DECODER_H_ +#define CUTTLEFISH_HOST_FRONTEND_WEBRTC_LIBCOMMON_STUB_DECODER_H_ + +#include "api/video_codecs/video_decoder.h" +#include "modules/video_coding/include/video_error_codes.h" +#include "rtc_base/logging.h" + +namespace cuttlefish { +namespace webrtc_streaming { + +// A stub decoder that does nothing. Cuttlefish only sends video; +// the browser decodes it. This stub satisfies WebRTC's codec +// negotiation requirement. +class StubDecoder : public webrtc::VideoDecoder { + public: + bool Configure(const Settings& settings) override { return true; } + + int32_t Decode(const webrtc::EncodedImage& input_image, + bool missing_frames, + int64_t render_time_ms) override { + RTC_LOG(LS_WARNING) << "StubDecoder::Decode called - unexpected"; + return WEBRTC_VIDEO_CODEC_OK; + } + + int32_t RegisterDecodeCompleteCallback( + webrtc::DecodedImageCallback* callback) override { + return WEBRTC_VIDEO_CODEC_OK; + } + + int32_t Release() override { return WEBRTC_VIDEO_CODEC_OK; } + + DecoderInfo GetDecoderInfo() const override { + DecoderInfo info; + info.implementation_name = "StubDecoder"; + info.is_hardware_accelerated = false; + return info; + } +}; + +} // namespace webrtc_streaming +} // namespace cuttlefish + +#endif // CUTTLEFISH_HOST_FRONTEND_WEBRTC_LIBCOMMON_STUB_DECODER_H_ diff --git a/base/cvd/cuttlefish/host/frontend/webrtc/libdevice/BUILD.bazel b/base/cvd/cuttlefish/host/frontend/webrtc/libdevice/BUILD.bazel index f4a74bd6bf2..b56cc1d5f91 100644 --- a/base/cvd/cuttlefish/host/frontend/webrtc/libdevice/BUILD.bazel +++ b/base/cvd/cuttlefish/host/frontend/webrtc/libdevice/BUILD.bazel @@ -46,6 +46,7 @@ cf_cc_library( "//cuttlefish/common/libs/fs", "//cuttlefish/common/libs/utils:json", "//cuttlefish/common/libs/utils:vsock_connection", + "//cuttlefish/host/frontend/webrtc/libcommon:abgr_buffer", "//cuttlefish/host/frontend/webrtc/libcommon:audio_device", "//cuttlefish/host/frontend/webrtc/libcommon:audio_source", "//cuttlefish/host/frontend/webrtc/libcommon:connection_controller", @@ -61,6 +62,7 @@ cf_cc_library( "@abseil-cpp//absl/log:check", "@boringssl//:crypto", "@jsoncpp", + "@libdrm//:libdrm_fourcc", "@libvpx", "@libwebm//:mkvmuxer", "@libwebrtc", diff --git a/base/cvd/cuttlefish/host/frontend/webrtc/libdevice/client_handler.cpp b/base/cvd/cuttlefish/host/frontend/webrtc/libdevice/client_handler.cpp index 22179874962..80a293e0a5d 100644 --- a/base/cvd/cuttlefish/host/frontend/webrtc/libdevice/client_handler.cpp +++ b/base/cvd/cuttlefish/host/frontend/webrtc/libdevice/client_handler.cpp @@ -104,7 +104,25 @@ ClientHandler::AddTrackToConnection( LOG(ERROR) << "Failed to add track to the peer connection"; return nullptr; } - return err_or_sender.MoveValue(); + auto sender = err_or_sender.MoveValue(); + + // Set higher bitrate limits for video tracks to improve quality at higher + // resolutions. Without this, WebRTC defaults to conservative per-track + // limits that cause poor quality even when bandwidth is available. + if (track->kind() == webrtc::MediaStreamTrackInterface::kVideoKind) { + auto params = sender->GetParameters(); + for (auto& encoding : params.encodings) { + encoding.min_bitrate_bps = 500000; // 500 kbps + encoding.max_bitrate_bps = 15000000; // 15 Mbps per track + } + auto result = sender->SetParameters(params); + if (!result.ok()) { + LOG(WARNING) << "Failed to set video encoding parameters: " + << result.message(); + } + } + + return sender; } bool ClientHandler::AddDisplay( @@ -187,12 +205,13 @@ ClientHandler::Build( CF_EXPECT(AddTrackToConnection(audio_track, peer_connection, label).get()); } - // libwebrtc configures the video encoder with a start bitrate of just 300kbs - // which causes it to drop the first 4 frames it receives. Any value over 2Mbs - // will be capped at 2Mbs when passed to the encoder by the peer_connection - // object, so we pass the maximum possible value here. + // Set explicit bitrate limits for WebRTC's bandwidth estimator. Without + // max_bitrate_bps, the estimator defaults to -1 which causes it to be overly + // conservative, resulting in poor video quality at higher resolutions. webrtc::BitrateSettings bitrate_settings; - bitrate_settings.start_bitrate_bps = 2000000; // 2Mbs + bitrate_settings.min_bitrate_bps = 500000; // 500 kbps + bitrate_settings.start_bitrate_bps = 4000000; // 4 Mbps + bitrate_settings.max_bitrate_bps = 20000000; // 20 Mbps peer_connection->SetBitrate(bitrate_settings); // At least one data channel needs to be created on the side that creates the @@ -226,7 +245,7 @@ void ClientHandler::Close() { void ClientHandler::OnConnectionStateChange( Result new_state) { if (!new_state.ok()) { - LOG(ERROR) << "Connection error: " << new_state.error(); + LOG(ERROR) << "Connection error: " << new_state.error().Message(); Close(); return; } diff --git a/base/cvd/cuttlefish/host/frontend/webrtc/libdevice/data_channels.cpp b/base/cvd/cuttlefish/host/frontend/webrtc/libdevice/data_channels.cpp index 513b1925435..c3132738e0e 100644 --- a/base/cvd/cuttlefish/host/frontend/webrtc/libdevice/data_channels.cpp +++ b/base/cvd/cuttlefish/host/frontend/webrtc/libdevice/data_channels.cpp @@ -440,7 +440,7 @@ void DataChannelHandler::OnMessage(const webrtc::DataBuffer &msg) { } auto res = OnMessageInner(msg); if (!res.ok()) { - LOG(ERROR) << res.error(); + LOG(ERROR) << res.error().Message(); } } diff --git a/base/cvd/cuttlefish/host/frontend/webrtc/libdevice/streamer.cpp b/base/cvd/cuttlefish/host/frontend/webrtc/libdevice/streamer.cpp index aae4aa1036f..2d8a20468ba 100644 --- a/base/cvd/cuttlefish/host/frontend/webrtc/libdevice/streamer.cpp +++ b/base/cvd/cuttlefish/host/frontend/webrtc/libdevice/streamer.cpp @@ -206,21 +206,21 @@ std::unique_ptr Streamer::Create( auto network_thread_result = CreateAndStartThread("network-thread"); if (!network_thread_result.ok()) { - LOG(ERROR) << network_thread_result.error(); + LOG(ERROR) << network_thread_result.error().Message(); return nullptr; } impl->network_thread_ = std::move(*network_thread_result); auto worker_thread_result = CreateAndStartThread("worker-thread"); if (!worker_thread_result.ok()) { - LOG(ERROR) << worker_thread_result.error(); + LOG(ERROR) << worker_thread_result.error().Message(); return nullptr; } impl->worker_thread_ = std::move(*worker_thread_result); auto signal_thread_result = CreateAndStartThread("signal-thread"); if (!signal_thread_result.ok()) { - LOG(ERROR) << signal_thread_result.error(); + LOG(ERROR) << signal_thread_result.error().Message(); return nullptr; } impl->signal_thread_ = std::move(*signal_thread_result); @@ -234,7 +234,7 @@ std::unique_ptr Streamer::Create( impl->signal_thread_.get(), impl->audio_device_module_->device_module()); if (!result.ok()) { - LOG(ERROR) << result.error(); + LOG(ERROR) << result.error().Message(); return nullptr; } impl->peer_connection_factory_ = *result; @@ -548,7 +548,7 @@ void Streamer::Impl::HandleConfigMessage(const Json::Value& server_message) { auto result = ParseIceServersMessage(server_message); if (!result.ok()) { LOG(WARNING) << "Failed to parse ice servers message from server: " - << result.error(); + << result.error().Message(); } operator_config_.servers = *result; } diff --git a/base/cvd/cuttlefish/host/frontend/webrtc/libdevice/video_track_source_impl.cpp b/base/cvd/cuttlefish/host/frontend/webrtc/libdevice/video_track_source_impl.cpp index 9e51c67a49e..31df05e0827 100644 --- a/base/cvd/cuttlefish/host/frontend/webrtc/libdevice/video_track_source_impl.cpp +++ b/base/cvd/cuttlefish/host/frontend/webrtc/libdevice/video_track_source_impl.cpp @@ -16,8 +16,11 @@ #include "cuttlefish/host/frontend/webrtc/libdevice/video_track_source_impl.h" +#include #include +#include "cuttlefish/host/frontend/webrtc/libcommon/abgr_buffer.h" + namespace cuttlefish { namespace webrtc_streaming { @@ -54,10 +57,29 @@ VideoTrackSourceImpl::VideoTrackSourceImpl(int width, int height) void VideoTrackSourceImpl::OnFrame(std::shared_ptr frame, int64_t timestamp_us) { + // Ensure strictly monotonic timestamps to prevent WebRTC from dropping + // frames with "Same/old NTP timestamp" errors. This can happen when + // frames arrive in bursts faster than system clock resolution. + if (timestamp_us <= last_timestamp_us_) { + timestamp_us = last_timestamp_us_ + 1; + } + last_timestamp_us_ = timestamp_us; + + rtc::scoped_refptr buffer; + + uint32_t fmt = frame->PixelFormat(); + if (fmt == DRM_FORMAT_ABGR8888 || fmt == DRM_FORMAT_XBGR8888 || + fmt == DRM_FORMAT_ARGB8888 || fmt == DRM_FORMAT_XRGB8888) { + buffer = rtc::scoped_refptr( + new rtc::RefCountedObject(frame)); + } else { + buffer = rtc::scoped_refptr( + new rtc::RefCountedObject(frame)); + } + auto video_frame = webrtc::VideoFrame::Builder() - .set_video_frame_buffer(rtc::scoped_refptr( - new rtc::RefCountedObject(frame))) + .set_video_frame_buffer(buffer) .set_timestamp_us(timestamp_us) .build(); broadcaster_.OnFrame(video_frame); diff --git a/base/cvd/cuttlefish/host/frontend/webrtc/libdevice/video_track_source_impl.h b/base/cvd/cuttlefish/host/frontend/webrtc/libdevice/video_track_source_impl.h index ce3fe74e5a2..af953435b84 100644 --- a/base/cvd/cuttlefish/host/frontend/webrtc/libdevice/video_track_source_impl.h +++ b/base/cvd/cuttlefish/host/frontend/webrtc/libdevice/video_track_source_impl.h @@ -24,6 +24,18 @@ namespace cuttlefish { namespace webrtc_streaming { +/// Video source implementation that bridges Cuttlefish display frames to WebRTC. +/// +/// Receives video frames from the display compositor (via VideoSink interface) +/// and broadcasts them to WebRTC video tracks. Handles both YUV (I420) and +/// native RGBA frame formats. +/// +/// Key features: +/// - Supports native ARGB/ABGR buffers for hardware encoder passthrough +/// - Enforces strictly monotonic timestamps to prevent frame drops +/// - Broadcasts frames to all registered WebRTC sinks +/// +/// Thread safety: OnFrame() may be called from any thread after construction. class VideoTrackSourceImpl : public webrtc::VideoTrackSource { public: VideoTrackSourceImpl(int width, int height); @@ -48,6 +60,7 @@ class VideoTrackSourceImpl : public webrtc::VideoTrackSource { private: int width_; int height_; + int64_t last_timestamp_us_ = 0; // Ensure strictly monotonic timestamps rtc::VideoBroadcaster broadcaster_; }; diff --git a/base/cvd/cuttlefish/host/libs/screen_connector/video_frame_buffer.h b/base/cvd/cuttlefish/host/libs/screen_connector/video_frame_buffer.h index 90f46729be0..5ae0d4f61e4 100644 --- a/base/cvd/cuttlefish/host/libs/screen_connector/video_frame_buffer.h +++ b/base/cvd/cuttlefish/host/libs/screen_connector/video_frame_buffer.h @@ -19,6 +19,8 @@ #include #include +#include + namespace cuttlefish { class VideoFrameBuffer { @@ -27,15 +29,26 @@ class VideoFrameBuffer { virtual int width() const = 0; virtual int height() const = 0; - virtual int StrideY() const = 0; - virtual int StrideU() const = 0; - virtual int StrideV() const = 0; - virtual uint8_t* DataY() = 0; - virtual uint8_t* DataU() = 0; - virtual uint8_t* DataV() = 0; - virtual size_t DataSizeY() const = 0; - virtual size_t DataSizeU() const = 0; - virtual size_t DataSizeV() const = 0; + + // YUV Interfaces (Planar) + virtual int StrideY() const { return 0; } + virtual int StrideU() const { return 0; } + virtual int StrideV() const { return 0; } + virtual uint8_t* DataY() { return nullptr; } + virtual uint8_t* DataU() { return nullptr; } + virtual uint8_t* DataV() { return nullptr; } + virtual size_t DataSizeY() const { return 0; } + virtual size_t DataSizeU() const { return 0; } + virtual size_t DataSizeV() const { return 0; } + + // Packed Interfaces (ABGR/ARGB) + virtual uint8_t* Data() const { return nullptr; } + virtual int Stride() const { return 0; } + virtual size_t DataSize() const { return 0; } + virtual uint32_t PixelFormat() const { return 0; } + + // Clone interface + virtual std::unique_ptr Clone() const = 0; }; } // namespace cuttlefish diff --git a/base/debian/rules b/base/debian/rules index 01d58f4ba2f..94db693d87b 100755 --- a/base/debian/rules +++ b/base/debian/rules @@ -103,6 +103,12 @@ override_dh_install: rm -rf ${cuttlefish_metrics}/bin/cvd.repo_mapping rm -rf ${cuttlefish_metrics}/bin/cvd.runfiles* +# CUDA/NVENC libraries are installed from NVIDIA's repo, not standard Debian +# packages, so dpkg-shlibdeps cannot find their metadata. Allow missing info. +.PHONY: override_dh_shlibdeps +override_dh_shlibdeps: + dh_shlibdeps --dpkg-shlibdeps-params=--ignore-missing-info + # Repair output files bazel actions made executable .PHONY: override_dh_fixperms override_dh_fixperms: