From 4130ad3a082cccec3a0660cfa9e7e060bcf2ba74 Mon Sep 17 00:00:00 2001 From: Manato HIRABAYASHI Date: Fri, 13 Feb 2026 22:22:50 +0900 Subject: [PATCH 1/9] feat: add accelerated_image_processor_decompression package Signed-off-by: Manato HIRABAYASHI --- .../CMakeLists.txt | 69 ++++ .../builder.hpp | 117 ++++++ .../video_decompressor.hpp | 55 +++ .../package.xml | 21 + .../src/builder.cpp | 74 ++++ .../src/video_decompressor/ffmpeg.cpp | 388 ++++++++++++++++++ 6 files changed, 724 insertions(+) create mode 100644 src/accelerated_image_processor_decompression/CMakeLists.txt create mode 100644 src/accelerated_image_processor_decompression/include/accelerated_image_processor_decompression/builder.hpp create mode 100644 src/accelerated_image_processor_decompression/include/accelerated_image_processor_decompression/video_decompressor.hpp create mode 100644 src/accelerated_image_processor_decompression/package.xml create mode 100644 src/accelerated_image_processor_decompression/src/builder.cpp create mode 100644 src/accelerated_image_processor_decompression/src/video_decompressor/ffmpeg.cpp diff --git a/src/accelerated_image_processor_decompression/CMakeLists.txt b/src/accelerated_image_processor_decompression/CMakeLists.txt new file mode 100644 index 0000000..c07c865 --- /dev/null +++ b/src/accelerated_image_processor_decompression/CMakeLists.txt @@ -0,0 +1,69 @@ +cmake_minimum_required(VERSION 3.14) +project(accelerated_image_processor_decompression LANGUAGES CXX CUDA) + +# Set CMAKE_MODULE_PATH to include the custom cmake modules +set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/cmake/") + +if(NOT CMAKE_CXX_STANDARD) + set(CMAKE_CXX_STANDARD 17) + set(CMAKE_CXX_STANDARD_REQUIRED ON) + set(CMAKE_CXX_EXTENSIONS OFF) +endif() + +if(CMAKE_COMPILER_IS_GNUCXX OR CMAKE_CXX_COMPILER_ID MATCHES "Clang") + add_compile_options(-Wall -Wextra -Wpedantic -Wunused-function) +endif() + +find_package(ament_cmake_auto REQUIRED) +ament_auto_find_build_dependencies() + +# find_package(CUDA) +find_package(CUDAToolkit) +find_library(CUDA_nppicc_LIBRARY nppicc ${CMAKE_CUDA_IMPLICIT_LINK_DIRECTORIES}) +find_library(CUDA_nppidei_LIBRARY nppidei + ${CMAKE_CUDA_IMPLICIT_LINK_DIRECTORIES}) +find_library(CUDA_nppisu_LIBRARY nppisu ${CMAKE_CUDA_IMPLICIT_LINK_DIRECTORIES}) + +# -------------------------------------------------------------------------------- +# Find FFMpeg shared libraries manually ref: +# https://github.com/ros-misc-utilities/ffmpeg_encoder_decoder/blob/master/CMakeLists.txt +# -------------------------------------------------------------------------------- +find_package(PkgConfig REQUIRED) +pkg_check_modules(LIBAV REQUIRED IMPORTED_TARGET libavcodec libavutil) +# -------------------------------------------------------------------------------- + +add_library(${PROJECT_NAME} SHARED src/builder.cpp + src/video_decompressor/ffmpeg.cpp) + +target_include_directories( + ${PROJECT_NAME} PUBLIC $ + $) + +target_include_directories( + ${PROJECT_NAME} + PRIVATE $<$:${CUDAToolkit_INCLUDE_DIRS}>) + +target_link_libraries( + ${PROJECT_NAME} + PRIVATE PkgConfig::LIBAV + CUDA::cudart + CUDA::cuda_driver + ${CUDA_nppicc_LIBRARY} # color conversion + ${CUDA_nppisu_LIBRARY} # memory support functions + ${CUDA_nppidei_LIBRARY} # data exchange and initialization functions +) + +ament_target_dependencies(${PROJECT_NAME} PUBLIC + accelerated_image_processor_common) + +install(TARGETS ${PROJECT_NAME} EXPORT export_${PROJECT_NAME}) +install(DIRECTORY include/${PROJECT_NAME} DESTINATION include) + +if(BUILD_TESTING) + # TODO(manato): add tests +endif() + +ament_export_include_directories(include) +ament_export_targets(export_${PROJECT_NAME}) +ament_export_dependencies(accelerated_image_processor_common) +ament_package() diff --git a/src/accelerated_image_processor_decompression/include/accelerated_image_processor_decompression/builder.hpp b/src/accelerated_image_processor_decompression/include/accelerated_image_processor_decompression/builder.hpp new file mode 100644 index 0000000..b393105 --- /dev/null +++ b/src/accelerated_image_processor_decompression/include/accelerated_image_processor_decompression/builder.hpp @@ -0,0 +1,117 @@ +// Copyright 2026 TIER IV, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include + +#include +#include +#include + +namespace accelerated_image_processor::decompression +{ +/** + * @brief Type alias for decompressor processor. + * @note This might be a specific implementation of a decompressor processor in the future. + */ +using Decompressor = common::BaseProcessor; + +/** + * @brief Compression type enum + */ +enum class DecompressionType : uint8_t { VIDEO }; + +/** + * @brief Convert a string to a decompression type + * @param str String to convert expected strings ["VIDEO"] + * @return DecompressionType + */ +DecompressionType to_decompression_type(const std::string & str); + +/** + * @brief Create a decompressor processor. + * + * @param type Decompression type + * @return std::unique_ptr + */ +std::unique_ptr create_decompressor(DecompressionType type); + +/** + * @brief Create a decompressor processor. + * + * @param type Decompression type name in string format + * @return std::unique_ptr + */ +std::unique_ptr create_decompressor(const std::string & type); + +/** + * @brief Create a decompressor processor with a free function for the postprocess. + * + * @param type Decompression type + * @param fn Free function for the postprocess + * @return std::unique_ptr + */ +template < + typename F, std::enable_if_t, int> = 0> +inline std::unique_ptr create_decompressor(DecompressionType type, F fn) +{ + auto processor = create_decompressor(type); + auto fp = static_cast(fn); + if (fp) processor->register_postprocess(fp); + return processor; +} + +/** + * @brief Create a decompressor processor with a free function for the postprocess. + * + * @param type Decompression type name in string format + * @param fn Free function for the postprocess + * @return std::unique_ptr + */ +template < + typename F, std::enable_if_t, int> = 0> +inline std::unique_ptr create_decompressor(const std::string & type, F fn) +{ + return create_decompressor(to_decompression_type(type), fn); +} + +/** + * @brief Create a decompressor processor with a member function for the postprocess. + * + * @param type Decompression type + * @param obj Object that has a member function for the postprocess + * @return std::unique_ptr + */ +template +inline std::unique_ptr create_decompressor(DecompressionType type, Obj * obj) +{ + auto processor = create_decompressor(type); + processor->register_postprocess(obj); + return processor; +} + +/** + * @brief Create a decompressor processor with a member function for the postprocess. + * + * @param type Decompression type name in string format + * @param obj Object that has a member function for the postprocess + * @return std::unique_ptr + */ +template +inline std::unique_ptr create_decompressor(const std::string & type, Obj * obj) +{ + return create_decompressor(to_decompression_type(type), obj); +} +} // namespace accelerated_image_processor::decompression diff --git a/src/accelerated_image_processor_decompression/include/accelerated_image_processor_decompression/video_decompressor.hpp b/src/accelerated_image_processor_decompression/include/accelerated_image_processor_decompression/video_decompressor.hpp new file mode 100644 index 0000000..42a328d --- /dev/null +++ b/src/accelerated_image_processor_decompression/include/accelerated_image_processor_decompression/video_decompressor.hpp @@ -0,0 +1,55 @@ +// Copyright 2026 TIER IV, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include +#include +#include + +#include + +namespace accelerated_image_processor::decompression +{ +class FfmpegVideoDecompressor; + +/** + * @brief Enumeration of video decompression backends. + */ +enum class VideoBackend : uint8_t { FFMPEG }; + +/** + * @brief Abstract base class for Jetson Video compressors. + */ +class VideoDecompressor : public common::BaseProcessor +{ +public: + explicit VideoDecompressor(VideoBackend backend, common::ParameterMap dedicated_parameters = {}) + : BaseProcessor(dedicated_parameters += {}), backend_(backend) + { + } + + ~VideoDecompressor() override = default; + + /** + * @brief Return the backend enum + */ + VideoBackend backend() const { return backend_; } + +private: + const VideoBackend backend_; //!< Compression backend type. +}; +//!< @brief Factory function to create a FfmpegVideoDecompressor. +std::unique_ptr make_ffmpeg_video_decompressor(); +} // namespace accelerated_image_processor::decompression diff --git a/src/accelerated_image_processor_decompression/package.xml b/src/accelerated_image_processor_decompression/package.xml new file mode 100644 index 0000000..ab252e8 --- /dev/null +++ b/src/accelerated_image_processor_decompression/package.xml @@ -0,0 +1,21 @@ + + + + accelerated_image_processor_decompression + 0.0.0 + Decompression library for accelerated image processing + TIER IV + Apache-2.0 + + ament_cmake_auto + + accelerated_image_processor_common + libavcodec-dev + + ament_lint_auto + ament_lint_common + + + ament_cmake + + diff --git a/src/accelerated_image_processor_decompression/src/builder.cpp b/src/accelerated_image_processor_decompression/src/builder.cpp new file mode 100644 index 0000000..4b3500c --- /dev/null +++ b/src/accelerated_image_processor_decompression/src/builder.cpp @@ -0,0 +1,74 @@ +// Copyright 2026 TIER IV, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "accelerated_image_processor_decompression/builder.hpp" + +#include "accelerated_image_processor_decompression/video_decompressor.hpp" + +#include +#include +#include +#include +#include + +namespace accelerated_image_processor::decompression +{ +namespace +{ +/** + * @brief Normalize a string by removing leading and trailing whitespace and converting to + * uppercase. + * + * @param s The string to normalize. + * @return std::string The normalized string. + */ +inline std::string normalize_str(std::string s) +{ + auto not_space = [](unsigned char c) { return !std::isspace(c); }; + s.erase(s.begin(), std::find_if(s.begin(), s.end(), not_space)); + s.erase(std::find_if(s.rbegin(), s.rend(), not_space).base(), s.end()); + + // upper + for (auto & c : s) { + c = static_cast(std::toupper(static_cast(c))); + } + return s; +} +} // namespace + +DecompressionType to_decompression_type(const std::string & str) +{ + const auto s = normalize_str(str); + if (s == "VIDEO") { + return DecompressionType::VIDEO; + } else { + throw std::invalid_argument("Invalid decompression type: " + str); + } +} + +std::unique_ptr create_decompressor(DecompressionType type) +{ + switch (type) { + case DecompressionType::VIDEO: + return make_ffmpeg_video_decompressor(); + default: + throw std::invalid_argument("Invalid decompression type"); + } +} + +std::unique_ptr create_decompressor(const std::string & type) +{ + return create_decompressor(to_decompression_type(type)); +} +} // namespace accelerated_image_processor::decompression diff --git a/src/accelerated_image_processor_decompression/src/video_decompressor/ffmpeg.cpp b/src/accelerated_image_processor_decompression/src/video_decompressor/ffmpeg.cpp new file mode 100644 index 0000000..784b31b --- /dev/null +++ b/src/accelerated_image_processor_decompression/src/video_decompressor/ffmpeg.cpp @@ -0,0 +1,388 @@ +// Copyright 2026 TIER IV, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "accelerated_image_processor_common/helper.hpp" +#include "accelerated_image_processor_decompression/video_decompressor.hpp" + +#include + +#include // for driver API +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +extern "C" { +#include +#include +#include +#include +#include +#include +#include +} + +namespace accelerated_image_processor::decompression +{ +class FfmpegVideoDecompressor final : public VideoDecompressor +{ +public: + explicit FfmpegVideoDecompressor(common::ParameterMap dedicated_parameters = {}) + : VideoDecompressor(VideoBackend::FFMPEG, dedicated_parameters) + { + // Create a CUDA stream and shared it with NPP + CHECK_CUDA(cudaStreamCreate(&stream_)); + { + npp_stream_ctx_.hStream = stream_; + CHECK_CUDA(cudaGetDevice(&npp_stream_ctx_.nCudaDeviceId)); + cudaDeviceProp dev_prop; + CHECK_CUDA(cudaGetDeviceProperties(&dev_prop, npp_stream_ctx_.nCudaDeviceId)); + npp_stream_ctx_.nMultiProcessorCount = dev_prop.multiProcessorCount; + npp_stream_ctx_.nMaxThreadsPerMultiProcessor = dev_prop.maxThreadsPerMultiProcessor; + npp_stream_ctx_.nMaxThreadsPerBlock = dev_prop.maxThreadsPerBlock; + npp_stream_ctx_.nSharedMemPerBlock = dev_prop.sharedMemPerBlock; + CHECK_CUDA(cudaDeviceGetAttribute( + &npp_stream_ctx_.nCudaDevAttrComputeCapabilityMajor, cudaDevAttrComputeCapabilityMajor, + npp_stream_ctx_.nCudaDeviceId)); + CHECK_CUDA(cudaDeviceGetAttribute( + &npp_stream_ctx_.nCudaDevAttrComputeCapabilityMinor, cudaDevAttrComputeCapabilityMinor, + npp_stream_ctx_.nCudaDeviceId)); + CHECK_CUDA(cudaStreamGetFlags(npp_stream_ctx_.hStream, &npp_stream_ctx_.nStreamFlags)); + } + } + + ~FfmpegVideoDecompressor() + { + cleanup_decoder(); + if (dst_bgr_dev_) { + nppiFree(dst_bgr_dev_); + } + } + + /** + * Override process function that takes one ffmpeg packet and calls the registered post process + * for every decoded frame (NOTE: one ffmpeg packet may contain multiple frames) + */ + std::optional process(const common::Image & image) override + { + auto processed_vec = this->process_packet(image); + + for (auto & processed : processed_vec) { + this->post_process(processed); + } + + // always returns nullopt because returned value will not be consumed + return std::nullopt; + } + +private: + struct DecoderInitResult + { + bool success; + std::string error_msg; + + DecoderInitResult(bool is_success, std::string msg) : success(is_success), error_msg(msg) {} + }; + + void cleanup_decoder() + { + if (packet_) { + av_packet_free(&packet_); + } + if (codec_ctx_) { + avcodec_free_context(&codec_ctx_); + } + if (hw_device_ctx_) { + av_buffer_unref(&hw_device_ctx_); + } + } + + std::string image_format_to_string(const common::ImageFormat & fmt) const + { + switch (fmt) { + case common::ImageFormat::H264: + return "h264"; + case common::ImageFormat::H265: + return "hevc"; + case common::ImageFormat::AV1: + return "av1"; + default: + throw std::runtime_error("Unsuported format was detected"); + } + } + + DecoderInitResult init_decoder(const common::ImageFormat & fmt) + { + // Allocate the HW device context (but not initialize it yet) + // This creates the structure but leaves the internal CUDA context pending + // TODO(manato): expose device type as a parameter would beneficial especially for non-CUDA user + hw_device_ctx_ = av_hwdevice_ctx_alloc(AV_HWDEVICE_TYPE_CUDA); + if (!hw_device_ctx_) { + return DecoderInitResult(false, "Failed to allocate CUDA HW device context"); + } + + // Inject the CUDA Stream + // safely modify the specific HW context settings nwo because initialization hasn't happened + // yet + { + // Extract CUcontext from cuda stream using CUDA driver API + CUcontext cu_context; + if (cuStreamGetCtx(reinterpret_cast(stream_), &cu_context) != CUDA_SUCCESS) { + return DecoderInitResult(false, "Extracting CUcontext failed"); + } + AVHWDeviceContext * av_hw_device_ctx = + reinterpret_cast(hw_device_ctx_->data); + AVCUDADeviceContext * av_cuda_device_ctx = + reinterpret_cast(av_hw_device_ctx->hwctx); + av_cuda_device_ctx->stream = stream_; + av_cuda_device_ctx->cuda_ctx = cu_context; + } + + // Initialize the HW device context + // FFmpeg will now create the underlying CUContext (since we left cuda_ctx->cuda_ctx null) + // but will respect the stream we already assigned + int err = av_hwdevice_ctx_init(hw_device_ctx_); + if (err < 0) { + char err_buf[128]; + av_strerror(err, err_buf, sizeof(err_buf)); + return DecoderInitResult(false, std::string("Failed to init HW device context: ") + err_buf); + } + + // Find decoder + std::string codec_name = image_format_to_string(fmt); + const AVCodec * codec = avcodec_find_decoder_by_name(codec_name.c_str()); + if (!codec) { + return DecoderInitResult(false, "Codec not found: " + codec_name); + } + + // Allocate and Setup context + codec_ctx_ = avcodec_alloc_context3(codec); + codec_ctx_->hw_device_ctx = av_buffer_ref(hw_device_ctx_); + codec_ctx_->get_format = + []([[maybe_unused]] AVCodecContext * ctx, const enum AVPixelFormat * pix_fmts) { + const enum AVPixelFormat * p; + for (p = pix_fmts; *p != -1; p++) { // We strictly look for the CUDA format + if (*p == AV_PIX_FMT_CUDA) { + return *p; + } + } + return AV_PIX_FMT_NONE; + }; // Set the callback to enforce CUDA format + + // Open Codec + if (avcodec_open2(codec_ctx_, codec, nullptr) < 0) { + return DecoderInitResult(false, "Failed to open codec"); + } + + // Allocate packet wrapper + packet_ = av_packet_alloc(); + if (!packet_) { + return DecoderInitResult(false, "Failed to allcate AVPacket"); + } + + // Allocate region to store the decoded result + decoded_frame_ = av_frame_alloc(); + if (!decoded_frame_) { + return DecoderInitResult(false, "Failed to allcate AVFrame"); + } + + return DecoderInitResult(true, ""); + } + + /** + * @brief core process implementation that handles input image (ffmpeg packat) and returns the + * vector of Image + * + * Because one ffmpeg packet may include multiple frames, process_imple, which is the pure virtual + * function that returns one image as a decoding result, can not be applicable for this class. + */ + std::vector process_packet(const common::Image & image) + { + // Initialize decoder for the first attempt + if (!is_ready()) { + if (auto res = init_decoder(image.format); !res.success) { + std::cerr << "Failed to initialize decoder: " << res.error_msg << std::endl; + return std::vector(); + } + log_once("Succeed to init_decoder"); + } + + // Pack received data into ffmpeg format + packet_->data = const_cast(image.data.data()); + packet_->size = image.data.size(); + packet_->pts = image.pts.value(); + packet_->flags = image.flags.value(); + + // Feed packet data to the decoder + int ret = avcodec_send_packet(codec_ctx_, packet_); + if (ret < 0) { + std::cerr << "Packet send error" << std::endl; + return std::vector(); + } + + std::vector processed_vec; + while (ret >= 0) { + // Receive decoded result + // NOTE: avcodec_receive_frame automatically calls av_frame_unref(frame_) + // internally before writing new data, so it is safe to pass a used frame + ret = avcodec_receive_frame(codec_ctx_, decoded_frame_); + if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF) { + av_frame_unref(decoded_frame_); + break; + } else if (ret < 0) { + std::cerr << "Error during decoding" << std::endl; + // Return the results so far + return processed_vec; + } + + // color space conversion + common::Image output_image; + { + output_image.frame_id = image.frame_id; + output_image.timestamp = image.timestamp; + output_image.height = image.height; + output_image.width = image.width; + output_image.format = common::ImageFormat::RAW; + output_image.step = image.width * 3; + output_image.encoding = common::ImageEncoding::BGR; + output_image.data = yuv_to_bgr(decoded_frame_); + } + processed_vec.push_back(std::move(output_image)); + + // Release references explicitly + // Although receive_frame unrefs at the start, we manually unref here + // to return the GPU surface to the pool IMMEDIATELY. + // If we wait until the next callback, we might starve teh decoder's surface pool + av_frame_unref(decoded_frame_); + } + + return processed_vec; + } + + std::vector yuv_to_bgr(AVFrame * frame) + { + size_t width_in_byte = frame->width * 3; // BGR + + // Re-allocate presistent GPU BGR buffer if resolution changes + if (dst_width_ != frame->width || dst_height_ != frame->height) { + if (dst_bgr_dev_) { + nppiFree(dst_bgr_dev_); + } + + dst_bgr_dev_ = nppiMalloc_8u_C3(frame->width, frame->height, &dst_pitch_); + + dst_width_ = frame->width; + dst_height_ = frame->height; + } + + // Color conversion (NV12/YUV444 -> BGR) using NPP + NppiSize roi = {frame->width, frame->height}; + int src_step = frame->linesize[0]; + + // Identify the decoded pixel format + AVPixelFormat decoded_format = AV_PIX_FMT_NONE; + if (frame->hw_frames_ctx) { + auto * ctx = reinterpret_cast(frame->hw_frames_ctx->data); + decoded_format = ctx->sw_format; + } else { + decoded_format = static_cast(frame->format); + } + + // Check the underlying software format to decide the conversion API + // (The output format is automatically determined (will be the same as) encoded data) + switch (decoded_format) { + case AV_PIX_FMT_NV12: { + log_once("AV_PIX_FMT_NV12 format detected"); + // NV12 holds 2 planes + const Npp8u * d_src[2] = {frame->data[0], frame->data[1]}; + CHECK_NPP(nppiNV12ToBGR_8u_P2C3R_Ctx( + d_src, src_step, dst_bgr_dev_, dst_pitch_, roi, npp_stream_ctx_)); + break; + } + case AV_PIX_FMT_YUV444P: { + log_once("AV_PIX_FMT_YUV444P format detected"); + // YUV444 holds 3 planes + const Npp8u * d_src[3] = {frame->data[0], frame->data[1], frame->data[2]}; + CHECK_NPP(nppiYUVToBGR_8u_P3C3R_Ctx( + d_src, src_step, dst_bgr_dev_, dst_pitch_, roi, npp_stream_ctx_)); + break; + } + default: { + std::cerr << "Unsupported format: " << av_get_pix_fmt_name(decoded_format) << std::endl; + } + } + + // Download the data + std::vector bgr_host; + bgr_host.resize(width_in_byte * dst_height_); + CHECK_CUDA(cudaMemcpy2DAsync( + bgr_host.data(), width_in_byte, dst_bgr_dev_, dst_pitch_, width_in_byte, dst_height_, + cudaMemcpyDeviceToHost, stream_)); + + CHECK_CUDA(cudaStreamSynchronize(stream_)); + return bgr_host; + } + + /** + * @brief Utility function that print message only once + */ + inline void log_once(const std::string & msg) + { + static std::once_flag flag; + std::call_once(flag, [&]() { std::cout << msg << std::endl; }); + } + + /** + * @brief dummy override for the pure virtual function. This will not be used in this class + */ + common::Image process_impl(const common::Image & image) override { return image; } + + /** + * @brief Checks if the decompressor is ready for processing. + * + * @return true if the codec context, hardware device context, and decoded frame + * have been successfully initialized; false otherwise. + */ + bool is_ready() const override { return (codec_ctx_) && (hw_device_ctx_) && (decoded_frame_); } + + // FFmpeg stuff + AVPacket * packet_{nullptr}; + AVCodecContext * codec_ctx_{nullptr}; + AVBufferRef * hw_device_ctx_{nullptr}; + AVFrame * decoded_frame_{nullptr}; + + // CUDA resources to store BGR converted image + uint8_t * dst_bgr_dev_{nullptr}; + int dst_pitch_{0}; + int dst_width_{0}; + int dst_height_{0}; + cudaStream_t stream_; + NppStreamContext npp_stream_ctx_; +}; + +std::unique_ptr make_ffmpeg_video_decompressor() +{ + return std::make_unique(); +} + +} // namespace accelerated_image_processor::decompression From 2f09dfb7935e013b24c60f73c81d5554cc16dcf4 Mon Sep 17 00:00:00 2001 From: Manato HIRABAYASHI Date: Tue, 17 Feb 2026 23:48:18 +0900 Subject: [PATCH 2/9] feat: add `decompress_node` in `accelerated_image_processor_ros` Signed-off-by: Manato HIRABAYASHI --- .../CMakeLists.txt | 23 +++- .../config/decompress.param.yaml | 3 + .../conversion.hpp | 24 ++++ .../accelerated_image_processor_ros/qos.hpp | 24 ++++ .../launch/decompress.launch.xml | 16 +++ .../package.xml | 1 + .../src/conversion.cpp | 59 +++++++++ .../src/decompress_node.cpp | 113 ++++++++++++++++++ .../src/decompress_node.hpp | 71 +++++++++++ .../src/qos.cpp | 37 ++++++ 10 files changed, 369 insertions(+), 2 deletions(-) create mode 100644 src/accelerated_image_processor_ros/config/decompress.param.yaml create mode 100644 src/accelerated_image_processor_ros/launch/decompress.launch.xml create mode 100644 src/accelerated_image_processor_ros/src/decompress_node.cpp create mode 100644 src/accelerated_image_processor_ros/src/decompress_node.hpp diff --git a/src/accelerated_image_processor_ros/CMakeLists.txt b/src/accelerated_image_processor_ros/CMakeLists.txt index 5719aa5..97baf7b 100644 --- a/src/accelerated_image_processor_ros/CMakeLists.txt +++ b/src/accelerated_image_processor_ros/CMakeLists.txt @@ -15,17 +15,36 @@ find_package(ament_cmake_auto REQUIRED) find_package(CUDAToolkit) ament_auto_find_build_dependencies() -file(GLOB_RECURSE SOURCE_FILES src/*.cpp) -ament_auto_add_library(${PROJECT_NAME} SHARED ${SOURCE_FILES}) +ament_auto_add_library(${PROJECT_NAME}_peripheral SHARED src/parameter.cpp + src/qos.cpp src/conversion.cpp) + +target_include_directories( + ${PROJECT_NAME}_peripheral + PUBLIC $ + $) + +ament_auto_add_library(${PROJECT_NAME} SHARED src/imgproc_node.cpp) target_include_directories( ${PROJECT_NAME} PUBLIC $ $) +target_link_libraries(${PROJECT_NAME} ${PROJECT_NAME}_peripheral) + +ament_auto_add_library(${PROJECT_NAME}_decompress SHARED + src/decompress_node.cpp) + +target_link_libraries(${PROJECT_NAME}_decompress ${PROJECT_NAME}_peripheral) + rclcpp_components_register_node( ${PROJECT_NAME} PLUGIN "accelerated_image_processor::ros::ImgProcNode" EXECUTABLE ${PROJECT_NAME}_imgproc_node) +rclcpp_components_register_node( + ${PROJECT_NAME}_decompress PLUGIN + "accelerated_image_processor::ros::DecompressNode" EXECUTABLE + ${PROJECT_NAME}_decompress_node) + if(BUILD_TESTING) find_package(ament_cmake_gtest REQUIRED) set(test_files test/conversion.cpp test/parameter.cpp test/qos.cpp) diff --git a/src/accelerated_image_processor_ros/config/decompress.param.yaml b/src/accelerated_image_processor_ros/config/decompress.param.yaml new file mode 100644 index 0000000..6c86aee --- /dev/null +++ b/src/accelerated_image_processor_ros/config/decompress.param.yaml @@ -0,0 +1,3 @@ +/**: + ros__parameters: + max_task_length: 5 diff --git a/src/accelerated_image_processor_ros/include/accelerated_image_processor_ros/conversion.hpp b/src/accelerated_image_processor_ros/include/accelerated_image_processor_ros/conversion.hpp index 04eba83..3aba595 100644 --- a/src/accelerated_image_processor_ros/include/accelerated_image_processor_ros/conversion.hpp +++ b/src/accelerated_image_processor_ros/include/accelerated_image_processor_ros/conversion.hpp @@ -25,6 +25,7 @@ #include #include +#include namespace accelerated_image_processor::ros { @@ -163,4 +164,27 @@ ffmpeg_image_transport_msgs::msg::FFMPEGPacket to_ros_ffmpeg(const common::Image * @return std::string */ std::string to_ros_ffmpeg_encoding(common::ImageFormat format); + +/// --- From ffmpeg_image_transport_msgs::msg::FFMPEGPacket to common::Image --- +/** + * @brief split string by ',' or ';' and return separated elements + * @param input_string std::string to be split + * @return std::vector + */ +std::vector split_string_by_comma_and_semicolon(const std::string & input_string); + +/** + * @brief Convert std::string to common::ImageFormat + * @param encoding_str std::string to be converted + * @return common::ImageFormat + */ +common::ImageFormat from_ros_ffmpeg_encoding(const std::string & encoding_str); + +/** + * @brief Convert ffmpeg_image_transport_msgs::msg::FFMPEGPacket to common::Image + * @param msg ffmpeg_image_transport_msgs::msg::FFMPEGPacket to be converted + * @return common::Image + */ +common::Image from_ros_ffmpeg(const ffmpeg_image_transport_msgs::msg::FFMPEGPacket & msg); + } // namespace accelerated_image_processor::ros diff --git a/src/accelerated_image_processor_ros/include/accelerated_image_processor_ros/qos.hpp b/src/accelerated_image_processor_ros/include/accelerated_image_processor_ros/qos.hpp index 40692f9..3f2e1d7 100644 --- a/src/accelerated_image_processor_ros/include/accelerated_image_processor_ros/qos.hpp +++ b/src/accelerated_image_processor_ros/include/accelerated_image_processor_ros/qos.hpp @@ -44,4 +44,28 @@ std::optional find_qos( bool find_qos( rclcpp::Node * node, const std::string & topic_name, rclcpp::QoS & qos, int throttle_period_ms = 1000); + +/** + * @brief Try to find the topic type for a given topic. + * + * @param node The ROS node to use for querying the topic. + * @param topic_name The name of the topic to query. + * @param throttle_period_ms The period in milliseconds to throttle the logging. + * @return std::string The type for the topic, or std::nullopt if no + * publishers are found or if multiple publishers are found. + */ +std::string find_topic_type( + rclcpp::Node * node, const std::string & topic_name, int throttle_period_ms = 1000); + +/** + * @brief Try to find the topic type for a given topic. + * + * @param node The ROS node to use for querying the topic. + * @param topic_name The name of the topic to query. + * @param topic_type The type for the topic. + * @return bool True if the topic type was found, false otherwise. + */ +bool find_topic_type( + rclcpp::Node * node, const std::string & topic_name, std::string & topic_type, + int throttle_period_ms = 1000); } // namespace accelerated_image_processor::ros diff --git a/src/accelerated_image_processor_ros/launch/decompress.launch.xml b/src/accelerated_image_processor_ros/launch/decompress.launch.xml new file mode 100644 index 0000000..bfcd6b6 --- /dev/null +++ b/src/accelerated_image_processor_ros/launch/decompress.launch.xml @@ -0,0 +1,16 @@ + + + + + + + + + + + + + + + + diff --git a/src/accelerated_image_processor_ros/package.xml b/src/accelerated_image_processor_ros/package.xml index 4d97365..edf1bae 100644 --- a/src/accelerated_image_processor_ros/package.xml +++ b/src/accelerated_image_processor_ros/package.xml @@ -11,6 +11,7 @@ accelerated_image_processor_common accelerated_image_processor_compression + accelerated_image_processor_decompression accelerated_image_processor_pipeline builtin_interfaces ffmpeg_image_transport_msgs diff --git a/src/accelerated_image_processor_ros/src/conversion.cpp b/src/accelerated_image_processor_ros/src/conversion.cpp index e772682..fb17d58 100644 --- a/src/accelerated_image_processor_ros/src/conversion.cpp +++ b/src/accelerated_image_processor_ros/src/conversion.cpp @@ -22,6 +22,7 @@ #include #include +#include namespace accelerated_image_processor::ros { @@ -231,4 +232,62 @@ ffmpeg_image_transport_msgs::msg::FFMPEGPacket to_ros_ffmpeg(const common::Image .is_bigendian(image.is_bigendian) .data(image.data); } + +/// --- From ffmpeg_image_transport_msgs::msg::FFMPEGPacket to common::Image --- + +std::vector split_string_by_comma_and_semicolon(const std::string & input_string) +{ + auto split_by_delimiter = [](const std::vector & str_vec, const char delimiter) { + std::vector return_vec; + for (const auto & str : str_vec) { + std::stringstream ss{str}; + std::string buf; + while (std::getline(ss, buf, delimiter)) { + return_vec.push_back(buf); + } + } + return return_vec; + }; + + std::vector input_string_vec = {input_string}; + auto comma_separated = split_by_delimiter(input_string_vec, ','); + auto semicolon_separated = split_by_delimiter(comma_separated, ';'); + return semicolon_separated; +} + +common::ImageFormat from_ros_ffmpeg_encoding(const std::string & encoding_str) +{ + // Because some encoders provide ';' or ',' separated strings that include codec, the + // original image plane format, ...etc., split it into the elements and try each to see if it + // can be recognized as a valid codec + auto codec_candidates = split_string_by_comma_and_semicolon(encoding_str); + for (const auto & candidate : codec_candidates) { + if (candidate == "h264") { + return common::ImageFormat::H264; + } else if (candidate == "hevc") { + return common::ImageFormat::H265; + } else if (candidate == "av1") { + return common::ImageFormat::AV1; + } else { + continue; + } + } + throw std::runtime_error("Supported codec is not found"); +} + +common::Image from_ros_ffmpeg(const ffmpeg_image_transport_msgs::msg::FFMPEGPacket & msg) +{ + common::Image output; + output.frame_id = msg.header.frame_id; + output.timestamp = from_ros_time(msg.header.stamp); + output.height = msg.height; + output.width = msg.width; + output.format = from_ros_ffmpeg_encoding(msg.encoding); + output.pts = msg.pts; + output.flags = msg.flags; + output.is_bigendian = msg.is_bigendian; + output.data = msg.data; + return output; +} + } // namespace accelerated_image_processor::ros diff --git a/src/accelerated_image_processor_ros/src/decompress_node.cpp b/src/accelerated_image_processor_ros/src/decompress_node.cpp new file mode 100644 index 0000000..8daf27e --- /dev/null +++ b/src/accelerated_image_processor_ros/src/decompress_node.cpp @@ -0,0 +1,113 @@ +// Copyright 2026 TIER IV, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "decompress_node.hpp" + +#include "accelerated_image_processor_decompression/builder.hpp" +#include "accelerated_image_processor_ros/conversion.hpp" +#include "accelerated_image_processor_ros/parameter.hpp" +#include "accelerated_image_processor_ros/qos.hpp" + +#include + +#include +#include +#include +#include + +namespace accelerated_image_processor::ros +{ +DecompressNode::DecompressNode(const rclcpp::NodeOptions & options) +: Node("decompression_node", options) +{ + auto max_task_length = this->declare_parameter("max_task_length"); + + qos_request_timer_ = rclcpp::create_timer( + this, this->get_clock(), std::chrono::milliseconds(100), + [this, max_task_length]() { this->determine_qos(max_task_length); }); +} + +void DecompressNode::determine_qos(const int max_task_length) +{ + auto compressed_topic = + this->get_node_topics_interface()->resolve_topic_name("image_raw/compressed", false); + + rclcpp::QoS compressed_qos(1); + constexpr int throttle_period_ms = 1000; + if (!find_qos(this, compressed_topic, compressed_qos, throttle_period_ms)) { + RCLCPP_ERROR_THROTTLE( + this->get_logger(), *this->get_clock(), throttle_period_ms, + "Failed to find image QoS settings"); + return; + } + + std::string topic_type = ""; + if (!find_topic_type(this, compressed_topic, topic_type, throttle_period_ms)) { + RCLCPP_ERROR_THROTTLE( + this->get_logger(), *this->get_clock(), throttle_period_ms, + "Failed to find compressed topic type"); + return; + } + + if (topic_type == "ffmpeg_image_transport_msgs/msg/FFMPEGPacket") { + compressed_subscription_ = + this->create_subscription( + compressed_topic, compressed_qos, + [this](const ffmpeg_image_transport_msgs::msg::FFMPEGPacket::ConstSharedPtr msg) { + this->on_ffmpeg_packet(msg); + }); + + // video decompressor + { + decompressor_ = + decompression::create_decompressor( + "video", this); + fetch_parameters(this, decompressor_.get(), "decompressor"); + } + } else { + throw std::runtime_error("Unsupported compression type"); + } + + decompressed_publisher_ = + this->create_publisher("image_raw", compressed_qos); + + decompression_worker_.emplace(max_task_length); + + // once all queries received, stop the timer callback1 + qos_request_timer_->cancel(); +} + +void DecompressNode::on_ffmpeg_packet( + const ffmpeg_image_transport_msgs::msg::FFMPEGPacket::ConstSharedPtr msg) +{ + const auto image = std::make_shared(from_ros_ffmpeg(*msg)); + + // image decompression + if (decompression_worker_) { + // NOTE: capture `msg` by value to extend the lifetime of the shared pointer at least until the + // task is completed + decompression_worker_->add_task([this, image, msg]() { decompressor_->process(*image); }); + } +} + +void DecompressNode::publish_decompressed(const common::Image & image) +{ + auto decompressed = to_ros_raw(image); + decompressed_publisher_->publish(std::move(decompressed)); +} + +} // namespace accelerated_image_processor::ros + +#include +RCLCPP_COMPONENTS_REGISTER_NODE(accelerated_image_processor::ros::DecompressNode) diff --git a/src/accelerated_image_processor_ros/src/decompress_node.hpp b/src/accelerated_image_processor_ros/src/decompress_node.hpp new file mode 100644 index 0000000..7dcd8f5 --- /dev/null +++ b/src/accelerated_image_processor_ros/src/decompress_node.hpp @@ -0,0 +1,71 @@ +// Copyright 2026 TIER IV, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include "accelerated_image_processor_ros/task_queue.hpp" + +#include +#include +#include +#include +#include + +#include +#include + +#include +#include + +namespace accelerated_image_processor::ros +{ +class DecompressNode : public rclcpp::Node +{ +public: + explicit DecompressNode(const rclcpp::NodeOptions & options); + +private: + /** + * @brief Determine the QoS for the node based on the input parameters. + * @param max_task_length Maximum task length. + */ + void determine_qos(const int max_task_length); + + /** + * @brief Callback function for video encoded messages. + * @param msg Shared pointer to the received ffmpeg packet message. + */ + void on_ffmpeg_packet(const ffmpeg_image_transport_msgs::msg::FFMPEGPacket::ConstSharedPtr msg); + + /** + * @brief Callback function for publishing decompressed images. + * @param image The decompressed image compressed and to be published. + */ + void publish_decompressed(const common::Image & image); + + // --- image processors --- + std::unique_ptr decompressor_; + + // --- subscriptions and publishers --- + rclcpp::SubscriptionBase::SharedPtr compressed_subscription_; + + rclcpp::Publisher::SharedPtr decompressed_publisher_; + + rclcpp::TimerBase::SharedPtr qos_request_timer_; + + std::optional decompression_worker_; + + bool use_jpeg_compression_; +}; +} // namespace accelerated_image_processor::ros diff --git a/src/accelerated_image_processor_ros/src/qos.cpp b/src/accelerated_image_processor_ros/src/qos.cpp index d59ce7b..c53f0b5 100644 --- a/src/accelerated_image_processor_ros/src/qos.cpp +++ b/src/accelerated_image_processor_ros/src/qos.cpp @@ -57,4 +57,41 @@ bool find_qos( return false; } } + +std::string find_topic_type( + rclcpp::Node * node, const std::string & topic_name, int throttle_period_ms) +{ + const auto qos_list = node->get_publishers_info_by_topic(topic_name); + if (qos_list.size() < 1) { + RCLCPP_INFO_STREAM_THROTTLE( + node->get_logger(), *node->get_clock(), throttle_period_ms, + "Waiting for topic: " << topic_name << " ..."); + return ""; + } else if (qos_list.size() > 1) { + RCLCPP_ERROR_STREAM_THROTTLE( + node->get_logger(), *node->get_clock(), throttle_period_ms, + "Multiple publishers found for topic: " << topic_name << ". Cannot determine proper type"); + + return ""; + } else { + RCLCPP_INFO_STREAM_THROTTLE( + node->get_logger(), *node->get_clock(), throttle_period_ms, + "Type is acquired for topic: " << topic_name); + + return qos_list[0].topic_type(); + } +} + +bool find_topic_type( + rclcpp::Node * node, const std::string & topic_name, std::string & topic_type, + int throttle_period_ms) +{ + topic_type = find_topic_type(node, topic_name, throttle_period_ms); + + if (topic_type.empty()) { + return false; + } else { + return true; + } +} } // namespace accelerated_image_processor::ros From f62b14ea50e910b424853f9c8baf6854cebc33e0 Mon Sep 17 00:00:00 2001 From: Manato HIRABAYASHI Date: Sun, 22 Feb 2026 22:05:46 +0900 Subject: [PATCH 3/9] feat: add actual decompression tests in accelerated_image_processor_decompression Signed-off-by: Manato HIRABAYASHI --- .../CMakeLists.txt | 10 +- .../src/video_decompressor/ffmpeg.cpp | 2 +- .../test/builder.cpp | 112 +++++++ .../test/ffmpeg_video_decompressor.cpp | 106 +++++++ .../test/test_utility.hpp | 295 ++++++++++++++++++ 5 files changed, 523 insertions(+), 2 deletions(-) create mode 100644 src/accelerated_image_processor_decompression/test/builder.cpp create mode 100644 src/accelerated_image_processor_decompression/test/ffmpeg_video_decompressor.cpp create mode 100644 src/accelerated_image_processor_decompression/test/test_utility.hpp diff --git a/src/accelerated_image_processor_decompression/CMakeLists.txt b/src/accelerated_image_processor_decompression/CMakeLists.txt index c07c865..47fbf23 100644 --- a/src/accelerated_image_processor_decompression/CMakeLists.txt +++ b/src/accelerated_image_processor_decompression/CMakeLists.txt @@ -60,7 +60,15 @@ install(TARGETS ${PROJECT_NAME} EXPORT export_${PROJECT_NAME}) install(DIRECTORY include/${PROJECT_NAME} DESTINATION include) if(BUILD_TESTING) - # TODO(manato): add tests + find_package(ament_cmake_gtest REQUIRED) + ament_add_gtest(test_builder test/builder.cpp) + target_link_libraries(test_builder ${PROJECT_NAME}) + + pkg_check_modules(LIBAVFILTER REQUIRED IMPORTED_TARGET libavfilter) + ament_add_gtest(test_ffmpeg_video_decompressor + test/ffmpeg_video_decompressor.cpp) + target_link_libraries(test_ffmpeg_video_decompressor ${PROJECT_NAME} + PkgConfig::LIBAV PkgConfig::LIBAVFILTER) endif() ament_export_include_directories(include) diff --git a/src/accelerated_image_processor_decompression/src/video_decompressor/ffmpeg.cpp b/src/accelerated_image_processor_decompression/src/video_decompressor/ffmpeg.cpp index 784b31b..dbdef38 100644 --- a/src/accelerated_image_processor_decompression/src/video_decompressor/ffmpeg.cpp +++ b/src/accelerated_image_processor_decompression/src/video_decompressor/ffmpeg.cpp @@ -88,7 +88,7 @@ class FfmpegVideoDecompressor final : public VideoDecompressor auto processed_vec = this->process_packet(image); for (auto & processed : processed_vec) { - this->post_process(processed); + this->postprocess(processed); } // always returns nullopt because returned value will not be consumed diff --git a/src/accelerated_image_processor_decompression/test/builder.cpp b/src/accelerated_image_processor_decompression/test/builder.cpp new file mode 100644 index 0000000..4c33c27 --- /dev/null +++ b/src/accelerated_image_processor_decompression/test/builder.cpp @@ -0,0 +1,112 @@ +// Copyright 2026 TIER IV, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include + +#include +#include +#include + +// Decompression headers +#include +#include +#include + +namespace accelerated_image_processor::decompression +{ + +namespace +{ +constexpr auto ExpectedBackend = VideoBackend::FFMPEG; + +/** + * @brief Check decompressor type by dynamic_cast. + */ +void check_decompressor_type(const std::unique_ptr & decompressor) +{ + EXPECT_NE(decompressor, nullptr); + + auto ptr = dynamic_cast(decompressor.get()); + EXPECT_NE(ptr, nullptr); + + EXPECT_EQ(ptr->backend(), ExpectedBackend); +} + +/** + * @brief Dummy class used to test the templated create_decompressor overload + */ +class DummyClass +{ +public: + void dummy_method(const common::Image & /*img*/) {} +}; + +/** + * @brief Dummy free function for postprocess. + */ +void dummy_function(const common::Image &) +{ +} +} // namespace + +TEST(TestDecompressorBuilder, CreateFFMPEGDecompressor1) +{ + auto decompressor = create_decompressor(DecompressionType::VIDEO); + check_decompressor_type(decompressor); +} + +TEST(TestDecompressorBuilder, CreateFFMPEGDecompressor2) +{ + auto decompressor = create_decompressor("video"); + check_decompressor_type(decompressor); +} + +TEST(TestDecompressorBuilder, CreateFFMPEGDecompressor3) +{ + DummyClass dummy; + + auto decompressor = + create_decompressor(DecompressionType::VIDEO, &dummy); + check_decompressor_type(decompressor); +} + +TEST(TestDecompressorBuilder, CreateFFMPEGDecompressor4) +{ + auto decompressor = create_decompressor(DecompressionType::VIDEO, &dummy_function); + check_decompressor_type(decompressor); +} + +TEST(TestDecompressorBuilder, CreateFFMPEGDecompressor5) +{ + DummyClass dummy; + + auto decompressor = create_decompressor("video", &dummy); + check_decompressor_type(decompressor); +} + +TEST(TestDecompressorBuilder, CreateFFMPEGDecompressor6) +{ + auto decompressor = create_decompressor("video", &dummy_function); + check_decompressor_type(decompressor); +} + +} // namespace accelerated_image_processor::decompression + +// Main function for the test executable +int main(int argc, char ** argv) +{ + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/src/accelerated_image_processor_decompression/test/ffmpeg_video_decompressor.cpp b/src/accelerated_image_processor_decompression/test/ffmpeg_video_decompressor.cpp new file mode 100644 index 0000000..425e397 --- /dev/null +++ b/src/accelerated_image_processor_decompression/test/ffmpeg_video_decompressor.cpp @@ -0,0 +1,106 @@ +// Copyright 2026 TIER IV, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "test_utility.hpp" + +#include + +#include +#include +#include +#include + +// Decompression headers +#include +#include +#include + +namespace accelerated_image_processor::decompression +{ +/** + * @brief a free function to check the decoded result + * + * In FFmpeg, a single packet can contain multiple frames. The ffmpeg_video_decompressor + * extracts each frame one by one and forwards it to postprocess(). + * Because of this, process() never returns a frame directly – it + * always returns std::nullopt. This function is therefore registered + * as a callback to handle decoded frames instead of producing them. + */ +template +void check_ffmpeg_video_decompressor_result(const common::Image & decoded) +{ + EXPECT_GT(decoded.data.size(), 0U); + EXPECT_EQ(decoded.width, WIDTH); + EXPECT_EQ(decoded.height, HEIGHT); + + // // This raw file can be converted to the bmp file like: + // // $convert -size 2880x1860 -depth 8 bgr:test_0.raw -colorspace RGB test_0.raw.bmp + // std::ofstream outfile; + // static int count = 0; + // outfile.open("/tmp/test_" + std::to_string(count++) + ".raw", std::ios_base::app); + // outfile.write(reinterpret_cast(decoded.data.data()), decoded.data.size()); + // outfile.close(); +} + +TEST(TestFFMPEGVideoDecompressor, DecompressH264) +{ + auto decompressor = make_ffmpeg_video_decompressor(); + ASSERT_NE(decompressor, nullptr); + + FfmpegTestDataProvider generator; + + auto checker = check_ffmpeg_video_decompressor_result; + decompressor->register_postprocess(checker); + + while (std::optional frame = generator.next()) { + decompressor->process(*frame); + } +} + +TEST(TestFFMPEGVideoDecompressor, DecompressH265) +{ + auto decompressor = make_ffmpeg_video_decompressor(); + ASSERT_NE(decompressor, nullptr); + + FfmpegTestDataProvider generator; + + auto checker = check_ffmpeg_video_decompressor_result; + decompressor->register_postprocess(checker); + + while (std::optional frame = generator.next()) { + decompressor->process(*frame); + } +} + +TEST(TestFFMPEGVideoDecompressor, DecompressAV1) +{ + auto decompressor = make_ffmpeg_video_decompressor(); + ASSERT_NE(decompressor, nullptr); + + FfmpegTestDataProvider generator; + + auto checker = check_ffmpeg_video_decompressor_result; + decompressor->register_postprocess(checker); + + while (std::optional frame = generator.next()) { + decompressor->process(*frame); + } +} +} // namespace accelerated_image_processor::decompression + +int main(int argc, char ** argv) +{ + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/src/accelerated_image_processor_decompression/test/test_utility.hpp b/src/accelerated_image_processor_decompression/test/test_utility.hpp new file mode 100644 index 0000000..7c13c63 --- /dev/null +++ b/src/accelerated_image_processor_decompression/test/test_utility.hpp @@ -0,0 +1,295 @@ +// Copyright 2026 TIER IV, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include +#include + +#include +#include +#include +#include +#include + +extern "C" { +#include +#include +#include +#include +#include +#include +#include +#include +} + +#include + +namespace accelerated_image_processor::decompression +{ + +/** + * @brief Test helper that produces a stream of encoded frames using FFmpeg. + * + * The class is parameterised by the codec name (`h264`, `h265`, `av1`) and + * the number of frames to generate. It exposes a `next()` method that + * returns a `common::Image` containing the *encoded* packet. The packet + * can then be fed to a `VideoDecompressor` instance. + * + * NOTE: This provider emulates the following command to generate encoded payload + * ```` + * ffmpeg -f lavfi -i testsrc=size=x:rate= \ + * -vframes -c:v -b:v -g 10 \ + * -pix_fmt yuv420p -f - + * ``` + *` + * Options' description + * -f lavfi -i testsrc=…= + * Creates a synthetic video source (the same filter graph the node builds). + * -vframes 20 + * Stop after 20 frames – matches NUM_FRAMES. + * -c:v libx264 + * Use the same encoder (libx264 for H.264). + * Replace with libx265 or libaom-av1 for the other codecs. + * -b:v 400k + * Target bitrate (400 kbps) + * -g 10 + * GOP size (10). + * -pix_fmt yuv420p + * Same pixel format as the node. + * -f h264 / h265 / ivf + * Output format – raw bitstream (no container). + * output. + * Destination file (or - to pipe to stdout). + */ +template +class FfmpegTestDataProvider +{ +public: + static constexpr int NUM_FRAMES = 20; + static constexpr int WIDTH = 2880; + static constexpr int HEIGHT = 1860; + static constexpr int FPS = 30; + static constexpr int BITRATE = 400000; // 400 kbps + + explicit FfmpegTestDataProvider() + { + // Embed frame count into the frame contents + // NOTE: Large counter shown on the right middle depicts second of the video + // NOTE: To embed frame count into the pixel value (so that we can confirm the count is as + // expected programatically), + // `std::string filter_descr = "geq=lum='mod(N*10, 255)',format=pix_fmts=yuv420p";` + // is another option. This increases pixel brightness by 10 (back to 0 if the frame number + // reaches 255) frame by frame + + std::string filter_descr = + "drawtext=text='%{n}':fontsize=150:fontcolor=white:x=100:y=100,format=pix_fmts=yuv420p"; + + AVFilterGraph * graph = avfilter_graph_alloc(); + AVFilterContext * src_ctx = nullptr; + AVFilterContext * sink_ctx = nullptr; + + const AVFilter * src = avfilter_get_by_name("testsrc"); + const AVFilter * sink = avfilter_get_by_name("buffersink"); + AVFilterInOut * inputs = avfilter_inout_alloc(); + AVFilterInOut * outputs = avfilter_inout_alloc(); + + av_opt_set_int(src_ctx, "sample_aspect_ratio", 0, 0); + av_opt_set_int(sink_ctx, "sample_aspect_ratio", 0, 0); + + // avfilter_graph_create_filter(&src_ctx, src, "src", filter_descr.c_str(), nullptr, graph); + std::string src_filter_descr = "size=" + std::to_string(WIDTH) + "x" + std::to_string(HEIGHT) + + ":rate=" + std::to_string(FPS); + avfilter_graph_create_filter(&src_ctx, src, "src", src_filter_descr.c_str(), nullptr, graph); + avfilter_graph_create_filter(&sink_ctx, sink, "sink", nullptr, nullptr, graph); + + outputs->name = av_strdup("in"); + outputs->filter_ctx = src_ctx; + outputs->pad_idx = 0; + outputs->next = nullptr; + + inputs->name = av_strdup("out"); + inputs->filter_ctx = sink_ctx; + inputs->pad_idx = 0; + inputs->next = nullptr; + + // av_log_set_level(AV_LOG_DEBUG); + + if (avfilter_graph_parse_ptr(graph, filter_descr.c_str(), &inputs, &outputs, nullptr) < 0) { + throw std::runtime_error("Failed to parse filter graph"); + } + if (avfilter_graph_config(graph, nullptr) < 0) { + throw std::runtime_error("Failed to configure filter graph"); + } + + // 2. Find encoder + const AVCodec * codec = avcodec_find_encoder_by_name(CodecName::value); + if (!codec) { + throw std::runtime_error("Codec not found"); + } + encoder_ctx_ = avcodec_alloc_context3(codec); + encoder_ctx_->width = WIDTH; + encoder_ctx_->height = HEIGHT; + encoder_ctx_->time_base = AVRational{1, FPS}; + encoder_ctx_->framerate = AVRational{FPS, 1}; + encoder_ctx_->gop_size = 10; + encoder_ctx_->max_b_frames = 0; + encoder_ctx_->pix_fmt = AV_PIX_FMT_YUV420P; + encoder_ctx_->bit_rate = BITRATE; + + // Enable encoder acceleration options to reduce test duration + if (std::string(CodecName::value) == "libx264" || std::string(CodecName::value) == "libx265") { + av_opt_set(encoder_ctx_->priv_data, "tune", "zerolatency", 0); + av_opt_set(encoder_ctx_->priv_data, "preset", "ultrafast", 0); + } else if (std::string(CodecName::value) == "libaom-av1") { + // accelerated options for AV1 + // Without these, the test cases for AV1 takes over 2min, which causes colcon test timeout + // cpu-used: It can specify 0--8. larger value lower compression (8 is fastest) + av_opt_set(encoder_ctx_->priv_data, "cpu-used", "8", 0); + // usage: set `realtime` to minimize delay and processing time + av_opt_set(encoder_ctx_->priv_data, "usage", "realtime", 0); + } + + if (avcodec_open2(encoder_ctx_, codec, nullptr) < 0) { + throw std::runtime_error("Could not open encoder"); + } + + // 3. Prepare packet buffer + pkt_ = av_packet_alloc(); + frame_ = av_frame_alloc(); + frame_->format = encoder_ctx_->pix_fmt; + frame_->width = encoder_ctx_->width; + frame_->height = encoder_ctx_->height; + av_frame_get_buffer(frame_, 32); + + // 4. Store graph for later use + graph_ = graph; + src_ctx_ = src_ctx; + sink_ctx_ = sink_ctx; + } + + ~FfmpegTestDataProvider() + { + av_packet_free(&pkt_); + av_frame_free(&frame_); + avcodec_free_context(&encoder_ctx_); + avfilter_graph_free(&graph_); + } + + /** + * @brief Return the next encoded frame as a `common::Image`. + * + * The returned image contains the raw packet data in `data`. The + * `format` field is set to `JPEG` (or whatever the codec produces) + * so that the decompressor can recognise it. + */ + std::optional next() + { + while (true) { + // Try to get encoded packet from encoder + int ret = avcodec_receive_packet(encoder_ctx_, pkt_); + + if (ret == 0) { + // Packet acquired successfully. Break the loop and return it + break; + } else if (ret == AVERROR(EAGAIN)) { + // Encoder requires some frames to output actual encoded data. This error indicates the + // encoder still needs more frame to start output encoded payload + + // If we already sent all frames to be tested, + // send empty frame (nullptr) to notify encoder to finish the execution (flush) + if (frame_index_ >= NUM_FRAMES) { + avcodec_send_frame(encoder_ctx_, nullptr); + continue; // Try to receive packet again + } + + // draw a new frame from filter graph + if (av_buffersink_get_frame(sink_ctx_, frame_) < 0) { + return std::nullopt; // Exit if a new frame is unable to draw + } + + // Set pts + frame_->pts = frame_index_; + + // Send the frame to the encoder + if (avcodec_send_frame(encoder_ctx_, frame_) < 0) { + av_frame_unref(frame_); // release the frame if it fails to send + return std::nullopt; + } + + // Once the sending is finished, release the frame to prepare next try + av_frame_unref(frame_); + + // Go next frame + frame_index_++; + } else if (ret == AVERROR_EOF) { + // Encode finished successfully + return std::nullopt; + } else { + // Unexpected behavior + throw std::runtime_error("Unexpected state found"); + } + } + + // Build the Image + common::Image img; + img.frame_id = "ffmpeg_test"; + img.timestamp = 123456789 + (frame_index_ - 1) * 1000000 / FPS; // 1s per frame + img.width = WIDTH; + img.height = HEIGHT; + img.step = 0; // not relevant for compressed data + img.format = CodecName::format; // placeholder, adjust if needed + img.pts = frame_index_ - 1; + img.flags = (pkt_->flags & AV_PKT_FLAG_KEY) != 0; + img.data.resize(pkt_->size); + std::memcpy(img.data.data(), pkt_->data, pkt_->size); + + // Clean up packet for next use + av_packet_unref(pkt_); + + return img; + } + +private: + AVFilterGraph * graph_{nullptr}; + AVFilterContext * src_ctx_{nullptr}; + AVFilterContext * sink_ctx_{nullptr}; + + AVCodecContext * encoder_ctx_{nullptr}; + AVFrame * frame_{nullptr}; + AVPacket * pkt_{nullptr}; + + int frame_index_{0}; +}; + +/** + * Helper type aliases for the three codecs. + */ +struct H264Provider +{ + static constexpr const char * value = "libx264"; + static constexpr const common::ImageFormat format = common::ImageFormat::H264; +}; +struct H265Provider +{ + static constexpr const char * value = "libx265"; + static constexpr const common::ImageFormat format = common::ImageFormat::H265; +}; +struct AV1Provider +{ + static constexpr const char * value = "libaom-av1"; + static constexpr const common::ImageFormat format = common::ImageFormat::AV1; +}; + +} // namespace accelerated_image_processor::decompression From 9ebf3f1b550b7be15cdf4d32f2b15c005063ee08 Mon Sep 17 00:00:00 2001 From: Manato HIRABAYASHI Date: Sun, 22 Feb 2026 22:18:27 +0900 Subject: [PATCH 4/9] feat: extend the test cases of accelerated_image_processor_ros to cover decompression related functions Signed-off-by: Manato HIRABAYASHI --- .../test/conversion.cpp | 61 +++++++++++++++++++ 1 file changed, 61 insertions(+) diff --git a/src/accelerated_image_processor_ros/test/conversion.cpp b/src/accelerated_image_processor_ros/test/conversion.cpp index 85fca45..c07eb70 100644 --- a/src/accelerated_image_processor_ros/test/conversion.cpp +++ b/src/accelerated_image_processor_ros/test/conversion.cpp @@ -503,6 +503,67 @@ TEST(TestConversionToRosRoi, CopyFields) EXPECT_EQ(result.width, roi.width); EXPECT_EQ(result.do_rectify, roi.do_rectify); } + +TEST(TestSplitStringByCommaAndSemicolon, Simple) +{ + auto result = split_string_by_comma_and_semicolon("foo,bar;baz"); + std::vector expected{"foo", "bar", "baz"}; + EXPECT_EQ(result, expected); +} + +TEST(TestSplitStringByCommaAndSemicolon, NoDelimiter) +{ + auto result = split_string_by_comma_and_semicolon("single"); + std::vector expected{"single"}; + EXPECT_EQ(result, expected); +} + +// Test from_ros_ffmpeg_encoding +TEST(TestFromRosFfmpegEncoding, Simple) +{ + EXPECT_EQ(from_ros_ffmpeg_encoding("h264"), common::ImageFormat::H264); + EXPECT_EQ(from_ros_ffmpeg_encoding("hevc"), common::ImageFormat::H265); + EXPECT_EQ(from_ros_ffmpeg_encoding("av1"), common::ImageFormat::AV1); +} + +TEST(TestFromRosFfmpegEncoding, CommaAndSemicolon) +{ + EXPECT_EQ(from_ros_ffmpeg_encoding("h264,foo"), common::ImageFormat::H264); + EXPECT_EQ(from_ros_ffmpeg_encoding("hevc;bar"), common::ImageFormat::H265); + EXPECT_EQ(from_ros_ffmpeg_encoding("av1,baz"), common::ImageFormat::AV1); +} + +TEST(TestFromRosFfmpegEncoding, UnsupportedThrows) +{ + EXPECT_THROW(from_ros_ffmpeg_encoding("foo"), std::runtime_error); +} + +// Test from_ros_ffmpeg +TEST(TestFromRosFfmpeg, ConvertMessage) +{ + ffmpeg_image_transport_msgs::msg::FFMPEGPacket pkt; + pkt.header.frame_id = "frame_id"; + pkt.header.stamp.sec = 1; + pkt.header.stamp.nanosec = 2; + pkt.width = 640; + pkt.height = 480; + pkt.encoding = "h264"; + pkt.pts = 12345678; + pkt.flags = 7u; + pkt.is_bigendian = true; + pkt.data = std::vector{1, 2, 3, 4, 5, 6}; + + auto img = from_ros_ffmpeg(pkt); + + EXPECT_EQ(img.frame_id, pkt.header.frame_id); + EXPECT_EQ(img.timestamp, 1000000000LL + 2LL); + EXPECT_EQ(img.width, pkt.width); + EXPECT_EQ(img.height, pkt.height); + EXPECT_EQ(img.format, common::ImageFormat::H264); + EXPECT_EQ(img.flags.value(), 7u); + EXPECT_EQ(img.is_bigendian, true); + EXPECT_EQ(img.data, pkt.data); +} } // namespace accelerated_image_processor::ros int main(int argc, char ** argv) From 0521e3a3c357f439bb0a4e833332ba4106014fd5 Mon Sep 17 00:00:00 2001 From: Manato HIRABAYASHI Date: Tue, 24 Feb 2026 10:50:37 +0900 Subject: [PATCH 5/9] build: specify the ffmpeg-related libraries' dependency clearly Signed-off-by: Manato HIRABAYASHI --- src/accelerated_image_processor_decompression/package.xml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/accelerated_image_processor_decompression/package.xml b/src/accelerated_image_processor_decompression/package.xml index ab252e8..0d1d642 100644 --- a/src/accelerated_image_processor_decompression/package.xml +++ b/src/accelerated_image_processor_decompression/package.xml @@ -11,9 +11,11 @@ accelerated_image_processor_common libavcodec-dev + libavutil-dev ament_lint_auto ament_lint_common + libavfilter-dev ament_cmake From 1f8aae3980fc63b43100379066a3a3c495660b64 Mon Sep 17 00:00:00 2001 From: Manato HIRABAYASHI Date: Tue, 24 Feb 2026 13:05:44 +0900 Subject: [PATCH 6/9] fix: update decompressor design to return the decoded frame Signed-off-by: Manato HIRABAYASHI --- .../src/video_decompressor/ffmpeg.cpp | 9 +++++++-- .../test/ffmpeg_video_decompressor.cpp | 6 ++---- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/src/accelerated_image_processor_decompression/src/video_decompressor/ffmpeg.cpp b/src/accelerated_image_processor_decompression/src/video_decompressor/ffmpeg.cpp index dbdef38..d92d683 100644 --- a/src/accelerated_image_processor_decompression/src/video_decompressor/ffmpeg.cpp +++ b/src/accelerated_image_processor_decompression/src/video_decompressor/ffmpeg.cpp @@ -91,8 +91,13 @@ class FfmpegVideoDecompressor final : public VideoDecompressor this->postprocess(processed); } - // always returns nullopt because returned value will not be consumed - return std::nullopt; + // For streaming cases, although one ffmpeg packet MAY contain multiple frames, it typically + // contains one frame. For this reason, here returns the first element of the decoded results. + if (processed_vec.empty()) { + return std::nullopt; + } else { + return processed_vec[0]; + } } private: diff --git a/src/accelerated_image_processor_decompression/test/ffmpeg_video_decompressor.cpp b/src/accelerated_image_processor_decompression/test/ffmpeg_video_decompressor.cpp index 425e397..19874b0 100644 --- a/src/accelerated_image_processor_decompression/test/ffmpeg_video_decompressor.cpp +++ b/src/accelerated_image_processor_decompression/test/ffmpeg_video_decompressor.cpp @@ -32,10 +32,8 @@ namespace accelerated_image_processor::decompression * @brief a free function to check the decoded result * * In FFmpeg, a single packet can contain multiple frames. The ffmpeg_video_decompressor - * extracts each frame one by one and forwards it to postprocess(). - * Because of this, process() never returns a frame directly – it - * always returns std::nullopt. This function is therefore registered - * as a callback to handle decoded frames instead of producing them. + * extracts each frame one by one and feeds it to postprocess(). + * This function is registered as a callback to check the every decoded frames. */ template void check_ffmpeg_video_decompressor_result(const common::Image & decoded) From dbcef775407b074ad11f12e1c2b6dee7df8b78a9 Mon Sep 17 00:00:00 2001 From: Manato HIRABAYASHI Date: Fri, 27 Feb 2026 17:05:14 +0900 Subject: [PATCH 7/9] fix: correct typos Signed-off-by: Manato HIRABAYASHI --- .../video_decompressor.hpp | 4 ++-- .../src/video_decompressor/ffmpeg.cpp | 14 +++++++------- .../accelerated_image_processor_ros/qos.hpp | 2 +- .../launch/decompress.launch.xml | 1 - .../src/decompress_node.cpp | 2 +- .../src/decompress_node.hpp | 4 +--- 6 files changed, 12 insertions(+), 15 deletions(-) diff --git a/src/accelerated_image_processor_decompression/include/accelerated_image_processor_decompression/video_decompressor.hpp b/src/accelerated_image_processor_decompression/include/accelerated_image_processor_decompression/video_decompressor.hpp index 42a328d..e3e6a66 100644 --- a/src/accelerated_image_processor_decompression/include/accelerated_image_processor_decompression/video_decompressor.hpp +++ b/src/accelerated_image_processor_decompression/include/accelerated_image_processor_decompression/video_decompressor.hpp @@ -30,7 +30,7 @@ class FfmpegVideoDecompressor; enum class VideoBackend : uint8_t { FFMPEG }; /** - * @brief Abstract base class for Jetson Video compressors. + * @brief Abstract base class for video decompressors. */ class VideoDecompressor : public common::BaseProcessor { @@ -48,7 +48,7 @@ class VideoDecompressor : public common::BaseProcessor VideoBackend backend() const { return backend_; } private: - const VideoBackend backend_; //!< Compression backend type. + const VideoBackend backend_; //!< Decompression backend type. }; //!< @brief Factory function to create a FfmpegVideoDecompressor. std::unique_ptr make_ffmpeg_video_decompressor(); diff --git a/src/accelerated_image_processor_decompression/src/video_decompressor/ffmpeg.cpp b/src/accelerated_image_processor_decompression/src/video_decompressor/ffmpeg.cpp index d92d683..934b41b 100644 --- a/src/accelerated_image_processor_decompression/src/video_decompressor/ffmpeg.cpp +++ b/src/accelerated_image_processor_decompression/src/video_decompressor/ffmpeg.cpp @@ -132,7 +132,7 @@ class FfmpegVideoDecompressor final : public VideoDecompressor case common::ImageFormat::AV1: return "av1"; default: - throw std::runtime_error("Unsuported format was detected"); + throw std::runtime_error("Unsupported format was detected"); } } @@ -147,7 +147,7 @@ class FfmpegVideoDecompressor final : public VideoDecompressor } // Inject the CUDA Stream - // safely modify the specific HW context settings nwo because initialization hasn't happened + // safely modify the specific HW context settings now because initialization hasn't happened // yet { // Extract CUcontext from cuda stream using CUDA driver API @@ -202,23 +202,23 @@ class FfmpegVideoDecompressor final : public VideoDecompressor // Allocate packet wrapper packet_ = av_packet_alloc(); if (!packet_) { - return DecoderInitResult(false, "Failed to allcate AVPacket"); + return DecoderInitResult(false, "Failed to allocate AVPacket"); } // Allocate region to store the decoded result decoded_frame_ = av_frame_alloc(); if (!decoded_frame_) { - return DecoderInitResult(false, "Failed to allcate AVFrame"); + return DecoderInitResult(false, "Failed to allocate AVFrame"); } return DecoderInitResult(true, ""); } /** - * @brief core process implementation that handles input image (ffmpeg packat) and returns the + * @brief core process implementation that handles input image (ffmpeg packet) and returns the * vector of Image * - * Because one ffmpeg packet may include multiple frames, process_imple, which is the pure virtual + * Because one ffmpeg packet may include multiple frames, process_impl, which is the pure virtual * function that returns one image as a decoding result, can not be applicable for this class. */ std::vector process_packet(const common::Image & image) @@ -277,7 +277,7 @@ class FfmpegVideoDecompressor final : public VideoDecompressor // Release references explicitly // Although receive_frame unrefs at the start, we manually unref here // to return the GPU surface to the pool IMMEDIATELY. - // If we wait until the next callback, we might starve teh decoder's surface pool + // If we wait until the next callback, we might starve the decoder's surface pool av_frame_unref(decoded_frame_); } diff --git a/src/accelerated_image_processor_ros/include/accelerated_image_processor_ros/qos.hpp b/src/accelerated_image_processor_ros/include/accelerated_image_processor_ros/qos.hpp index 3f2e1d7..083c8c5 100644 --- a/src/accelerated_image_processor_ros/include/accelerated_image_processor_ros/qos.hpp +++ b/src/accelerated_image_processor_ros/include/accelerated_image_processor_ros/qos.hpp @@ -51,7 +51,7 @@ bool find_qos( * @param node The ROS node to use for querying the topic. * @param topic_name The name of the topic to query. * @param throttle_period_ms The period in milliseconds to throttle the logging. - * @return std::string The type for the topic, or std::nullopt if no + * @return std::string The type for the topic, or an empty string if no * publishers are found or if multiple publishers are found. */ std::string find_topic_type( diff --git a/src/accelerated_image_processor_ros/launch/decompress.launch.xml b/src/accelerated_image_processor_ros/launch/decompress.launch.xml index bfcd6b6..61cfda2 100644 --- a/src/accelerated_image_processor_ros/launch/decompress.launch.xml +++ b/src/accelerated_image_processor_ros/launch/decompress.launch.xml @@ -8,7 +8,6 @@ - diff --git a/src/accelerated_image_processor_ros/src/decompress_node.cpp b/src/accelerated_image_processor_ros/src/decompress_node.cpp index 8daf27e..0c10b48 100644 --- a/src/accelerated_image_processor_ros/src/decompress_node.cpp +++ b/src/accelerated_image_processor_ros/src/decompress_node.cpp @@ -84,7 +84,7 @@ void DecompressNode::determine_qos(const int max_task_length) decompression_worker_.emplace(max_task_length); - // once all queries received, stop the timer callback1 + // once all queries received, stop the timer callback qos_request_timer_->cancel(); } diff --git a/src/accelerated_image_processor_ros/src/decompress_node.hpp b/src/accelerated_image_processor_ros/src/decompress_node.hpp index 7dcd8f5..bc9a4a4 100644 --- a/src/accelerated_image_processor_ros/src/decompress_node.hpp +++ b/src/accelerated_image_processor_ros/src/decompress_node.hpp @@ -50,7 +50,7 @@ class DecompressNode : public rclcpp::Node /** * @brief Callback function for publishing decompressed images. - * @param image The decompressed image compressed and to be published. + * @param image The decompressed image to be published. */ void publish_decompressed(const common::Image & image); @@ -65,7 +65,5 @@ class DecompressNode : public rclcpp::Node rclcpp::TimerBase::SharedPtr qos_request_timer_; std::optional decompression_worker_; - - bool use_jpeg_compression_; }; } // namespace accelerated_image_processor::ros From 04ffdc23080753bb0db25bc1a82fa386ec46c783 Mon Sep 17 00:00:00 2001 From: Manato HIRABAYASHI Date: Fri, 27 Feb 2026 17:05:54 +0900 Subject: [PATCH 8/9] fix: release allocated resource on destructor or before throwing exception Signed-off-by: Manato HIRABAYASHI --- .../src/video_decompressor/ffmpeg.cpp | 11 ++++++++++ .../test/test_utility.hpp | 22 +++++++++++++++++++ 2 files changed, 33 insertions(+) diff --git a/src/accelerated_image_processor_decompression/src/video_decompressor/ffmpeg.cpp b/src/accelerated_image_processor_decompression/src/video_decompressor/ffmpeg.cpp index 934b41b..07c3e91 100644 --- a/src/accelerated_image_processor_decompression/src/video_decompressor/ffmpeg.cpp +++ b/src/accelerated_image_processor_decompression/src/video_decompressor/ffmpeg.cpp @@ -77,6 +77,7 @@ class FfmpegVideoDecompressor final : public VideoDecompressor if (dst_bgr_dev_) { nppiFree(dst_bgr_dev_); } + CHECK_CUDA(cudaStreamDestroy(stream_)); } /** @@ -114,6 +115,9 @@ class FfmpegVideoDecompressor final : public VideoDecompressor if (packet_) { av_packet_free(&packet_); } + if (decoded_frame_) { + av_frame_free(&decoded_frame_); + } if (codec_ctx_) { avcodec_free_context(&codec_ctx_); } @@ -143,6 +147,7 @@ class FfmpegVideoDecompressor final : public VideoDecompressor // TODO(manato): expose device type as a parameter would beneficial especially for non-CUDA user hw_device_ctx_ = av_hwdevice_ctx_alloc(AV_HWDEVICE_TYPE_CUDA); if (!hw_device_ctx_) { + cleanup_decoder(); return DecoderInitResult(false, "Failed to allocate CUDA HW device context"); } @@ -153,6 +158,7 @@ class FfmpegVideoDecompressor final : public VideoDecompressor // Extract CUcontext from cuda stream using CUDA driver API CUcontext cu_context; if (cuStreamGetCtx(reinterpret_cast(stream_), &cu_context) != CUDA_SUCCESS) { + cleanup_decoder(); return DecoderInitResult(false, "Extracting CUcontext failed"); } AVHWDeviceContext * av_hw_device_ctx = @@ -168,6 +174,7 @@ class FfmpegVideoDecompressor final : public VideoDecompressor // but will respect the stream we already assigned int err = av_hwdevice_ctx_init(hw_device_ctx_); if (err < 0) { + cleanup_decoder(); char err_buf[128]; av_strerror(err, err_buf, sizeof(err_buf)); return DecoderInitResult(false, std::string("Failed to init HW device context: ") + err_buf); @@ -177,6 +184,7 @@ class FfmpegVideoDecompressor final : public VideoDecompressor std::string codec_name = image_format_to_string(fmt); const AVCodec * codec = avcodec_find_decoder_by_name(codec_name.c_str()); if (!codec) { + cleanup_decoder(); return DecoderInitResult(false, "Codec not found: " + codec_name); } @@ -196,18 +204,21 @@ class FfmpegVideoDecompressor final : public VideoDecompressor // Open Codec if (avcodec_open2(codec_ctx_, codec, nullptr) < 0) { + cleanup_decoder(); return DecoderInitResult(false, "Failed to open codec"); } // Allocate packet wrapper packet_ = av_packet_alloc(); if (!packet_) { + cleanup_decoder(); return DecoderInitResult(false, "Failed to allocate AVPacket"); } // Allocate region to store the decoded result decoded_frame_ = av_frame_alloc(); if (!decoded_frame_) { + cleanup_decoder(); return DecoderInitResult(false, "Failed to allocate AVFrame"); } diff --git a/src/accelerated_image_processor_decompression/test/test_utility.hpp b/src/accelerated_image_processor_decompression/test/test_utility.hpp index 7c13c63..fc9e265 100644 --- a/src/accelerated_image_processor_decompression/test/test_utility.hpp +++ b/src/accelerated_image_processor_decompression/test/test_utility.hpp @@ -124,18 +124,36 @@ class FfmpegTestDataProvider inputs->pad_idx = 0; inputs->next = nullptr; + auto free_allocated_resources = [&graph, &inputs, &outputs]() { + if (graph) { + avfilter_graph_free(&graph); + } + if (inputs) { + avfilter_inout_free(&inputs); + } + if (outputs) { + avfilter_inout_free(&outputs); + } + }; + // av_log_set_level(AV_LOG_DEBUG); if (avfilter_graph_parse_ptr(graph, filter_descr.c_str(), &inputs, &outputs, nullptr) < 0) { + free_allocated_resources(); throw std::runtime_error("Failed to parse filter graph"); } if (avfilter_graph_config(graph, nullptr) < 0) { + free_allocated_resources(); throw std::runtime_error("Failed to configure filter graph"); } + avfilter_inout_free(&inputs); + avfilter_inout_free(&outputs); + // 2. Find encoder const AVCodec * codec = avcodec_find_encoder_by_name(CodecName::value); if (!codec) { + free_allocated_resources(); throw std::runtime_error("Codec not found"); } encoder_ctx_ = avcodec_alloc_context3(codec); @@ -162,6 +180,10 @@ class FfmpegTestDataProvider } if (avcodec_open2(encoder_ctx_, codec, nullptr) < 0) { + free_allocated_resources(); + if (encoder_ctx_) { + avcodec_free_context(&encoder_ctx_); + } throw std::runtime_error("Could not open encoder"); } From 920dbf438b0b537fdca109e43e2a6499ef5aa639 Mon Sep 17 00:00:00 2001 From: Manato HIRABAYASHI Date: Fri, 27 Feb 2026 17:21:37 +0900 Subject: [PATCH 9/9] docs: add README of accelerated_image_processor_decompression package Signed-off-by: Manato HIRABAYASHI --- .../README.md | 79 +++++++++++++++++++ 1 file changed, 79 insertions(+) create mode 100644 src/accelerated_image_processor_decompression/README.md diff --git a/src/accelerated_image_processor_decompression/README.md b/src/accelerated_image_processor_decompression/README.md new file mode 100644 index 0000000..a63e5b6 --- /dev/null +++ b/src/accelerated_image_processor_decompression/README.md @@ -0,0 +1,79 @@ +# accelerated_image_processor_decompression + +This package provides decompression functionalities for compressed video streams +using a CUDA‑accelerated FFmpeg backend. It can be used on Jetson devices (GPU) as well as on a generic CUDA‑capable +platform. + +## Decompressor Supports + +| Decompressor | Format | Backend | Device | +| ------------------------- | ----------------------- | ------- | ------ | +| `FfmpegVideoDecompressor` | `VIDEO` (H264/H265/AV1) | FFmpeg | GPU | + +> **Note** +> _Only a single hardware‑accelerated backend is currently supported:_ +> _`FfmpegVideoDecompressor` uses FFmpeg + NPP to decode the video and convert +> the decoded frames to RGB (`BGR` on the GPU). The decompressor accepts a +> `common::Image` whose `format` field indicates the encoded format +> (`H264`, `H265`, or `AV1`). It outputs a `common::Image` with +> `format` set to `RAW`/`BGR`._ + +## Example Usage in ROS 2 + +Below is a minimal example of how to use the decompressor in a ROS 2 node. +It mirrors the example in the _compression_ package, but uses the +`decompression::create_decompressor` factory. + +```c++ +#include +#include +#include + +using namespace accelerated_image_processor; + +class SomeNode final : public rclcpp::Node +{ +public: + explicit SomeNode(const rclcpp::NodeOptions & options) + : Node("some_node", options) + { + // Choose decompression type + decompression::DecompressionType type = decompression::DecompressionType::VIDEO; + decompressor_ = decompression::create_decompressor(type, this); + + // Update parameters of the decompressor + for (auto & [name, value] : decompressor_->parameters()) { + std::visit([&](auto & v) { + using T = std::decay_t; + v = this->declare_parameter(name, v); + }, value); + } + + // Subscription to compressed stream and publisher for raw image + subscription_ = this->create_subscription( + "~/input/image/compressed", 10, + [this](const ffmpeg_image_transport_msgs::msg::FFMPEGPacket::ConstSharedPtr msg) + { this->callback(msg); }); + publisher_ = this->create_publisher("~/output/image", 10); + } + +private: + void callback(const ffmpeg_image_transport_msgs::msg::FFMPEGPacket::ConstSharedPtr msg) + { + common::Image image; + // Convert ROS message → accelerated_image_processor::common::Image … + decompressor_->process(image); + } + + void publish(const common::Image & image) + { + sensor_msgs::msg::Image msg; + // Convert accelerated_image_processor::common::Image → ROS message … + publisher_->publish(msg); + } + + std::unique_ptr decompressor_; + rclcpp::Subscription::SharedPtr subscription_; + rclcpp::Publisher::SharedPtr publisher_; +}; +```