diff --git a/src/accelerated_image_processor_common/include/accelerated_image_processor_common/datatype.hpp b/src/accelerated_image_processor_common/include/accelerated_image_processor_common/datatype.hpp index 0dc9b45..6b26f02 100644 --- a/src/accelerated_image_processor_common/include/accelerated_image_processor_common/datatype.hpp +++ b/src/accelerated_image_processor_common/include/accelerated_image_processor_common/datatype.hpp @@ -17,6 +17,7 @@ #include #include #include +#include #include #include @@ -30,7 +31,7 @@ enum class ImageEncoding : uint8_t { RGB, BGR }; /** * @brief Enumeration of image compression formats. */ -enum class ImageFormat : uint8_t { RAW, JPEG, PNG }; +enum class ImageFormat : uint8_t { RAW, JPEG, PNG, H264, H265, AV1 }; /** * @brief Structure representing an image. @@ -41,10 +42,17 @@ struct Image int64_t timestamp; //!< Timestamp at the image is captured uint32_t height; //!< Image height, that is, number of rows uint32_t width; //!< Image width, that is, number of columns - uint32_t step; //!< Full row length in bytes - ImageEncoding encoding; //!< Image color encoding ImageFormat format; //!< Image compression format std::vector data; //!< Actual matrix data + bool is_bigendian; //!< true if machine stores in big endian format + + // Image / CompressedImage dedicated fields + uint32_t step; //!< Full row length in bytes + ImageEncoding encoding; //!< Image color encoding + + // Video frame dedicated fields + std::optional pts; //!< packet time stamp + std::optional flags; //!< flag representing whether this is the key frame /** * @brief Check the specified image is valid. diff --git a/src/accelerated_image_processor_common/include/accelerated_image_processor_common/helper.hpp b/src/accelerated_image_processor_common/include/accelerated_image_processor_common/helper.hpp index 1f581e7..902f363 100644 --- a/src/accelerated_image_processor_common/include/accelerated_image_processor_common/helper.hpp +++ b/src/accelerated_image_processor_common/include/accelerated_image_processor_common/helper.hpp @@ -16,6 +16,7 @@ #include #include +#include namespace accelerated_image_processor::common { @@ -67,4 +68,23 @@ namespace accelerated_image_processor::common exit(1); \ } \ } + +/** + * @brief Macro to check for errors and print a message if the VPI status is not + * VPI_SUCCESS + * + * @param status The VPI status. + */ +#define CHECK_VPI(call) \ + { \ + VPIStatus _e = (call); \ + if (_e != VPI_SUCCESS) { \ + char msg_buf[VPI_MAX_STATUS_MESSAGE_LENGTH]; \ + vpiGetLastStatusMessage(msg_buf, VPI_MAX_STATUS_MESSAGE_LENGTH); \ + std::cerr << "VPI failure: \'#" << _e << "\' at " << __FILE__ << ":" << __LINE__ \ + << ", (reason: " << std::string(msg_buf) << ")" << std::endl; \ + exit(1); \ + } \ + } + } // namespace accelerated_image_processor::common diff --git a/src/accelerated_image_processor_common/include/accelerated_image_processor_common/processor.hpp b/src/accelerated_image_processor_common/include/accelerated_image_processor_common/processor.hpp index d171286..4fdee98 100644 --- a/src/accelerated_image_processor_common/include/accelerated_image_processor_common/processor.hpp +++ b/src/accelerated_image_processor_common/include/accelerated_image_processor_common/processor.hpp @@ -97,7 +97,7 @@ class BaseProcessor * @param image The input image. * @return The processed image if the process is successful, otherwise std::nullopt. */ - std::optional process(const Image & image) + virtual std::optional process(const Image & image) { if (!is_ready()) { // TODO(ktro2828): Update to return a type that describes if the process success or not @@ -113,6 +113,17 @@ class BaseProcessor return std::nullopt; } + postprocess(processed); + + return processed; + }; + + /** + * @brief Execute post process + * @param processed The image to be post-processed + */ + void postprocess(Image & processed) + { std::visit( [&processed](auto & f) { using T = std::decay_t; @@ -125,9 +136,7 @@ class BaseProcessor } }, storage_); - - return processed; - }; + } /** * @brief Check the processor is ready to run processing. diff --git a/src/accelerated_image_processor_compression/CMakeLists.txt b/src/accelerated_image_processor_compression/CMakeLists.txt index 8926058..ca19ea5 100644 --- a/src/accelerated_image_processor_compression/CMakeLists.txt +++ b/src/accelerated_image_processor_compression/CMakeLists.txt @@ -45,6 +45,18 @@ if(NOT JETSON_FOUND message(FATAL_ERROR "No JPEG encoder found") endif() +# --- CUDA VPI available? --- +find_package(VPI) +if(${VPI_FOUND}) + message("VPI found") +endif() + +# --- Jetson multimedia API available? --- +find_package(NVMMAPI) +if(${NVMMAPI_FOUND}) + message("Jetson multimedia API found") +endif() + add_library(${PROJECT_NAME} SHARED src/builder.cpp src/jpeg_compressor/cpu.cpp src/jpeg_compressor/nvjpeg.cpp) @@ -76,12 +88,22 @@ if(JETSON_FOUND) PRIVATE /usr/src/jetson_multimedia_api/include) add_library( ${PROJECT_NAME}_jetson SHARED + # JPEG compression src/jpeg_compressor/jetson.cpp /usr/src/jetson_multimedia_api/samples/common/classes/NvBuffer.cpp /usr/src/jetson_multimedia_api/samples/common/classes/NvElement.cpp /usr/src/jetson_multimedia_api/samples/common/classes/NvElementProfiler.cpp /usr/src/jetson_multimedia_api/samples/common/classes/NvJpegEncoder.cpp - /usr/src/jetson_multimedia_api/samples/common/classes/NvLogging.cpp) + /usr/src/jetson_multimedia_api/samples/common/classes/NvLogging.cpp + # video compression + src/video_compressor/jetson.cpp + src/video_compressor/jetson_h264.cpp + src/video_compressor/jetson_h265.cpp + src/video_compressor/jetson_av1.cpp + /usr/src/jetson_multimedia_api/samples/common/classes/NvVideoEncoder.cpp + /usr/src/jetson_multimedia_api/samples/common/classes/NvV4l2Element.cpp + /usr/src/jetson_multimedia_api/samples/common/classes/NvV4l2ElementPlane.cpp + /usr/src/jetson_multimedia_api/samples/common/classes/NvBufSurface.cpp) target_compile_definitions(${PROJECT_NAME}_jetson PRIVATE JETSON_AVAILABLE) ament_target_dependencies(${PROJECT_NAME}_jetson @@ -100,12 +122,14 @@ if(JETSON_FOUND) target_link_libraries( ${PROJECT_NAME}_jetson ${JETSON_NVJPEG_LIB} + ${NVMMAPI_LIBRARIES} $<$:CUDA::cudart> $<$:CUDA::nppc> $<$:CUDA::nppicc> $<$:CUDA::nppidei> $<$:CUDA::nppig> - $<$:CUDA::nppisu>) + $<$:CUDA::nppisu> + $<$:vpi>) set(JETSON_LIB_DIRS "/usr/lib/aarch64-linux-gnu/nvidia;/usr/lib/aarch64-linux-gnu/tegra") @@ -132,11 +156,36 @@ install(DIRECTORY include/${PROJECT_NAME} DESTINATION include) if(BUILD_TESTING) set(test_files test/builder.cpp test/cpu_jpeg_compressor.cpp test/jetson_jpeg_compressor.cpp test/nv_jpeg_compressor.cpp) + foreach(test_file IN LISTS test_files) get_filename_component(test_file_name ${test_file} NAME) ament_auto_add_gtest(${test_file_name}_${PROJECT_NAME} ${test_file}) target_link_libraries(${test_file_name}_${PROJECT_NAME} ${PROJECT_NAME}) endforeach() + + # Treat some tests separately because they contains many test cases and cause + # colcon test timeout if the tests are generated in the above way + set(long_test_files + test/jetson_video_compressor_h264.cpp + test/jetson_video_compressor_h265.cpp + test/jetson_video_compressor_av1.cpp) + + find_package(GTest) + enable_testing() + include(GoogleTest) # enable the GoogleTest integration + + foreach(test_file IN LISTS long_test_files) + get_filename_component(test_file_name ${test_file} NAME) + + set(LONG_TEST_EXEC ${test_file_name}_${PROJECT_NAME}) + add_executable(${LONG_TEST_EXEC} ${test_file}) + target_include_directories(${LONG_TEST_EXEC} + PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include) + target_link_libraries(${LONG_TEST_EXEC} ${PROJECT_NAME} GTest::GTest + GTest::Main) + # divide a single test into separated cases + gtest_discover_tests(${LONG_TEST_EXEC}) + endforeach() endif() ament_export_include_directories(include) diff --git a/src/accelerated_image_processor_compression/README.md b/src/accelerated_image_processor_compression/README.md index 4bd110e..e115096 100644 --- a/src/accelerated_image_processor_compression/README.md +++ b/src/accelerated_image_processor_compression/README.md @@ -5,11 +5,14 @@ It also includes support for hardware acceleration on NVIDIA Jetson devices usin ## Compressor Supports -| Compressor | Format | Backend | Device | -| ---------------------- | ------ | ----------------------------------------------------------------------------------- | ------ | -| `JetsonJPEGCompressor` | `JPEG` | [jetsonJPEG](https://docs.nvidia.com/jetson/l4t-multimedia/classNvJPEGEncoder.html) | Jetson | -| `NvJPEGCompressor` | `JPEG` | [nvJPEG](https://developer.nvidia.com/nvjpeg) | GPU | -| `CpuJPEGCompressor` | `JPEG` | [TurboJPEG](https://github.com/libjpeg-turbo/libjpeg-turbo) | CPU | +| Compressor | Format | Backend | Device | +| ---------------------- | ------ | ---------------------------------------------------------------------------------------- | ------ | +| `JetsonJPEGCompressor` | `JPEG` | [jetsonJPEG](https://docs.nvidia.com/jetson/l4t-multimedia/classNvJPEGEncoder.html) | Jetson | +| `NvJPEGCompressor` | `JPEG` | [nvJPEG](https://developer.nvidia.com/nvjpeg) | GPU | +| `CpuJPEGCompressor` | `JPEG` | [TurboJPEG](https://github.com/libjpeg-turbo/libjpeg-turbo) | CPU | +| `JetsonH264Compressor` | `H264` | [NvVideoEncoder](https://docs.nvidia.com/jetson/l4t-multimedia/classNvVideoEncoder.html) | Jetson | +| `JetsonH265Compressor` | `H265` | [NvVideoEncoder](https://docs.nvidia.com/jetson/l4t-multimedia/classNvVideoEncoder.html) | Jetson | +| `JetsonAV1Compressor` | `AV1` | [NvVideoEncoder](https://docs.nvidia.com/jetson/l4t-multimedia/classNvVideoEncoder.html) | Jetson | ## Example Usage in ROS 2 diff --git a/src/accelerated_image_processor_compression/cmake/FindNVMMAPI.cmake b/src/accelerated_image_processor_compression/cmake/FindNVMMAPI.cmake new file mode 100644 index 0000000..eebc0e0 --- /dev/null +++ b/src/accelerated_image_processor_compression/cmake/FindNVMMAPI.cmake @@ -0,0 +1,109 @@ +# ############################################################################## +# +# Original Copyright +# +# ############################################################################## +# Copyright (c) 2017-2022, NVIDIA CORPORATION. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of NVIDIA CORPORATION nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY +# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# - Try to find the NVIDIA Tegra Multimedia API +# Once done this will define +# NVMMAPI_FOUND - System has NVMMAPI +# NVMMAPI_INCLUDE_DIRS - The NVMMAPI include directories +# NVMMAPI_LIBRARIES - The libraries needed to use the NVMMAPI +# NVMMAPI_DEFINITIONS - Compiler switches required for using NVMMAPI +# ############################################################################## +# +# Modifications Copyright +# +# ############################################################################## +# Copyright 2026 TIER IV, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. +# ############################################################################## + +find_package(PkgConfig) + +find_path(NVMMAPI_INCLUDE_DIR NvVideoEncoder.h + HINTS ${SYS_ROOT}/usr/src/jetson_multimedia_api/include) + +find_library( + NVMMAPI_NVV4L2_LIBRARY + NAMES nvv4l2 + HINTS /usr/lib/${CMAKE_LIBRARY_ARCHITECTURE}/tegra) +find_library( + NVMMAPI_NVMEDIA_LIBRARY + NAMES nvmedia + HINTS /usr/lib/${CMAKE_LIBRARY_ARCHITECTURE}/tegra) +find_library( + NVMMAPI_V4L2_NVVIDEOCODEC_LIBRARY + NAMES v4l2_nvvideocodec + HINTS /usr/lib/${CMAKE_LIBRARY_ARCHITECTURE}/tegra) +find_library( + NVMMAPI_NVBUFSURFACE_LIBRARY + NAMES nvbufsurface + HINTS /usr/lib/${CMAKE_LIBRARY_ARCHITECTURE}/tegra) +find_library( + NVMMAPI_NVBUFSURF_TRANSFORM_LIBRARY + NAMES nvbufsurftransform + HINTS /usr/lib/${CMAKE_LIBRARY_ARCHITECTURE}/tegra) + +set(NVMMAPI_LIBRARIES + ${NVMMAPI_NVV4L2_LIBRARY} ${NVMMAPI_NVMEDIA_LIBRARY} + ${NVMMAPI_V4L2_NVVIDEOCODEC_LIBRARY} ${NVMMAPI_NVBUFSURFACE_LIBRARY} + ${NVMMAPI_NVBUFSURF_TRANSFORM_LIBRARY}) + +set(NVMMAPI_INCLUDE_DIRS ${NVMMAPI_INCLUDE_DIR}) +set(NVMMAPI_DEFINITIONS -DNVMMAPI_SUPPORTED) + +include(FindPackageHandleStandardArgs) +# handle the QUIETLY and REQUIRED arguments and set ARGUS_FOUND to TRUE if all +# listed variables are TRUE +find_package_handle_standard_args( + NVMMAPI + DEFAULT_MSG + NVMMAPI_INCLUDE_DIR + NVMMAPI_NVV4L2_LIBRARY + NVMMAPI_NVMEDIA_LIBRARY + NVMMAPI_V4L2_NVVIDEOCODEC_LIBRARY + NVMMAPI_NVBUFSURFACE_LIBRARY + NVMMAPI_NVBUFSURF_TRANSFORM_LIBRARY) + +# mark_as_advanced(NVMMAPI_INCLUDE_DIR NVMMAPI_LIBRARY) +mark_as_advanced( + NVMMAPI_INCLUDE_DIR NVMMAPI_NVV4L2_LIBRARY NVMMAPI_NVMEDIA_LIBRARY + NVMMAPI_V4L2_NVVIDEOCODEC_LIBRARY NVMMAPI_NVBUFSURFACE_LIBRARY + NVMMAPI_NVBUFSURF_TRANSFORM_LIBRARY) diff --git a/src/accelerated_image_processor_compression/include/accelerated_image_processor_compression/builder.hpp b/src/accelerated_image_processor_compression/include/accelerated_image_processor_compression/builder.hpp index 583b728..5d46387 100644 --- a/src/accelerated_image_processor_compression/include/accelerated_image_processor_compression/builder.hpp +++ b/src/accelerated_image_processor_compression/include/accelerated_image_processor_compression/builder.hpp @@ -15,6 +15,7 @@ #pragma once #include +#include #include #include @@ -22,17 +23,6 @@ namespace accelerated_image_processor::compression { -/** - * @brief Type alias for compressor processor. - * @note This might be a specific implementation of a compressor processor in the future. - */ -using Compressor = common::BaseProcessor; - -/** - * @brief Compression type enum - */ -enum class CompressionType : uint8_t { JPEG, VIDEO }; - /** * @brief Convert a string to a compression type * @param str String to convert expected strings ["JPEG", "VIDEO"] @@ -85,7 +75,7 @@ template < inline std::unique_ptr create_compressor(const std::string & type, F fn) { return create_compressor(to_compression_type(type), fn); -}; +} /** * @brief Create a compressor processor with a member function for the postprocess. diff --git a/src/accelerated_image_processor_compression/include/accelerated_image_processor_compression/compressor.hpp b/src/accelerated_image_processor_compression/include/accelerated_image_processor_compression/compressor.hpp new file mode 100644 index 0000000..0c5f7f6 --- /dev/null +++ b/src/accelerated_image_processor_compression/include/accelerated_image_processor_compression/compressor.hpp @@ -0,0 +1,71 @@ +// Copyright 2026 TIER IV, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include +#include + +namespace accelerated_image_processor::compression +{ + +/** + * @enum CompressorBackend + * @brief Compression backend type. + * + * @value JETSON Jetson hardware accelerated backend. + * @value NVJPEG NVIDIA NVJPEG backend. + * @value CPU CPU based backend. + */ +enum class CompressorBackend : uint8_t { JETSON, NVJPEG, CPU }; + +/** + * @brief Compression type enum + */ +enum class CompressionType : uint8_t { JPEG, H264, H265, AV1 }; + +/** + * @brief Base abstract class for image compression processors. + * + * The Compressor class provides a common interface for different compression backends + * such as Jetson hardware accelerated, NVIDIA NVJPEG, and CPU-based implementations. + * It inherits from common::BaseProcessor and stores the selected backend type. + * + * @param backend The compression backend to use. + * @param dedicated_parameters Optional map of parameters specific to the compressor. + * + * @note The class is intended to be subclassed by concrete compressor implementations. + */ +class Compressor : public common::BaseProcessor +{ +public: + explicit Compressor(CompressorBackend backend, common::ParameterMap dedicated_parameters = {}) + : BaseProcessor(dedicated_parameters), backend_(backend) + { + } + + virtual ~Compressor() {} + + /** + * @brief Get the compression backend type. + * + * @return The backend type used by this compressor. + */ + CompressorBackend backend() const { return backend_; } + +private: + const CompressorBackend backend_; //!< Compression backend type +}; + +} // namespace accelerated_image_processor::compression diff --git a/src/accelerated_image_processor_compression/include/accelerated_image_processor_compression/jpeg_compressor.hpp b/src/accelerated_image_processor_compression/include/accelerated_image_processor_compression/jpeg_compressor.hpp index c14a717..55b3b4a 100644 --- a/src/accelerated_image_processor_compression/include/accelerated_image_processor_compression/jpeg_compressor.hpp +++ b/src/accelerated_image_processor_compression/include/accelerated_image_processor_compression/jpeg_compressor.hpp @@ -17,6 +17,7 @@ #include #include #include +#include #include @@ -26,19 +27,14 @@ class JetsonJPEGCompressor; class NvJPEGCompressor; class CpuJPEGCompressor; -/** - * @brief Enumeration of available JPEG compression backends. - */ -enum class JPEGBackend : uint8_t { JETSON, NVJPEG, CPU }; - /** * @brief Abstract base class for JPEG compressors. */ -class JPEGCompressor : public common::BaseProcessor +class JPEGCompressor : public Compressor { public: - explicit JPEGCompressor(JPEGBackend backend, common::ParameterMap dedicated_parameters = {}) - : BaseProcessor(dedicated_parameters += {{"quality", 90}}), backend_(backend) + explicit JPEGCompressor(CompressorBackend backend, common::ParameterMap dedicated_parameters = {}) + : Compressor(backend, dedicated_parameters += {{"quality", 90}}) { } @@ -48,11 +44,6 @@ class JPEGCompressor : public common::BaseProcessor * @brief Return the quality of the JPEG compression. */ int quality() const { return this->parameter_value("quality"); } - - JPEGBackend backend() const { return backend_; } - -private: - const JPEGBackend backend_; //!< Compression backend type. }; //!< @brief Factory function to create a CPUJPEGCompressor. std::unique_ptr make_cpujpeg_compressor(); diff --git a/src/accelerated_image_processor_compression/include/accelerated_image_processor_compression/video_compressor.hpp b/src/accelerated_image_processor_compression/include/accelerated_image_processor_compression/video_compressor.hpp new file mode 100644 index 0000000..ca28e00 --- /dev/null +++ b/src/accelerated_image_processor_compression/include/accelerated_image_processor_compression/video_compressor.hpp @@ -0,0 +1,111 @@ +// Copyright 2026 TIER IV, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace accelerated_image_processor::compression +{ +class JetsonVideoCompressor; +class JetsonH264Compressor; +class JetsonH265Compressor; +class JetsonAV1Compressor; + +/** + * @brief Enumeration of video encoder mode and string map + */ +enum class VideoCompressionType : uint8_t { LOSSY, LOSSLESS }; +const std::unordered_map video_compression_type_map = { + {"LOSSY", VideoCompressionType::LOSSY}, {"LOSSLESS", VideoCompressionType::LOSSLESS}}; + +/** + * @brief Utility function to convert std::string to enum class + */ +template +EnumType string_to_enum( + const std::string & str, const std::unordered_map & mapping) +{ + std::string uppercase_str = str; + std::transform(str.begin(), str.end(), uppercase_str.begin(), [](const unsigned char & c) { + return std::toupper(c); + }); + + auto it = mapping.find(uppercase_str); + if (it != mapping.end()) { + return it->second; + } else { + throw std::runtime_error("Invalid parameter was set: " + str); + } +} + +/** + * @brief Abstract base class for Jetson Video compressors. + */ +class VideoCompressor : public Compressor +{ +public: + explicit VideoCompressor( + CompressorBackend backend, common::ParameterMap dedicated_parameters = {}) + : Compressor( + backend, dedicated_parameters += { + {"compression_type", static_cast("lossy")}, + {"idr_frame_interval", static_cast(10)}, + {"i_frame_interval", static_cast(10)}, + {"frame_rate_numerator", static_cast(10)}, // frame + {"frame_rate_denominator", static_cast(1)}, // Second + }) + { + } + + ~VideoCompressor() override = default; + + /** + * @brief Validates the compatibility of the current compression parameters. + * + * This method checks whether the configured compression settings are compatible + * with the underlying hardware and software capabilities. It returns a tuple + * containing a boolean flag indicating success and an optional message + * describing any incompatibilities. + * + * @return std::tuple A tuple where the first element is + * true if the parameters are compatible, false otherwise; the second element + * contains an explanatory message when the parameters are not compatible. + */ + virtual std::tuple validate_compression_type_compatibility() + { + bool is_compatible = true; + std::string message = ""; + return {is_compatible, message}; + } +}; +//!< @brief Factory function to create a JetsonH264Compressor. +std::unique_ptr make_jetson_h264_compressor(); +//!< @brief Factory function to create a JetsonH265Compressor. +std::unique_ptr make_jetson_h265_compressor(); +//!< @brief Factory function to create a JetsonAV1Compressor. +std::unique_ptr make_jetson_av1_compressor(); +} // namespace accelerated_image_processor::compression diff --git a/src/accelerated_image_processor_compression/src/builder.cpp b/src/accelerated_image_processor_compression/src/builder.cpp index d21bc6f..2b77e78 100644 --- a/src/accelerated_image_processor_compression/src/builder.cpp +++ b/src/accelerated_image_processor_compression/src/builder.cpp @@ -14,7 +14,9 @@ #include "accelerated_image_processor_compression/builder.hpp" +#include "accelerated_image_processor_compression/compressor.hpp" #include "accelerated_image_processor_compression/jpeg_compressor.hpp" +#include "accelerated_image_processor_compression/video_compressor.hpp" #include #include @@ -52,8 +54,12 @@ CompressionType to_compression_type(const std::string & str) const auto s = normalize_str(str); if (s == "JPEG") { return CompressionType::JPEG; - } else if (s == "VIDEO") { - return CompressionType::VIDEO; + } else if (s == "H264") { + return CompressionType::H264; + } else if (s == "H265") { + return CompressionType::H265; + } else if (s == "AV1") { + return CompressionType::AV1; } else { throw std::invalid_argument("Invalid compression type: " + str); } @@ -72,8 +78,24 @@ std::unique_ptr create_compressor(CompressionType type) #else throw std::runtime_error("No JPEG compressor available"); #endif - case CompressionType::VIDEO: - throw std::runtime_error("VIDEO compression is not supported yet"); + case CompressionType::H264: +#ifdef JETSON_AVAILABLE + return make_jetson_h264_compressor(); +#else + throw std::runtime_error("H264 compression is not supported on this platform"); +#endif + case CompressionType::H265: +#ifdef JETSON_AVAILABLE + return make_jetson_h265_compressor(); +#else + throw std::runtime_error("H265 compression is not supported on this platform"); +#endif + case CompressionType::AV1: +#ifdef JETSON_AVAILABLE + return make_jetson_av1_compressor(); +#else + throw std::runtime_error("AV1 compression is not supported on this platform"); +#endif default: throw std::invalid_argument("Invalid compression type"); } diff --git a/src/accelerated_image_processor_compression/src/jpeg_compressor/cpu.cpp b/src/accelerated_image_processor_compression/src/jpeg_compressor/cpu.cpp index 2525821..e3de02f 100644 --- a/src/accelerated_image_processor_compression/src/jpeg_compressor/cpu.cpp +++ b/src/accelerated_image_processor_compression/src/jpeg_compressor/cpu.cpp @@ -34,7 +34,7 @@ namespace accelerated_image_processor::compression class CpuJPEGCompressor final : public JPEGCompressor { public: - CpuJPEGCompressor() : JPEGCompressor(JPEGBackend::CPU) { handle_ = tjInitCompress(); } + CpuJPEGCompressor() : JPEGCompressor(CompressorBackend::CPU) { handle_ = tjInitCompress(); } ~CpuJPEGCompressor() override { if (buffer_) { diff --git a/src/accelerated_image_processor_compression/src/jpeg_compressor/jetson.cpp b/src/accelerated_image_processor_compression/src/jpeg_compressor/jetson.cpp index 787ff51..3d30827 100644 --- a/src/accelerated_image_processor_compression/src/jpeg_compressor/jetson.cpp +++ b/src/accelerated_image_processor_compression/src/jpeg_compressor/jetson.cpp @@ -39,7 +39,7 @@ namespace accelerated_image_processor::compression class JetsonJPEGCompressor final : public JPEGCompressor { public: - JetsonJPEGCompressor() : JPEGCompressor(JPEGBackend::JETSON) + JetsonJPEGCompressor() : JPEGCompressor(CompressorBackend::JETSON) { CHECK_CUDA(cudaStreamCreate(&stream_)); encoder_ = NvJPEGEncoder::createJPEGEncoder("jpeg_encoder"); diff --git a/src/accelerated_image_processor_compression/src/jpeg_compressor/nvjpeg.cpp b/src/accelerated_image_processor_compression/src/jpeg_compressor/nvjpeg.cpp index 2241fd7..508bb5b 100644 --- a/src/accelerated_image_processor_compression/src/jpeg_compressor/nvjpeg.cpp +++ b/src/accelerated_image_processor_compression/src/jpeg_compressor/nvjpeg.cpp @@ -37,7 +37,7 @@ namespace accelerated_image_processor::compression class NvJPEGCompressor final : public JPEGCompressor { public: - NvJPEGCompressor() : JPEGCompressor(JPEGBackend::NVJPEG) + NvJPEGCompressor() : JPEGCompressor(CompressorBackend::NVJPEG) { CHECK_CUDA(cudaStreamCreate(&stream_)); CHECK_NVJPEG(nvjpegCreateSimple(&handle_)); diff --git a/src/accelerated_image_processor_compression/src/video_compressor/jetson.cpp b/src/accelerated_image_processor_compression/src/video_compressor/jetson.cpp new file mode 100644 index 0000000..8996db2 --- /dev/null +++ b/src/accelerated_image_processor_compression/src/video_compressor/jetson.cpp @@ -0,0 +1,533 @@ +// Copyright 2026 TIER IV, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "jetson.hpp" + +#include "accelerated_image_processor_compression/video_compressor.hpp" +#include "jetson_error_helper.hpp" +#include "jetson_precise_timestamp_map.hpp" + +#include + +#include + +#include +#include +#include +#include +#include +#include + +// Header that defines ffmpeg flags +extern "C" { +#include +} + +namespace +{ +constexpr bool is_big_endian = (__BYTE_ORDER__ == __BIG_ENDIAN); +} // namespace + +namespace accelerated_image_processor::compression +{ + +EncResult JetsonVideoCompressor::collect_params(EncoderParameter & params) +{ + params.buffer_length = this->parameter_value("buffer_length"); + params.compression_type = string_to_enum( + this->parameter_value("compression_type"), video_compression_type_map); + + params.idr_interval = this->parameter_value("idr_frame_interval"); + params.i_frame_interval = this->parameter_value("i_frame_interval"); + params.frame_rate_numerator = this->parameter_value("frame_rate_numerator"); + params.frame_rate_denominator = this->parameter_value("frame_rate_denominator"); + params.hw_preset_type = string_to_enum( + this->parameter_value("hw_preset_type"), hardware_preset_map); + params.use_max_performance_mode = this->parameter_value("use_max_performance"); + params.target_bits_per_pixel = this->parameter_value("target_bits_per_pixel"); + + return EncResult{EncStatus{true, ""}}; +} + +EncResult JetsonVideoCompressor::init_encoder(const common::Image & image) +{ + // gather parameters + if (auto res = collect_params(encoder_params_); !res.ok) { + return EncResult(record_error("Failed to correct parameters (" + res.status.message + ")")); + } + + if (auto res = this->collect_codec_params_impl(encoder_params_); !res.ok) { + return EncResult( + record_error("Failed to correct codec dedicated parameters (" + res.status.message + ")")); + } + + // Confirm the given combination of parameters is valid + if (auto [is_valid, msg] = validate_compression_type_compatibility(); !is_valid) { + return EncResult(record_error("Invalid parameters (" + msg + ")")); + } + + output_plane_fds_.assign(encoder_params_.buffer_length, -1); + output_nvsurface_.assign(encoder_params_.buffer_length, nullptr); + timestamp_map_ = std::make_unique(encoder_params_.buffer_length); + + // Configure encoder output (codec individual) + { + if (auto res = + this->set_capture_plane_format_impl(image.width, image.height, image.step * image.height); + !res.ok) + return EncResult( + record_error("Failed to set capture plane format (" + res.status.message + ")")); + } + + // Configure encoder input + { + auto pixel_format = pixel_format_map.at(encoder_params_.compression_type); + CHECK_NVENC( + encoder_->setOutputPlaneFormat(pixel_format, image.width, image.height), + "Failed to set output plane format"); + } + + // Do codec specific configuration + // almost all them are specified to be executed after setting ouput/capture plane format + // and before requesting any plane buffers + { + if (auto res = this->init_codec_impl(); !res.ok) { + return EncResult( + record_error("Codec specific configuration failed (" + res.status.message + ")")); + } + } + + // Common configurations + { + if (encoder_params_.compression_type == VideoCompressionType::LOSSLESS) { + CHECK_NVENC(encoder_->setLossless(true), "Could not set lossless encoding"); + } else { + // Enable variable rate control (VRC) + CHECK_NVENC( + encoder_->setRateControlMode(V4L2_MPEG_VIDEO_BITRATE_MODE_VBR), + "Failed to set variable rate control mode"); + + // compute the target bit rate from input streaming rate + double frame_rate = static_cast(encoder_params_.frame_rate_numerator) / + static_cast(encoder_params_.frame_rate_denominator); + auto target_bit_rate = + image.height * image.width * frame_rate * encoder_params_.target_bits_per_pixel; + auto peak_bit_rate = 1.2 * target_bit_rate; // set 1.2x of average bitrate as peak bitrate + CHECK_NVENC(encoder_->setBitrate(target_bit_rate), "Failed to set bit rate"); + CHECK_NVENC(encoder_->setPeakBitrate(peak_bit_rate), "Failed to set peak bit rate"); + } + + // Set IDR (Instantaneous Decoding Refresh) frame interval + // The IDR frame is a special format of I frame that ensures later P (and B) frames never refer + // the I frames before this frame. Decoders can restart from this frame in cases of seek or + // error. + CHECK_NVENC( + encoder_->setIDRInterval(encoder_params_.idr_interval), "Failed to set encoder IDR interval"); + + // Set I frame interval + // I frame is self-decodable frame, which can be decoded without referring other frames + CHECK_NVENC( + encoder_->setIFrameInterval(encoder_params_.i_frame_interval), + "Failed to set I Frame interval"); + + // Set frame rate + // rate is specified in [numerator (frames), denominator (second)] format + CHECK_NVENC( + encoder_->setFrameRate( + static_cast(encoder_params_.frame_rate_numerator), + static_cast(encoder_params_.frame_rate_denominator)), + "Failed to set frame rate"); + + CHECK_NVENC( + encoder_->setTemporalTradeoff( + V4L2_ENC_TEMPORAL_TRADEOFF_LEVEL_DROPNONE), // encode all frames + "Failed to set temporal trade off level to DROPNONE"); + + CHECK_NVENC( + encoder_->setHWPresetType(encoder_params_.hw_preset_type), + "Failed to set encoder hardware preset type"); + + // Video Usability Information (VUI) and extended color format are required to embed source + // image information properly + CHECK_NVENC( + encoder_->setInsertVuiEnabled(true), "Failed to set insert Video Usability information"); + CHECK_NVENC(encoder_->setExtendedColorFormat(true), "Failed to set extended color format"); + + CHECK_NVENC(encoder_->setAlliFramesEncode(false), "Failed to set number of all I frame"); + + // Disable B-Frame for streaming compression + CHECK_NVENC(encoder_->setNumBFrames(0), "Failed to set number of B-Frames"); + + CHECK_NVENC( + encoder_->setMaxPerfMode(static_cast(encoder_params_.use_max_performance_mode)), + "Error while setting encoder to max performance"); + } + + // Configure output plane (encoder input) so that it allows direct memory access buffer + { + if (auto res = setup_output_plane(image.height, image.width); !res.ok) { + return EncResult( + record_error("Failed to setup output DMA buffer (" + res.status.message + ")")); + } + } + + // Export and Map the capture plane buffers so that we can write encoded + // bistream data into the buffers + // NOTE: "capture_plane" represents "OUTPUT" of the encoder (the place + // where the application receives encoded images) + { + CHECK_NVENC( + encoder_->capture_plane.setupPlane( + V4L2_MEMORY_MMAP, encoder_params_.buffer_length, true, false), + "Could not setup capture plane"); + } + + // Boot encoder + { + // Subscribe for End Of Stream event + CHECK_NVENC(encoder_->subscribeEvent(V4L2_EVENT_EOS, 0, 0), "Failed to subscribe EOS event"); + + // set encoder output plane (encoder input) STREAMON + CHECK_NVENC(encoder_->output_plane.setStreamStatus(true), "Error in output plane streamon"); + + // set encoder capture plane (encoder output) STREAMON + CHECK_NVENC(encoder_->capture_plane.setStreamStatus(true), "Error in capture plane streamon"); + } + + // Start to run encoder callback thread that receives encoded results + { + // Set encoder capture plane dq thread callback for blocking io mode + encoder_->capture_plane.setDQThreadCallback(encoder_capture_plane_dq_callback); + + // startDQThread starts a thread internally which calls the encoder_capture_plane_dq_callback + // whenever a buffer is dequeued on the plane + dq_callback_args_ = + std::make_unique(image.frame_id, image.width, image.height, this); + + CHECK_NVENC( + encoder_->capture_plane.startDQThread(dq_callback_args_.get()), + "Failed to satart encoder callback thread"); + } + + // Enqueue all the empty capture_plane buffers + for (uint32_t i = 0; i < encoder_->capture_plane.getNumBuffers(); i++) { + struct v4l2_buffer v4l2_buf; + struct v4l2_plane planes[MAX_PLANES]; + + std::memset(&v4l2_buf, 0, sizeof(v4l2_buf)); + std::memset(&planes, 0, sizeof(v4l2_plane)); + + v4l2_buf.index = i; + v4l2_buf.m.planes = planes; + + CHECK_NVENC( + encoder_->capture_plane.qBuffer(v4l2_buf, NULL), + "Error while queueing buffer at capture plane"); + } + + // Enqueue all the empty output_plane buffers + for (uint32_t i = 0; i < encoder_->output_plane.getNumBuffers(); i++) { + struct v4l2_buffer v4l2_buf; + struct v4l2_plane planes[MAX_PLANES]; + + std::memset(&v4l2_buf, 0, sizeof(v4l2_buf)); + std::memset(planes, 0, sizeof(planes)); + + v4l2_buf.index = i; + v4l2_buf.m.planes = planes; + v4l2_buf.type = V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE; + v4l2_buf.memory = V4L2_MEMORY_DMABUF; + CHECK_NVENC( + encoder_->output_plane.mapOutputBuffers(v4l2_buf, output_plane_fds_[i]), + "Error while mapping buffer at output plane"); + + // zero clear output_plane memory at once + CHECK_NVENC(NvBufSurfaceMemSet(output_nvsurface_[i], 0, -1, 0), "Failed to NvBufSurfaceMemSet"); + + NvBuffer * buffer = encoder_->output_plane.getNthBuffer(i); + // Sync the hardware memory cache for the device + for (uint32_t j = 0; j < buffer->n_planes; j++) { + // zero clear and set bytesused member to non zero value + auto & plane = buffer->planes[j]; + uint32_t max_size = plane.fmt.sizeimage; + + plane.bytesused = max_size; + + v4l2_buf.m.planes[j].bytesused = max_size; + + CHECK_NVENC( + NvBufSurfaceSyncForDevice(output_nvsurface_[i], 0, -1), + "Error while NvBufSurfaceSyncFor Device at output plane for V4L2_MEMORY_DMABUF"); + } + + CHECK_NVENC( + encoder_->output_plane.qBuffer(v4l2_buf, NULL), + "Error while queueing buffer at output plane"); + } + + // Now, ready to process + state_ = State::READY; + return EncResult::success(); +} + +EncResult JetsonVideoCompressor::setup_output_plane(const int & height, const int & width) +{ + CHECK_NVENC( + encoder_->output_plane.reqbufs(V4L2_MEMORY_DMABUF, encoder_params_.buffer_length), + "Failed to request buffer for output plane as V4L2_MEMORY_DMABUF"); + + for (uint32_t i = 0; i < encoder_->output_plane.getNumBuffers(); i++) { + NvBufSurfaceCreateParams nvbuf_surface_create_params = {}; + nvbuf_surface_create_params.width = width; + nvbuf_surface_create_params.height = height; + nvbuf_surface_create_params.layout = NVBUF_LAYOUT_PITCH; + nvbuf_surface_create_params.colorFormat = + nvbuf_color_format_map.at(encoder_params_.compression_type); + nvbuf_surface_create_params.memType = NVBUF_MEM_SURFACE_ARRAY; + + /* Create output plane fd for DMABUF io-mode */ + CHECK_NVENC( + NvBufSurfaceCreate(&output_nvsurface_[i], 1, &nvbuf_surface_create_params), + "Failed to create NvBufSurface"); + output_plane_fds_[i] = output_nvsurface_[i]->surfaceList[0].bufferDesc; + output_nvsurface_[i]->numFilled = 1; + } + + return EncResult::success(); +} + +common::Image JetsonVideoCompressor::process_impl(const common::Image & image) +{ + if (state_ != State::READY) { + if (!init_encoder(image).ok) { + throw std::runtime_error("Encoder initialization failed: " + last_error_); + } + } + + // queue video frame the output plane buffer + struct v4l2_buffer v4l2_buf; + struct v4l2_plane planes[MAX_PLANES]; + NvBuffer * buffer; + + std::memset(&v4l2_buf, 0, sizeof(v4l2_buf)); + std::memset(planes, 0, sizeof(planes)); + v4l2_buf.m.planes = planes; + + // Dequeue buffer from encoder output plane + if (auto ret = encoder_->output_plane.dqBuffer(v4l2_buf, &buffer, NULL, 10); ret < 0) { + std::cerr << "ERROR while DQing buffer at output plane" << std::endl; + return common::Image(); + } + + // Wrap input data in VPIImage so that the data can be handled by VPI transparently + { + VPIImageData input_data_params = {}; + input_data_params.bufferType = VPI_IMAGE_BUFFER_HOST_PITCH_LINEAR; + if (image.encoding == common::ImageEncoding::RGB) { + input_data_params.buffer.pitch.format = VPI_IMAGE_FORMAT_RGB8; + } else if (image.encoding == common::ImageEncoding::BGR) { + input_data_params.buffer.pitch.format = VPI_IMAGE_FORMAT_BGR8; + } else { + std::cerr << "Unsupported input encoding detected" << std::endl; + return common::Image(); + } + input_data_params.buffer.pitch.numPlanes = 1; + input_data_params.buffer.pitch.planes[0].pixelType = VPI_PIXEL_TYPE_3U8; + input_data_params.buffer.pitch.planes[0].width = image.width; + input_data_params.buffer.pitch.planes[0].height = image.height; + input_data_params.buffer.pitch.planes[0].pitchBytes = image.step; + input_data_params.buffer.pitch.planes[0].data = const_cast(image.data.data()); + + VPIImageWrapperParams input_wrapper_params = {}; + CHECK_VPI(vpiInitImageWrapperParams(&input_wrapper_params)); + input_wrapper_params.colorSpec = + VPI_COLOR_SPEC_DEFAULT; // Informs that the color spec is to be inferred. + // input_wrapper_params.colorSpec = VPI_COLOR_SPEC_sRGB; + if (!input_rgb_dev_) { + CHECK_VPI( + vpiImageCreateWrapper(&input_data_params, &input_wrapper_params, 0, &input_rgb_dev_)); + } else { + CHECK_VPI(vpiImageSetWrapper(input_rgb_dev_, &input_data_params)); + } + } + + // Wrap the memory region held by NvBuffer so that VPI can write the + // color conversion result to it directly + { + VPIImageData data_params = {}; + data_params.bufferType = VPI_IMAGE_BUFFER_NVBUFFER; + data_params.buffer.fd = output_plane_fds_[v4l2_buf.index]; + + VPIImageWrapperParams wrapper_params = {}; + CHECK_VPI(vpiInitImageWrapperParams(&wrapper_params)); + wrapper_params.colorSpec = + VPI_COLOR_SPEC_DEFAULT; // Informs that the color spec is to be inferred. + + uint64_t wrapper_flag = 0; // The backend selection happens during the algorithm submission + if (!output_yuv_dev_) { + // Create an image object by wrapping an existing memory block. + CHECK_VPI( + vpiImageCreateWrapper(&data_params, &wrapper_params, wrapper_flag, &output_yuv_dev_)); + } else { + // Redefines the wrapped memory in an existing VPIImage wrapper. + CHECK_VPI(vpiImageSetWrapper(output_yuv_dev_, &data_params)); + } + } + + // fill encoder input memory region with the color converted image data + fill_encoder_input_async(); + + // Copy timestamp from source image + { + // NOTE: Since nanosecond order timestamp resolution, such as provided by ROS timestamp, will be + // lost in v4l2_buf.timestamp (microsecond order), actual timestamp is derived to the output + // result via timestamp_map_ + // NOTE: If the time gap between the current and previous frames is much larger/smaller than + // what the frame rate (encoder_params_.frame_rate_numerator / + // encoder_params_.frame_rate_denominator) expects, the encoder may output a warning such as: + // "NVENC_H265: Unsupported frameRate (Supported: 1.0 - 60.0), setting to default value 30.00" + v4l2_buf.flags |= V4L2_BUF_FLAG_TIMESTAMP_COPY; + v4l2_buf.timestamp.tv_sec = image.timestamp / 1'000'000'000ULL; + v4l2_buf.timestamp.tv_usec = (image.timestamp % 1'000'000'000ULL) / 1'000ULL; + timestamp_map_->set(v4l2_buf.index, TimestampMap::PreciseTimestamp(image.timestamp)); + } + + // Since filling frame data to the buffer goes asynchronously, + // wait its completion here before conduct qBuffer (i.e., encoding) + { + CHECK_CUDA(cudaStreamSynchronize(stream_)); + CHECK_VPI(vpiStreamSync(vpi_stream_)); + } + + // Sync the hardware memory cache for the device + for (uint32_t j = 0; j < buffer->n_planes; j++) { + NvBufSurface * nvbuf_surf = 0; + if (auto ret = NvBufSurfaceFromFd(buffer->planes[j].fd, reinterpret_cast(&nvbuf_surf)); + ret < 0) { + std::cerr << "Error while NvBufSurfaceFromFd" << std::endl; + return common::Image(); + } + if (auto ret = NvBufSurfaceSyncForDevice(nvbuf_surf, 0, j); ret < 0) { + std::cerr << "Error while NvBuSurfaceSyncForDevice at output plane for V4L2_MEMORY_MMAP" + << std::endl; + return common::Image(); + } + + buffer->planes[j].bytesused = buffer->planes[j].fmt.stride * buffer->planes[j].fmt.height; + v4l2_buf.m.planes[j].bytesused = buffer->planes[j].bytesused; + } + + // feed input data to the encoder + { + if (auto ret = encoder_->output_plane.qBuffer(v4l2_buf, NULL); ret < 0) { + std::cerr << "Error while queueing buffer at output plane" << std::endl; + return common::Image(); + } + } + + // Since the compression result will be acquired on the other thread, + // just return input as a valid data + return image; +} + +void JetsonVideoCompressor::fill_encoder_input_async(void) +{ + uint64_t backend = VPI_BACKEND_CUDA; + VPIConvertImageFormatParams cvt_params; + vpiInitConvertImageFormatParams(&cvt_params); + cvt_params.policy = VPI_CONVERSION_CAST; + cvt_params.flags = VPI_PRECISE; + + CHECK_VPI(vpiSubmitConvertImageFormat( + vpi_stream_, backend, input_rgb_dev_, output_yuv_dev_, &cvt_params)); +} + +bool JetsonVideoCompressor::encoder_capture_plane_dq_callback( + struct v4l2_buffer * v4l2_buf, NvBuffer * buffer, [[maybe_unused]] NvBuffer * shared_buffer, + void * arg) +{ + // Argument provided via startDQThread() in void* form + DqCallbackArgs * callback_args = reinterpret_cast(arg); + auto * compressor_object = callback_args->obj; + auto * encoder = compressor_object->encoder(); + + if (v4l2_buf == nullptr) { + encoder->abort(); + std::cerr << "Error while dequeuing buffer from capture plane" << std::endl; + return false; + } + + // Execute preprocess for the encoded payload (some codec requires dedicated handling) + auto payload_info = + compressor_object->payload_preprocess_impl(callback_args, v4l2_buf->bytesused, buffer); + + // Since this function will also be called during initialization, which dummy frames are fed, + // skip such dummy data + { + auto & initial_frame_count = compressor_object->initial_frame_count(); + const auto buffer_length = compressor_object->encoder_params().buffer_length; + if (initial_frame_count < buffer_length) { + // Do nothing for the dummy data. Just return (queue) buffer to the capture plane so that + // successive actual frames arrive + initial_frame_count++; + CHECK_ERROR( + encoder->capture_plane.qBuffer(*v4l2_buf, NULL) < 0, + "Failed to queuing buffer to the capture plane"); + return true; + } + } + + // Get encode metadata + v4l2_ctrl_videoenc_outputbuf_metadata enc_metadata; + encoder->getMetadata(v4l2_buf->index, enc_metadata); + + // Create result data + common::Image processed; + { + auto & timestamp_map = compressor_object->timestamp_map(); + int64_t stamp_in_nanosecond = 0; + TimestampMap::PreciseTimestamp ps; + if (!timestamp_map->get(v4l2_buf->index, ps)) { + // fail to fetch precise timestamp. Fallback to use v4l2 buffer timestamp + stamp_in_nanosecond = static_cast(v4l2_buf->timestamp.tv_sec) * 1e9 + + static_cast(v4l2_buf->timestamp.tv_usec) * 1e3; + } else { + stamp_in_nanosecond = static_cast(ps); + } + + processed.frame_id = callback_args->frame_id; + processed.timestamp = stamp_in_nanosecond; + processed.height = callback_args->input_height; + processed.width = callback_args->input_width; + processed.format = supported_codec_format_map.at(compressor_object->codec()); + processed.pts = stamp_in_nanosecond / 1e3; // [us] + processed.flags = enc_metadata.KeyFrame ? AV_PKT_FLAG_KEY : 0; + processed.is_bigendian = is_big_endian; + + compressor_object->payload_copy_impl( + enc_metadata.KeyFrame, payload_info, callback_args, processed.data); + } + + // Now, v4l2_buffer can be queued again + CHECK_ERROR( + encoder->capture_plane.qBuffer(*v4l2_buf, NULL) < 0, + "Failed to Queuing buffer to capture plane"); + + // call postprocess + compressor_object->postprocess(processed); + + return true; +} + +} // namespace accelerated_image_processor::compression diff --git a/src/accelerated_image_processor_compression/src/video_compressor/jetson.hpp b/src/accelerated_image_processor_compression/src/video_compressor/jetson.hpp new file mode 100644 index 0000000..b8bf338 --- /dev/null +++ b/src/accelerated_image_processor_compression/src/video_compressor/jetson.hpp @@ -0,0 +1,381 @@ +// Copyright 2026 TIER IV, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +#pragma once +#include "accelerated_image_processor_common/helper.hpp" +#include "accelerated_image_processor_compression/video_compressor.hpp" +#include "jetson_error_helper.hpp" +#include "jetson_precise_timestamp_map.hpp" + +#include +#include +#include +#include +#include +#include + +#ifdef JETSON_AVAILABLE +#include "NvBufSurface.h" +#include "NvUtils.h" +#include "NvVideoEncoder.h" + +#include +#include +#include +#include +#include +#endif + +namespace accelerated_image_processor::compression +{ +#ifdef JETSON_AVAILABLE +/** + * @brief Enumeration of available codecs + */ +enum class SupportedCodec : uint8_t { H264, H265, AV1 }; + +/** + * @brief Abstract base class for Video compressor working on Jetson devices. + */ +class JetsonVideoCompressor : public VideoCompressor +{ +protected: + /** + * @brief Arguments passed to the capture plane dequeue callback. + * + * This structure holds information about the frame being processed, + * including its identifier, dimensions, and a pointer to the owning + * JetsonVideoCompressor instance. It is used to associate + * the callback with the correct compressor context. + * + * @var std::string frame_id + * The frame ID derived from the input + * @var int input_width + * Width of the input image in pixels. + * @var int input_height + * Height of the input image in pixels. + * @var JetsonVideoCompressor* obj + * Pointer to the compressor instance that owns this callback. + */ + struct DqCallbackArgs + { + std::string frame_id; + int input_width; + int input_height; + JetsonVideoCompressor * obj; + + DqCallbackArgs( + const std::string f_id, const int width, const int height, JetsonVideoCompressor * obj) + : frame_id(f_id), input_width(width), input_height(height), obj(obj) + { + } + }; + +public: + /** + * @brief Lookup table to tie the string to the corresponding codecs + */ + inline static const std::unordered_map supported_codec_map = { + {"H264", SupportedCodec::H264}, + {"H265", SupportedCodec::H265}, + {"AV1", SupportedCodec::AV1}, + }; + + /** + * @brief Lookup table to tie the supported codecs enumeration to the common::ImageFormat + */ + inline static const std::unordered_map + supported_codec_format_map = { + {SupportedCodec::H264, common::ImageFormat::H264}, + {SupportedCodec::H265, common::ImageFormat::H265}, + {SupportedCodec::AV1, common::ImageFormat::AV1}, + }; + + /** + * @brief Map between strings and corresponding hardware preset types + */ + inline static const std::unordered_map hardware_preset_map = + { + {"DISABLE", V4L2_ENC_HW_PRESET_DISABLE}, {"SLOW", V4L2_ENC_HW_PRESET_SLOW}, + {"MEDIUM", V4L2_ENC_HW_PRESET_MEDIUM}, {"FAST", V4L2_ENC_HW_PRESET_FAST}, + {"ULTRAFAST", V4L2_ENC_HW_PRESET_ULTRAFAST}, + }; + + /** + * @brief Map between compression type and pixel format to be used for configuring encoder input + * (consumed by encoder API) + */ + inline static const std::unordered_map pixel_format_map = { + {VideoCompressionType::LOSSY, V4L2_PIX_FMT_NV12M}, + {VideoCompressionType::LOSSLESS, V4L2_PIX_FMT_NV24M}, + }; + + /** + * @brief Map between compression type and NvBuffer pixel format to be used for configuring + * encoder input DMA buffer (consumed by NvBuffer API) + */ + inline static const std::unordered_map + nvbuf_color_format_map = { + {VideoCompressionType::LOSSY, + NVBUF_COLOR_FORMAT_NV12_ER}, // Y/CbCr 4:2:0 multi-planar, extended range (full color) + {VideoCompressionType::LOSSLESS, + NVBUF_COLOR_FORMAT_NV24_ER}, // Y/CbCr 4:4:4 multi-planar, extended range (full color) + }; + + /** + * @brief Configuration parameters for the Jetson video encoder. + * + * This structure encapsulates all the tunable settings that control the + * behavior of the NvVideoEncoder. The values are typically derived from + * the user supplied parameter map and are validated during the + * initialization phase. + * + * @var int buffer_length + * Number of buffers reserved for the encoder output plane. + * + * @var VideoCompressionType compression_type + * The compression mode (lossy or lossless) selected for the stream. + * + * @var int idr_interval + * Interval (in frames) between IDR (Instantaneous Decoder Refresh) + * keyframes. + * + * @var int i_frame_interval + * Interval (in frames) between I‑frames. + * + * @var int frame_rate_numerator + * Numerator of the target frame rate (numerator in second). + * + * @var int frame_rate_denominator + * Denominator of the target frame rate (denominator in frames). + * + * @var v4l2_enc_hw_preset_type hw_preset_type + * Hardware preset that tunes the encoder for speed or quality. + * + * @var bool use_max_performance_mode + * When true the encoder is forced into a high‑performance mode, + * potentially at the cost of increased power consumption. + * + * @var double target_bits_per_pixel + * Target bitrate expressed as bits per pixel. This value is used + * by the encoder to determine the target bit rate, which mainly affects encoded image quality + * and payload size. + */ + struct EncoderParameter + { + int buffer_length; + VideoCompressionType compression_type; + int idr_interval; + int i_frame_interval; + int frame_rate_numerator; + int frame_rate_denominator; + v4l2_enc_hw_preset_type hw_preset_type; + bool use_max_performance_mode; + double target_bits_per_pixel; + }; + + /** + * @brief Constructor + */ + explicit JetsonVideoCompressor( + SupportedCodec codec, common::ParameterMap dedicated_parameters = {}) + : VideoCompressor( + CompressorBackend::JETSON, dedicated_parameters += + {{"buffer_length", static_cast(4)}, + {"use_max_performance", static_cast(true)}, + {"hw_preset_type", static_cast("disable")}, + {"target_bits_per_pixel", static_cast(0.1)}}), + codec_(codec) + { + // To make EGL + // (https://developer.nvidia.com/blog/egl-eye-opengl-visualization-without-x-server/) , which is + // utilized VPI underhood, work headless, unset DISPLAY environment variable to avoid it affects + // EGL behavior + int replace_env = 1; + setenv("EGL_PLATFORM", "surfaceless", replace_env); + + CHECK_CUDA(cudaStreamCreate(&stream_)); + uint64_t vpi_stream_flag = 0; // No flag is specified + CHECK_VPI(vpiStreamCreateWrapperCUDA( + stream_, vpi_stream_flag, &vpi_stream_)); // Share stream between CUDA and VPI + + encoder_ = NvVideoEncoder::createVideoEncoder("encoder"); + if (!encoder_) { + throw std::runtime_error("Failed to create NvVideoEncoder"); + } + } + + /** + * @brief Destructor: clean up the resources accordingly + */ + ~JetsonVideoCompressor() + { + // Wait till capture plane DQ Thread finishes + // i.e. all the capture plane buffers are dequeued + encoder_->capture_plane.waitForDQThread(1000); + + if (input_rgb_dev_) { + vpiImageDestroy(input_rgb_dev_); + } + + if (output_yuv_dev_) { + vpiImageDestroy(output_yuv_dev_); + } + + for (uint32_t i = 0; i < encoder_->output_plane.getNumBuffers(); i++) { + /* Unmap output plane buffer for memory type DMABUF. */ + CHECK_ERROR( + encoder_->output_plane.unmapOutputBuffers(i, output_plane_fds_[i]) < 0, + "Error while unmapping buffer at output plane"); + + // ERROR_CHECK(NvBufSurf::NvDestroy(output_plane_fds_[i]), "Failed to Destroy NvBuffer"); + NvBufSurfaceDestroy(output_nvsurface_[i]); + output_plane_fds_[i] = -1; + } + + vpiStreamDestroy(vpi_stream_); // this returns void, so VPI_CHECK cannot be applicable + CHECK_CUDA(cudaStreamDestroy(stream_)); + } + + /** + * @brief [override] process the input image without postprocessing + */ + std::optional process(const common::Image & image) override + { + auto processed = this->process_impl(image); + // always return nullopt because processed (encoded) result will be handled in the other thread + return std::nullopt; + } + + /** + * @brief [override] Check the encoder is ready to run processing. + */ + bool is_ready() const override { return state_ != State::ERROR; } + + // Getter functions to access members from static `encoder_capture_plane_dq_callback` + auto * encoder() { return this->encoder_; } + const auto & encoder_params() { return this->encoder_params_; } + auto & initial_frame_count() { return this->initial_frame_count_; } + auto & timestamp_map() { return this->timestamp_map_; } + const auto & codec() { return this->codec_; } + +protected: + enum class State : uint8_t { UNINITIALIZED, READY, ERROR }; + struct PayloadInfo + { + uint8_t * payload_ptr; + size_t payload_size; + size_t offset; + }; + State state_{State::UNINITIALIZED}; + + /** + * @brief codec dedicated parameter collection + */ + virtual EncResult collect_codec_params_impl( + [[maybe_unused]] const EncoderParameter & general_params) = 0; + + /** + * @brief codec dedicated setup steps for capture plane (encoder output) + */ + virtual EncResult set_capture_plane_format_impl( + const uint32_t & width, const uint32_t & height, const uint32_t & image_size) = 0; + + /** + * @brief codec dedicated initialization steps + */ + virtual EncResult init_codec_impl() = 0; + + /** + * @brief Payload preprocessing implementation + * Some codecs may need dedicated handling for the encoded payload, which this function handles + */ + virtual PayloadInfo payload_preprocess_impl( + [[maybe_unused]] DqCallbackArgs * callback_args, const size_t bytes_used, + const NvBuffer * buffer) + { + uint8_t * payload_ptr = reinterpret_cast(buffer->planes[0].data); + size_t payload_size = bytes_used; + size_t offset = 0; + + return {payload_ptr, payload_size, offset}; + } + + /** + * @brief Codec dedicated payload copy implementation + * Similar to payload preprocess, this function handles codec dedicated data copy + */ + virtual void payload_copy_impl( + [[maybe_unused]] const bool is_keyframe, const PayloadInfo & payload_info, + [[maybe_unused]] DqCallbackArgs * callback_args, std::vector & copy_destination) + { + auto & [payload_ptr, payload_size, offset] = payload_info; + copy_destination.resize(payload_size); + std::memcpy(copy_destination.data(), payload_ptr, payload_size); + } + + /** + * @brief Collects and validates encoder parameters from the compressor's + * dedicated parameter map. + * + * This function merges the general encoder parameters (buffer length, + * compression type, frame rate, etc.) with codec‑specific parameters + * collected by the derived class implementation of + * `collect_codec_params_impl`. The resulting `EncoderParameter` struct + * is stored in `encoder_params_` for later use during initialization. + * + * @param params Reference to an `EncoderParameter` struct that will be + * populated with the collected values. + * + * @return An `EncResult` indicating success or failure. On failure the + * `EncStatus` will contain an explanatory message and the + * compressor state will be set to `ERROR`. + */ + EncResult collect_params(EncoderParameter & params); + + NvVideoEncoder * encoder_; + EncoderParameter encoder_params_; + +private: + EncResult init_encoder(const common::Image & image); + EncResult setup_output_plane(const int & height, const int & width); + void fill_encoder_input_async(void); + common::Image process_impl(const common::Image & image) override; + inline EncStatus record_error(const std::string & msg) + { + last_error_ = msg; + state_ = State::ERROR; + return EncStatus(false, msg); + } + + static bool encoder_capture_plane_dq_callback( + struct v4l2_buffer * v4l2_buffer, NvBuffer * buffer, [[maybe_unused]] NvBuffer * shared_buffer, + void * arg); + + std::string last_error_{""}; + SupportedCodec codec_; + + std::vector output_plane_fds_{}; + std::vector output_nvsurface_{}; + VPIImage input_rgb_dev_{nullptr}; + VPIImage output_yuv_dev_{nullptr}; + cudaStream_t stream_{}; + VPIStream vpi_stream_{}; + + std::unique_ptr timestamp_map_; + std::unique_ptr dq_callback_args_; + + int initial_frame_count_{0}; +}; +#endif // JETSON_AVAILABLE +} // namespace accelerated_image_processor::compression diff --git a/src/accelerated_image_processor_compression/src/video_compressor/jetson_av1.cpp b/src/accelerated_image_processor_compression/src/video_compressor/jetson_av1.cpp new file mode 100644 index 0000000..944379d --- /dev/null +++ b/src/accelerated_image_processor_compression/src/video_compressor/jetson_av1.cpp @@ -0,0 +1,200 @@ +// Copyright 2026 TIER IV, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "jetson.hpp" + +#include + +#include +#include + +namespace accelerated_image_processor::compression +{ +#ifdef JETSON_AVAILABLE + +/** + * @brief AV1 encoder working on Jetson devices. + */ +class JetsonAV1Compressor final : public JetsonVideoCompressor +{ +public: + /** + * @brief IVF header size (32 byte) + */ + static constexpr size_t ivf_file_header_size_in_byte = 32; + + /** + * @brief IVF frame header size (12 byte) + */ + static constexpr size_t ivf_frame_header_size_in_byte = 12; + + /** + * @brief IVF total header size (44 byte) + */ + static constexpr size_t first_frame_header_size = + ivf_file_header_size_in_byte + ivf_frame_header_size_in_byte; + + /** + * @brief constructor + * + * exposed parameters are: + * - enable_tile: if true, enable tiling division in AV1 codec, which leads parallel encoding + * - log2_num_tile_row: how many rows consisting of a tile in log2. ex. If 1 is given, 1 = + * log2(2) -> 2 rows will be used. If 2 is given, 2 = log2(4) -> 4rows will be used + * - log2_num_tile_cols: how many columns consisting of a tile in log2. calculation is the same + * as row's pattern + * - enable_ssim_rdo: flag if SSIM RDO (Variance based Structural Similarity Rate + * Distortion Optimization) is enabled + * - enable_cdf_update: flag if CDF (Cumulative Distribution Function) is enabled. If true, the + * encoder updates CDF for entropy encoding every frame + */ + JetsonAV1Compressor() + : JetsonVideoCompressor( + SupportedCodec::AV1, {{"av1.enable_tile", static_cast(true)}, + {"av1.log2_num_tile_row", static_cast(1)}, + {"av1.log2_num_tile_col", static_cast(1)}, + {"av1.enable_ssim_rdo", static_cast(false)}, + {"av1.enable_cdf_update", static_cast(true)}}) + { + } + +protected: + /** + * @brief Getter function to access private member + */ + auto & header_cache() { return header_cache_; } + + EncResult collect_codec_params_impl( + [[maybe_unused]] const EncoderParameter & general_params) override + { + enable_tile_ = this->parameter_value("av1.enable_tile"); + log2_num_tile_row_ = this->parameter_value("av1.log2_num_tile_row"); + log2_num_tile_col_ = this->parameter_value("av1.log2_num_tile_col"); + enable_ssim_rdo_ = this->parameter_value("av1.enable_ssim_rdo"); + enable_cdf_update_ = this->parameter_value("av1.enable_cdf_update"); + + return EncResult::success(); + } + + EncResult set_capture_plane_format_impl( + const uint32_t & width, const uint32_t & height, const uint32_t & image_size) override + { + CHECK_NVENC( + encoder_->setCapturePlaneFormat(V4L2_PIX_FMT_AV1, width, height, image_size), + "Failed to set capture plane format to AV1"); + + return EncResult::success(); + } + + EncResult init_codec_impl() override + { + if (enable_tile_) { + v4l2_enc_av1_tile_config tile_config; + tile_config.bEnableTile = enable_tile_; + tile_config.nLog2RowTiles = log2_num_tile_row_; + tile_config.nLog2ColTiles = log2_num_tile_col_; + CHECK_NVENC(encoder_->enableAV1Tile(tile_config), "Failed to enable AV1 tile configuration"); + } + + CHECK_NVENC( + encoder_->setAV1SsimRdo(enable_ssim_rdo_), + "Failed to set AV1's SSIM RDO (variance based Structural Similarity Rate Distortion " + "Optimization)"); + + CHECK_NVENC( + encoder_->setAV1DisableCDFUpdate(!enable_cdf_update_), + "Failed to configure CDF update for AV1"); + + return EncResult::success(); + } + + PayloadInfo payload_preprocess_impl( + DqCallbackArgs * callback_args, const size_t bytes_used, const NvBuffer * buffer) override + { + auto [payload_ptr, payload_size, offset] = + JetsonVideoCompressor::payload_preprocess_impl(callback_args, bytes_used, buffer); + + // Because jetson AV1 encoder wrap AV1 payload by IVF header, which leads to the decoder unable + // to recognize AV1 payload, peel unnecessary header from the payload + if ( + payload_size > 4 && payload_ptr[0] == 'D' && payload_ptr[1] == 'K' && payload_ptr[2] == 'I' && + payload_ptr[3] == 'F') { + // IVF file header starts with 'DKIF' + // Initial frame: skip IVF file header (32byte) + IVF frame header (12 bytes) + offset = first_frame_header_size; + } else { + // 2nd frame and later: skip IVF frame header + offset = ivf_frame_header_size_in_byte; + } + + if (offset > payload_size) { + throw std::runtime_error("Invalid AV1 payload observed"); + } + + payload_size = payload_size - offset; + payload_ptr = payload_ptr + offset; + + // Caching the AV1 header information so that payload can be decoded without the very first + // frame + auto & header_cache = dynamic_cast(callback_args->obj)->header_cache(); + if (header_cache.empty() && offset == first_frame_header_size) { + // Save whole payload including sequence header + header_cache.resize(payload_size); + std::memcpy(header_cache.data(), payload_ptr, payload_size); + } + + return {payload_ptr, payload_size, offset}; + } + + void payload_copy_impl( + const bool is_keyframe, const PayloadInfo & payload_info, DqCallbackArgs * callback_args, + std::vector & copy_destination) override + { + auto & header_cache = dynamic_cast(callback_args->obj)->header_cache(); + auto & [payload_ptr, payload_size, offset] = payload_info; + + if (is_keyframe && !header_cache.empty() && offset != first_frame_header_size) { + // For the key frames, copy the AV1 sequence header so that decoders can start the process + // from any key frames + size_t header_size = header_cache.size(); + copy_destination.resize(header_size + payload_size); + + std::memcpy(copy_destination.data(), header_cache.data(), header_size); + std::memcpy(copy_destination.data() + header_size, payload_ptr, payload_size); + } else { + JetsonVideoCompressor::payload_copy_impl( + is_keyframe, payload_info, callback_args, copy_destination); + } + } + +private: + bool enable_tile_; + int log2_num_tile_row_; + int log2_num_tile_col_; + bool enable_ssim_rdo_; + bool enable_cdf_update_; + std::vector header_cache_; +}; + +std::unique_ptr make_jetson_av1_compressor() +{ + return std::make_unique(); +} +#else +std::unique_ptr make_jetson_av1_compressor() +{ + return nullptr; +} +#endif // JETSON_AVAILABLE +} // namespace accelerated_image_processor::compression diff --git a/src/accelerated_image_processor_compression/src/video_compressor/jetson_error_helper.hpp b/src/accelerated_image_processor_compression/src/video_compressor/jetson_error_helper.hpp new file mode 100644 index 0000000..ed5eb47 --- /dev/null +++ b/src/accelerated_image_processor_compression/src/video_compressor/jetson_error_helper.hpp @@ -0,0 +1,59 @@ +#pragma once +#include +#include +#include +#include + +namespace accelerated_image_processor::compression +{ +/** + * @brief Holds the raw result of an NvEncoder/V4L2 call and readable message. + */ +struct EncStatus +{ + bool ok{true}; // true if NvEncoder returned 0 + std::string message; // error text when !ok + + EncStatus() = default; + EncStatus(bool r, const std::string & m = "") : ok(r), message(m) {} +}; + +/** + * @brief Wraps a NvEncoder/V4L2 call that returns 0 on success and -1 on failure. + * + * The macro passes __FILE__ and __LINE__ automatically. + */ +inline EncStatus check_nvenc_call(int fn, const char * file, int line) +{ + int ret = fn; + if (ret == 0) return EncStatus{true}; + std::string err = std::strerror(errno); + std::string msg = std::string(file) + ":" + std::to_string(line) + " (" + err + ")"; + std::cerr << msg << std::endl; + return EncStatus{false, msg}; +} + +struct EncResult +{ + bool ok{true}; + EncStatus status; + + EncResult() = default; + explicit EncResult(const EncStatus & s) : ok(s.ok), status(s) {} + explicit EncResult(bool o, const EncStatus & s) : ok(o), status(s) {} + static EncResult success() { return EncResult(EncStatus{true, ""}); } +}; + +} // namespace accelerated_image_processor::compression + +/** + * @brief Helper macro so the caller need only write: CHECK_NVENC(f, "Some operation") + */ +#define CHECK_NVENC(fn, msg) \ + { \ + auto _res = \ + accelerated_image_processor::compression::check_nvenc_call(fn, __FILE__, __LINE__); \ + if (!_res.ok) { \ + return accelerated_image_processor::compression::EncResult{_res}; \ + } \ + } diff --git a/src/accelerated_image_processor_compression/src/video_compressor/jetson_h264.cpp b/src/accelerated_image_processor_compression/src/video_compressor/jetson_h264.cpp new file mode 100644 index 0000000..b204936 --- /dev/null +++ b/src/accelerated_image_processor_compression/src/video_compressor/jetson_h264.cpp @@ -0,0 +1,180 @@ +// Copyright 2026 TIER IV, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "jetson.hpp" +#include "jetson_error_helper.hpp" + +#include + +#include +#include +#include +#include + +namespace accelerated_image_processor::compression +{ +#ifdef JETSON_AVAILABLE +/** + * @brief H.264 encoder working on Jetson devices. + */ +class JetsonH264Compressor final : public JetsonVideoCompressor +{ +public: + inline static const std::unordered_map + h264_profile_map = { + {"BASELINE", V4L2_MPEG_VIDEO_H264_PROFILE_BASELINE}, + {"MAIN", V4L2_MPEG_VIDEO_H264_PROFILE_MAIN}, + {"HIGH", V4L2_MPEG_VIDEO_H264_PROFILE_HIGH}, + {"HIGH_444", V4L2_MPEG_VIDEO_H264_PROFILE_HIGH_444_PREDICTIVE}, + }; + + inline static const std::unordered_map h264_level_map = { + {"1_0", V4L2_MPEG_VIDEO_H264_LEVEL_1_0}, {"1B", V4L2_MPEG_VIDEO_H264_LEVEL_1B}, + {"1_1", V4L2_MPEG_VIDEO_H264_LEVEL_1_1}, {"1_2", V4L2_MPEG_VIDEO_H264_LEVEL_1_2}, + {"1_3", V4L2_MPEG_VIDEO_H264_LEVEL_1_3}, {"2_0", V4L2_MPEG_VIDEO_H264_LEVEL_2_0}, + {"2_1", V4L2_MPEG_VIDEO_H264_LEVEL_2_1}, {"2_2", V4L2_MPEG_VIDEO_H264_LEVEL_2_2}, + {"3_0", V4L2_MPEG_VIDEO_H264_LEVEL_3_0}, {"3_1", V4L2_MPEG_VIDEO_H264_LEVEL_3_1}, + {"3_2", V4L2_MPEG_VIDEO_H264_LEVEL_3_2}, {"4_0", V4L2_MPEG_VIDEO_H264_LEVEL_4_0}, + {"4_1", V4L2_MPEG_VIDEO_H264_LEVEL_4_1}, {"4_2", V4L2_MPEG_VIDEO_H264_LEVEL_4_2}, + {"5_0", V4L2_MPEG_VIDEO_H264_LEVEL_5_0}, {"5_1", V4L2_MPEG_VIDEO_H264_LEVEL_5_1}, + }; + + JetsonH264Compressor() + : JetsonVideoCompressor( + SupportedCodec::H264, {{"h264.profile", static_cast("HIGH")}, + {"h264.level", static_cast("5_1")}, + {"h264.enable_cabac", static_cast(true)}}) + { + } + + /** + * @brief override impelmentation to validate the compatibility of the current compression + * parameters. + */ + std::tuple validate_compression_type_compatibility() override + { + // Load the latest parameters if the encoder is uninitialized; use the current config otherwise + EncoderParameter latest_params; + if (this->state_ == State::UNINITIALIZED) { + this->collect_params(latest_params); + } else { + latest_params = this->encoder_params_; + } + + this->collect_codec_params_impl( + latest_params); // this udpates class member `h264_profile_` and `h264_level_` + + auto ret = validate_compression_type_compatibility_impl( + latest_params.compression_type, this->h264_profile_, this->h264_level_); + return {ret.ok, ret.status.message}; + } + +protected: + /** + * @brief class dedicated implementation to validate parameter compatibility + */ + EncResult validate_compression_type_compatibility_impl( + const VideoCompressionType & type, const v4l2_mpeg_video_h264_profile & profile, + const v4l2_mpeg_video_h264_level & level) + { + // Lossless encoding is supported only for HIGH_444 level + if ( + type == VideoCompressionType::LOSSLESS && + profile != V4L2_MPEG_VIDEO_H264_PROFILE_HIGH_444_PREDICTIVE) { + return EncResult( + EncStatus(false, "Lossless compression is only supported for HIGH_444 level in H264")); + } + + // HIGH_444 level does not support non-lossless (lossy) mode + if ( + type == VideoCompressionType::LOSSY && + profile == V4L2_MPEG_VIDEO_H264_PROFILE_HIGH_444_PREDICTIVE) { + return EncResult( + EncStatus(false, "HIGH_444 level only support Lossless compression type in H264")); + } + + // lossless compression requires high bitrate. lower level forces encoder work in bitrate that + // is not enough for lossless compression, which may causes encoder initialization failure + // and/or FD DMA mapping for capture plane. Though it is not explicitly documented, this class + // treat the highest level is the only valid one for lossless. + if (type == VideoCompressionType::LOSSLESS && level != V4L2_MPEG_VIDEO_H264_LEVEL_5_1) { + return EncResult( + EncStatus(false, "Lossless compression is only supported with level 5_1 in H264")); + } + + return EncResult::success(); + } + + EncResult collect_codec_params_impl( + [[maybe_unused]] const EncoderParameter & general_params) override + { + h264_profile_ = string_to_enum( + this->parameter_value("h264.profile"), h264_profile_map); + h264_level_ = string_to_enum( + this->parameter_value("h264.level"), h264_level_map); + enable_cabac_ = this->parameter_value("h264.enable_cabac"); + + return EncResult::success(); + } + + EncResult set_capture_plane_format_impl( + const uint32_t & width, const uint32_t & height, const uint32_t & image_size) override + { + CHECK_NVENC( + encoder_->setCapturePlaneFormat(V4L2_PIX_FMT_H264, width, height, image_size), + "Failed to set capture plane format to H264"); + + return EncResult::success(); + } + + EncResult init_codec_impl() override + { + CHECK_NVENC(encoder_->setProfile(h264_profile_), "Failed to set H264 profile"); + + CHECK_NVENC(encoder_->setLevel(h264_level_), "Failed to set H264 level"); + + // Enable/disable Context-Adaptive Binary Arithmetic Coding (CABAC) + // This option is valid only for H.264 + CHECK_NVENC(encoder_->setCABAC(enable_cabac_), "Failed to set H264 CABAC"); + + // Insert Access Unit Delimiter (AUD) into encoded video stream + // This insert Network Abstraction Layer (NAL) unit into the encoded + // stream to explicitly show border of access unit (typically frame or picture) + CHECK_NVENC(encoder_->setInsertAudEnabled(true), "Failed to set inserting AUD for H264"); + + // Insert Sequence Parameter Set (SPS) and Picture Parameter Set (PPS) to + // each IDR frame so that decoders are able to decode stream from IDR frame surely + CHECK_NVENC( + encoder_->setInsertSpsPpsAtIdrEnabled(true), "Failed to set insertSPSPPSAtIDR for H264"); + + return EncResult::success(); + } + +private: + v4l2_mpeg_video_h264_profile h264_profile_; + v4l2_mpeg_video_h264_level h264_level_; + bool enable_cabac_; +}; + +std::unique_ptr make_jetson_h264_compressor() +{ + return std::make_unique(); +} +#else +std::unique_ptr make_jetson_h264_compressor() +{ + return nullptr; +} +#endif // JETSON_AVAILABLE +} // namespace accelerated_image_processor::compression diff --git a/src/accelerated_image_processor_compression/src/video_compressor/jetson_h265.cpp b/src/accelerated_image_processor_compression/src/video_compressor/jetson_h265.cpp new file mode 100644 index 0000000..9609acc --- /dev/null +++ b/src/accelerated_image_processor_compression/src/video_compressor/jetson_h265.cpp @@ -0,0 +1,184 @@ +// Copyright 2026 TIER IV, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "jetson.hpp" + +#include + +#include +#include +#include +#include + +namespace accelerated_image_processor::compression +{ +#ifdef JETSON_AVAILABLE +/** + * @brief H.265 encoder working on Jetson devices. + */ +class JetsonH265Compressor final : public JetsonVideoCompressor +{ +public: + inline static const std::unordered_map + h265_profile_map = { + {"MAIN", V4L2_MPEG_VIDEO_H265_PROFILE_MAIN}, + {"MAIN10", V4L2_MPEG_VIDEO_H265_PROFILE_MAIN10}, + }; + + inline static const std::unordered_map h265_level_map = { + {"1_0_MAIN_TIER", V4L2_MPEG_VIDEO_H265_LEVEL_1_0_MAIN_TIER}, + {"1_0_HIGH_TIER", V4L2_MPEG_VIDEO_H265_LEVEL_1_0_HIGH_TIER}, + {"2_0_MAIN_TIER", V4L2_MPEG_VIDEO_H265_LEVEL_2_0_MAIN_TIER}, + {"2_0_HIGH_TIER", V4L2_MPEG_VIDEO_H265_LEVEL_2_0_HIGH_TIER}, + {"2_1_MAIN_TIER", V4L2_MPEG_VIDEO_H265_LEVEL_2_1_MAIN_TIER}, + {"2_1_HIGH_TIER", V4L2_MPEG_VIDEO_H265_LEVEL_2_1_HIGH_TIER}, + {"3_0_MAIN_TIER", V4L2_MPEG_VIDEO_H265_LEVEL_3_0_MAIN_TIER}, + {"3_0_HIGH_TIER", V4L2_MPEG_VIDEO_H265_LEVEL_3_0_HIGH_TIER}, + {"3_1_MAIN_TIER", V4L2_MPEG_VIDEO_H265_LEVEL_3_1_MAIN_TIER}, + {"3_1_HIGH_TIER", V4L2_MPEG_VIDEO_H265_LEVEL_3_1_HIGH_TIER}, + {"4_0_MAIN_TIER", V4L2_MPEG_VIDEO_H265_LEVEL_4_0_MAIN_TIER}, + {"4_0_HIGH_TIER", V4L2_MPEG_VIDEO_H265_LEVEL_4_0_HIGH_TIER}, + {"4_1_MAIN_TIER", V4L2_MPEG_VIDEO_H265_LEVEL_4_1_MAIN_TIER}, + {"4_1_HIGH_TIER", V4L2_MPEG_VIDEO_H265_LEVEL_4_1_HIGH_TIER}, + {"5_0_MAIN_TIER", V4L2_MPEG_VIDEO_H265_LEVEL_5_0_MAIN_TIER}, + {"5_0_HIGH_TIER", V4L2_MPEG_VIDEO_H265_LEVEL_5_0_HIGH_TIER}, + {"5_1_MAIN_TIER", V4L2_MPEG_VIDEO_H265_LEVEL_5_1_MAIN_TIER}, + {"5_1_HIGH_TIER", V4L2_MPEG_VIDEO_H265_LEVEL_5_1_HIGH_TIER}, + {"5_2_MAIN_TIER", V4L2_MPEG_VIDEO_H265_LEVEL_5_2_MAIN_TIER}, + {"5_2_HIGH_TIER", V4L2_MPEG_VIDEO_H265_LEVEL_5_2_HIGH_TIER}, + {"6_0_MAIN_TIER", V4L2_MPEG_VIDEO_H265_LEVEL_6_0_MAIN_TIER}, + {"6_0_HIGH_TIER", V4L2_MPEG_VIDEO_H265_LEVEL_6_0_HIGH_TIER}, + {"6_1_MAIN_TIER", V4L2_MPEG_VIDEO_H265_LEVEL_6_1_MAIN_TIER}, + {"6_1_HIGH_TIER", V4L2_MPEG_VIDEO_H265_LEVEL_6_1_HIGH_TIER}, + {"6_2_MAIN_TIER", V4L2_MPEG_VIDEO_H265_LEVEL_6_2_MAIN_TIER}, + {"6_2_HIGH_TIER", V4L2_MPEG_VIDEO_H265_LEVEL_6_2_HIGH_TIER}, + }; + + JetsonH265Compressor() + : JetsonVideoCompressor( + SupportedCodec::H265, {{"h265.profile", static_cast("MAIN")}, + {"h265.level", static_cast("5_1_MAIN_TIER")}}) + { + } + + /** + * @brief override impelmentation to validate the compatibility of the current compression + * parameters. + */ + std::tuple validate_compression_type_compatibility() override + { + // Load the latest parameters if the encoder is uninitialized; use the current config otherwise + EncoderParameter latest_params; + if (this->state_ == State::UNINITIALIZED) { + this->collect_params(latest_params); + } else { + latest_params = this->encoder_params_; + } + + this->collect_codec_params_impl( + latest_params); // this udpates class member `h264_profile_` and `h264_level_` + + auto ret = validate_compression_type_compatibility_impl( + latest_params.compression_type, this->h265_profile_, this->h265_level_); + return {ret.ok, ret.status.message}; + } + +protected: + /** + * @brief class dedicated implementation to validate parameter compatibility + */ + EncResult validate_compression_type_compatibility_impl( + const VideoCompressionType & type, const v4l2_mpeg_video_h265_profile & profile, + const v4l2_mpeg_video_h265_level & level) + { + // lossless compression requires high bitrate. lower level forces encoder work in bitrate that + // is not enough for lossless compression, which may causes encoder initialization failure + // and/or FD DMA mapping for capture plane. Though it is not explicitly documented, this class + // treat the higher level is the only valid one for lossless. + if ( + type == VideoCompressionType::LOSSLESS && + (level != V4L2_MPEG_VIDEO_H265_LEVEL_6_1_HIGH_TIER && + level != V4L2_MPEG_VIDEO_H265_LEVEL_6_2_HIGH_TIER)) { + return EncResult(EncStatus( + false, + "Lossless compression is only supported with " + "level 6_1_HIGH_TIER or 6_2_HIGH_TIER")); + } + + return EncResult::success(); + } + + EncResult collect_codec_params_impl( + [[maybe_unused]] const EncoderParameter & general_params) override + { + h265_profile_ = string_to_enum( + this->parameter_value("h265.profile"), h265_profile_map); + h265_level_ = string_to_enum( + this->parameter_value("h265.level"), h265_level_map); + + return EncResult::success(); + } + + EncResult set_capture_plane_format_impl( + const uint32_t & width, const uint32_t & height, const uint32_t & image_size) override + { + CHECK_NVENC( + encoder_->setCapturePlaneFormat(V4L2_PIX_FMT_H265, width, height, image_size), + "Failed to set capture plane format to H265"); + + return EncResult::success(); + } + + EncResult init_codec_impl() override + { + CHECK_NVENC(encoder_->setProfile(h265_profile_), "Failed to set H265 profile"); + + CHECK_NVENC(encoder_->setLevel(h265_level_), "Failed to set H265 level"); + + // Set chroma format and bit depth + // This option is valid only for H.265 + uint8_t chroma_factor_idc = + (encoder_params_.compression_type == VideoCompressionType::LOSSLESS) ? 3 : 1; + CHECK_NVENC( + encoder_->setChromaFactorIDC(chroma_factor_idc), "Failed to set H265 chroma factor IDC"); + + // Insert Access Unit Delimiter (AUD) into encoded video stream + // This insert Network Abstraction Layer (NAL) unit into the encoded + // stream to explicitly show border of access unit (typically frame or picture) + CHECK_NVENC(encoder_->setInsertAudEnabled(true), "Failed to set inserting AUD for H265"); + + // Insert Sequence Parameter Set (SPS) and Picture Parameter Set (PPS) to + // each IDR frame so that decoders are able to decode stream from IDR frame surely + CHECK_NVENC( + encoder_->setInsertSpsPpsAtIdrEnabled(true), "Failed to set insertSPSPPSAtIDR for H265"); + + return EncResult::success(); + } + +private: + v4l2_mpeg_video_h265_profile h265_profile_; + v4l2_mpeg_video_h265_level h265_level_; +}; + +std::unique_ptr make_jetson_h265_compressor() +{ + return std::make_unique(); +} +#else +std::unique_ptr make_jetson_h265_compressor() +{ + return nullptr; +} +#endif // JETSON_AVAILABLE +} // namespace accelerated_image_processor::compression diff --git a/src/accelerated_image_processor_compression/src/video_compressor/jetson_precise_timestamp_map.hpp b/src/accelerated_image_processor_compression/src/video_compressor/jetson_precise_timestamp_map.hpp new file mode 100644 index 0000000..0ebb363 --- /dev/null +++ b/src/accelerated_image_processor_compression/src/video_compressor/jetson_precise_timestamp_map.hpp @@ -0,0 +1,63 @@ +// Copyright 2026 TIER IV, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +#pragma once +#include +#include +#include +#include + +namespace accelerated_image_processor::compression +{ +class TimestampMap +{ +public: + using PreciseTimestamp = int64_t; + +private: + // Helper class to pass nanosecond order timestamp. + // + // Because v4l2 buffers express its timestamp by the combination of tv_sec and tv_usec, which is + // not sufficient to express nanosecond level ROS timestamp, use this class to maintain input + // and output topic consistency + using history_t = std::queue; + +public: + explicit TimestampMap(const int v4l2_buffer_len) + { + // Allocate history (queue of timestamp) for each v4l2_buffer + timestamp_history_ = std::make_unique(v4l2_buffer_len); + } + + void set(const uint32_t & buf_index, const PreciseTimestamp & ts) + { + std::lock_guard lock(mutex_); + timestamp_history_[buf_index].push(ts); + } + + bool get(const uint32_t & buf_index, PreciseTimestamp & ts) + { + std::lock_guard lock(mutex_); + if (timestamp_history_[buf_index].empty()) { + return false; + } + ts = timestamp_history_[buf_index].front(); + timestamp_history_[buf_index].pop(); + return true; + } + +private: + std::unique_ptr timestamp_history_; + std::mutex mutex_; +}; +} // namespace accelerated_image_processor::compression diff --git a/src/accelerated_image_processor_compression/test/builder.cpp b/src/accelerated_image_processor_compression/test/builder.cpp index 6f04062..7aed4dc 100644 --- a/src/accelerated_image_processor_compression/test/builder.cpp +++ b/src/accelerated_image_processor_compression/test/builder.cpp @@ -14,22 +14,28 @@ #include "accelerated_image_processor_compression/builder.hpp" +#include "accelerated_image_processor_compression/compressor.hpp" #include "accelerated_image_processor_compression/jpeg_compressor.hpp" +#include "accelerated_image_processor_compression/video_compressor.hpp" #include #include #include +#include namespace accelerated_image_processor::compression { #ifdef JETSON_AVAILABLE -constexpr auto ExpectedJPEGBackend = JPEGBackend::JETSON; +constexpr auto ExpectedJPEGBackend = CompressorBackend::JETSON; +constexpr std::optional ExpectedVideoBackend = CompressorBackend::JETSON; #elif NVJPEG_AVAILABLE -constexpr auto ExpectedJPEGBackend = JPEGBackend::NVJPEG; +constexpr auto ExpectedJPEGBackend = CompressorBackend::NVJPEG; +constexpr std::optional ExpectedVideoBackend = std::nullopt; #else -constexpr auto ExpectedJPEGBackend = JPEGBackend::CPU; +constexpr auto ExpectedJPEGBackend = CompressorBackend::CPU; +constexpr std::optional ExpectedVideoBackend = std::nullopt; #endif namespace @@ -47,6 +53,25 @@ void check_compressor_type(const std::unique_ptr & compressor) EXPECT_EQ(ptr->backend(), ExpectedJPEGBackend); } +/** + * @brief Check compressor type by dynamic_cast (for video encode). + */ +[[maybe_unused]] void check_video_compressor_type( + [[maybe_unused]] const std::unique_ptr & compressor) +{ + if (ExpectedVideoBackend) { + EXPECT_NE(compressor, nullptr); + + auto ptr = dynamic_cast(compressor.get()); + EXPECT_NE(ptr, nullptr); + + EXPECT_EQ(ptr->backend(), ExpectedVideoBackend); + } else { + // This function should not be called under the non-Jetson platform + FAIL(); + } +} + /** * @brief Dummy class to register postprocess function. */ @@ -106,4 +131,135 @@ TEST(TestCompressorBuilder, CreateJPEGCompressor6) auto compressor = create_compressor("jpeg", &dummy_function); check_compressor_type(compressor); } + +#ifdef JETSON_AVAILABLE +TEST(TestCompressorBuilder, CreateH264Compressor1) +{ + auto compressor = create_compressor(CompressionType::H264); + check_video_compressor_type(compressor); +} + +TEST(TestCompressorBuilder, CreateH264Compressor2) +{ + auto compressor = create_compressor("h264"); + check_video_compressor_type(compressor); +} + +TEST(TestCompressorBuilder, CreateH264Compressor3) +{ + DummyClass dummy; + + auto compressor = + create_compressor(CompressionType::H264, &dummy); + check_video_compressor_type(compressor); +} + +TEST(TestCompressorBuilder, CreateH264Compressor4) +{ + auto compressor = create_compressor(CompressionType::H264, &dummy_function); + check_video_compressor_type(compressor); +} + +TEST(TestCompressorBuilder, CreateH264Compressor5) +{ + DummyClass dummy; + + auto compressor = create_compressor("h264", &dummy); + check_video_compressor_type(compressor); +} + +TEST(TestCompressorBuilder, CreateH264Compressor6) +{ + auto compressor = create_compressor("h264", &dummy_function); + check_video_compressor_type(compressor); +} + +TEST(TestCompressorBuilder, CreateH265Compressor1) +{ + auto compressor = create_compressor(CompressionType::H265); + check_video_compressor_type(compressor); +} + +TEST(TestCompressorBuilder, CreateH265Compressor2) +{ + auto compressor = create_compressor("h265"); + check_video_compressor_type(compressor); +} + +TEST(TestCompressorBuilder, CreateH265Compressor3) +{ + DummyClass dummy; + + auto compressor = + create_compressor(CompressionType::H265, &dummy); + check_video_compressor_type(compressor); +} + +TEST(TestCompressorBuilder, CreateH265Compressor4) +{ + auto compressor = create_compressor(CompressionType::H265, &dummy_function); + check_video_compressor_type(compressor); +} + +TEST(TestCompressorBuilder, CreateH265Compressor5) +{ + DummyClass dummy; + + auto compressor = create_compressor("h265", &dummy); + check_video_compressor_type(compressor); +} + +TEST(TestCompressorBuilder, CreateH265Compressor6) +{ + auto compressor = create_compressor("h265", &dummy_function); + check_video_compressor_type(compressor); +} + +TEST(TestCompressorBuilder, CreateAV1Compressor1) +{ + auto compressor = create_compressor(CompressionType::AV1); + check_video_compressor_type(compressor); +} + +TEST(TestCompressorBuilder, CreateAV1Compressor2) +{ + auto compressor = create_compressor("av1"); + check_video_compressor_type(compressor); +} + +TEST(TestCompressorBuilder, CreateAV1Compressor3) +{ + DummyClass dummy; + + auto compressor = + create_compressor(CompressionType::AV1, &dummy); + check_video_compressor_type(compressor); +} + +TEST(TestCompressorBuilder, CreateAV1Compressor4) +{ + auto compressor = create_compressor(CompressionType::AV1, &dummy_function); + check_video_compressor_type(compressor); +} + +TEST(TestCompressorBuilder, CreateAV1Compressor5) +{ + DummyClass dummy; + + auto compressor = create_compressor("av1", &dummy); + check_video_compressor_type(compressor); +} + +TEST(TestCompressorBuilder, CreateAV1Compressor6) +{ + auto compressor = create_compressor("av1", &dummy_function); + check_video_compressor_type(compressor); +} +#else +TEST(TestCompressorBuilderSkip, JetsonUnavailable) +{ + GTEST_SKIP() + << "Jetson not available. Skipping TestCompressorBuilder (for video compressor) tests."; +} +#endif } // namespace accelerated_image_processor::compression diff --git a/src/accelerated_image_processor_compression/test/jetson_video_compressor_av1.cpp b/src/accelerated_image_processor_compression/test/jetson_video_compressor_av1.cpp new file mode 100644 index 0000000..1dc4218 --- /dev/null +++ b/src/accelerated_image_processor_compression/test/jetson_video_compressor_av1.cpp @@ -0,0 +1,140 @@ +// Copyright 2026 TIER IV, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "accelerated_image_processor_compression/video_compressor.hpp" +#include "test_utility.hpp" + +#include + +#include +#include +#include + +#ifdef JETSON_AVAILABLE +namespace accelerated_image_processor::compression +{ +using AV1ParamCombination = std::tuple< + bool /* av1.enable_tile */, int /* av1.log2_num_tile_row */, int /* av1.log2_num_tile_col */, + bool /* av1.enable_ssim_rdo */, bool /* av1.enable_cdf_update */, + std::string /* compression_type */>; +using TestAV1Compressor = TestVideoCompressor; + +TEST_F(TestAV1Compressor, JetsonVideoCompressorAV1Default) +{ + auto compressor = make_jetson_av1_compressor(); + constexpr int desired_i_frame_interval = 10; + + compressor->register_postprocess< + TestAV1Compressor, + &TestAV1Compressor::check>(this); + + for (auto & [name, value] : compressor->parameters()) { + if (name == "i_frame_interval") { + value = desired_i_frame_interval; + } + } + + EXPECT_EQ(compressor->parameter_value("i_frame_interval"), desired_i_frame_interval); + + for (auto i = 0; i < TestAV1Compressor::NUM_FRAMES; i++) { + compressor->process(get_image()); + } +} + +TEST_P(TestAV1Compressor, JetsonVideoCompressorAV1ProfileLevelTypeCombo) +{ + auto + [enable_tile, log2_num_tile_row, log2_num_tile_col, enable_ssim_rdo, enable_cdf_update, type] = + GetParam(); + auto compressor = make_jetson_av1_compressor(); + constexpr int desired_i_frame_interval = 10; + compressor->register_postprocess< + TestAV1Compressor, + &TestAV1Compressor::check>(this); + + for (auto & [name, value] : compressor->parameters()) { + if (name == "i_frame_interval") { + value = desired_i_frame_interval; + } else if (name == "av1.enable_tile") { + value = enable_tile; + } else if (name == "av1.log2_num_tile_row") { + value = log2_num_tile_row; + } else if (name == "av1.log2_num_tile_col") { + value = log2_num_tile_col; + } else if (name == "av1.enable_ssim_rdo") { + value = enable_ssim_rdo; + } else if (name == "av1.enable_cdf_update") { + value = enable_cdf_update; + } else if (name == "compression_type") { + value = type; + } + } + EXPECT_EQ(compressor->parameter_value("i_frame_interval"), desired_i_frame_interval); + EXPECT_EQ(compressor->parameter_value("av1.enable_tile"), enable_tile); + EXPECT_EQ(compressor->parameter_value("av1.log2_num_tile_row"), log2_num_tile_row); + EXPECT_EQ(compressor->parameter_value("av1.log2_num_tile_col"), log2_num_tile_col); + EXPECT_EQ(compressor->parameter_value("av1.enable_ssim_rdo"), enable_ssim_rdo); + EXPECT_EQ(compressor->parameter_value("av1.enable_cdf_update"), enable_cdf_update); + + for (auto i = 0; i < TestAV1Compressor::NUM_FRAMES; i++) { + compressor->process(get_image()); + } +} + +INSTANTIATE_TEST_SUITE_P( + JetsonVideoCompressorAV1ComboWithTiling, TestAV1Compressor, + ::testing::Combine( + // Enable tiling + ::testing::Values(true), + // log2 num tile row + ::testing::Values( + 1, 2), // NOTE: Judging from the actual behavior, value >= 3 does not seem to be supported + // log2 num tile col + ::testing::Values( + 1, 2), // NOTE: Judging from the actual behavior, value >= 3 does not seem to be supported + // enable_ssim_rdo + ::testing::Bool(), + // enable_cdf_update + ::testing::Bool(), + // lossy or lossless + ::testing::Values("lossy", "lossless"))); + +INSTANTIATE_TEST_SUITE_P( + JetsonVideoCompressorAV1ComboWithoutTiling, TestAV1Compressor, + ::testing::Combine( + // Disable tiling + ::testing::Values(false), + // log2 num tile row (ignored) + ::testing::Values(0), + // log2 num tile col (ignored) + ::testing::Values(0), + // enable_ssim_rdo + ::testing::Bool(), + // enable_cdf_update + ::testing::Bool(), + // lossy or lossless + ::testing::Values("lossy", "lossless"))); + +} // namespace accelerated_image_processor::compression +#else +TEST(JetsonVideoCompressorAV1Skip, JetsonUnavailable) +{ + GTEST_SKIP() << "Jetson not available. Skipping JetsonVideoCompressorAV1 tests."; +} +#endif +int main(int argc, char ** argv) +{ + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/src/accelerated_image_processor_compression/test/jetson_video_compressor_h264.cpp b/src/accelerated_image_processor_compression/test/jetson_video_compressor_h264.cpp new file mode 100644 index 0000000..784d126 --- /dev/null +++ b/src/accelerated_image_processor_compression/test/jetson_video_compressor_h264.cpp @@ -0,0 +1,113 @@ +// Copyright 2026 TIER IV, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "accelerated_image_processor_compression/video_compressor.hpp" +#include "test_utility.hpp" + +#include + +#include +#include +#include + +#ifdef JETSON_AVAILABLE +namespace accelerated_image_processor::compression +{ +using H264ParamCombination = std::tuple< + std::string /* h264.profile */, std::string /* h264.level */, std::string /* compression_type */>; +using TestH264Compressor = TestVideoCompressor; + +TEST_F(TestH264Compressor, JetsonVideoCompressorH264Default) +{ + auto compressor = make_jetson_h264_compressor(); + constexpr int desired_i_frame_interval = 10; + + compressor->register_postprocess< + TestH264Compressor, + &TestH264Compressor::check>(this); + + for (auto & [name, value] : compressor->parameters()) { + if (name == "i_frame_interval") { + value = desired_i_frame_interval; + } + } + + EXPECT_EQ(compressor->parameter_value("i_frame_interval"), desired_i_frame_interval); + + for (auto i = 0; i < TestH264Compressor::NUM_FRAMES; i++) { + compressor->process(get_image()); + } +} + +TEST_P(TestH264Compressor, JetsonVideoCompressorH264ProfileLevelTypeCombo) +{ + auto [profile, level, type] = GetParam(); + auto compressor = make_jetson_h264_compressor(); + constexpr int desired_i_frame_interval = 10; + compressor->register_postprocess< + TestH264Compressor, + &TestH264Compressor::check>(this); + + for (auto & [name, value] : compressor->parameters()) { + if (name == "i_frame_interval") { + value = desired_i_frame_interval; + } else if (name == "h264.profile") { + value = profile; + } else if (name == "h264.level") { + value = level; + } else if (name == "compression_type") { + value = type; + } + } + EXPECT_EQ(compressor->parameter_value("i_frame_interval"), desired_i_frame_interval); + EXPECT_EQ(compressor->parameter_value("h264.profile"), profile); + EXPECT_EQ(compressor->parameter_value("h264.level"), level); + EXPECT_EQ(compressor->parameter_value("compression_type"), type); + + auto [is_valid_combination, msg] = compressor->validate_compression_type_compatibility(); + + for (auto i = 0; i < TestH264Compressor::NUM_FRAMES; i++) { + if (!is_valid_combination) { + EXPECT_THROW(compressor->process(get_image()), std::runtime_error); + SUCCEED(); // If exception throw is correctly detected, this test case is success + } else { + EXPECT_NO_THROW(compressor->process(get_image())); + } + } +} + +INSTANTIATE_TEST_SUITE_P( + JetsonVideoCompressorH264Combo, TestH264Compressor, + ::testing::Combine( + // Available profiles + ::testing::Values("BASELINE", "MAIN", "HIGH", "HIGH_444"), + // Avaiable levels + ::testing::Values( + "1_0", "1B", "1_1", "1_2", "1_3", "2_0", "2_1", "2_2", "3_0", "3_1", "3_2", "4_0", "4_1", + "4_2", "5_0", "5_1"), + // lossy or lossless + ::testing::Values("lossy", "lossless"))); + +} // namespace accelerated_image_processor::compression +#else +TEST(JetsonVideoCompressorH264Skip, JetsonUnavailable) +{ + GTEST_SKIP() << "Jetson not available. Skipping JetsonVideoCompressorH264 tests."; +} +#endif +int main(int argc, char ** argv) +{ + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/src/accelerated_image_processor_compression/test/jetson_video_compressor_h265.cpp b/src/accelerated_image_processor_compression/test/jetson_video_compressor_h265.cpp new file mode 100644 index 0000000..a9875fb --- /dev/null +++ b/src/accelerated_image_processor_compression/test/jetson_video_compressor_h265.cpp @@ -0,0 +1,117 @@ +// Copyright 2026 TIER IV, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "accelerated_image_processor_compression/video_compressor.hpp" +#include "test_utility.hpp" + +#include + +#include +#include +#include + +#ifdef JETSON_AVAILABLE +namespace accelerated_image_processor::compression +{ +using H265ParamCombination = std::tuple< + std::string /* h265.profile */, std::string /* h265.level */, std::string /* compression_type */>; +using TestH265Compressor = TestVideoCompressor; + +TEST_F(TestH265Compressor, JetsonVideoCompressorH265Default) +{ + auto compressor = make_jetson_h265_compressor(); + constexpr int desired_i_frame_interval = 10; + + compressor->register_postprocess< + TestH265Compressor, + &TestH265Compressor::check>(this); + + for (auto & [name, value] : compressor->parameters()) { + if (name == "i_frame_interval") { + value = desired_i_frame_interval; + } + } + + EXPECT_EQ(compressor->parameter_value("i_frame_interval"), desired_i_frame_interval); + + for (auto i = 0; i < TestH265Compressor::NUM_FRAMES; i++) { + compressor->process(get_image()); + } +} + +TEST_P(TestH265Compressor, JetsonVideoCompressorH265ProfileLevelTypeCombo) +{ + auto [profile, level, type] = GetParam(); + auto compressor = make_jetson_h265_compressor(); + constexpr int desired_i_frame_interval = 10; + compressor->register_postprocess< + TestH265Compressor, + &TestH265Compressor::check>(this); + + for (auto & [name, value] : compressor->parameters()) { + if (name == "i_frame_interval") { + value = desired_i_frame_interval; + } else if (name == "h265.profile") { + value = profile; + } else if (name == "h265.level") { + value = level; + } else if (name == "compression_type") { + value = type; + } + } + EXPECT_EQ(compressor->parameter_value("i_frame_interval"), desired_i_frame_interval); + EXPECT_EQ(compressor->parameter_value("h265.profile"), profile); + EXPECT_EQ(compressor->parameter_value("h265.level"), level); + EXPECT_EQ(compressor->parameter_value("compression_type"), type); + + auto [is_valid_combination, msg] = compressor->validate_compression_type_compatibility(); + + for (auto i = 0; i < TestH265Compressor::NUM_FRAMES; i++) { + if (!is_valid_combination) { + EXPECT_THROW(compressor->process(get_image()), std::runtime_error); + SUCCEED(); // If exception throw is correctly detected, this test case is success + } else { + EXPECT_NO_THROW(compressor->process(get_image())); + } + } +} + +INSTANTIATE_TEST_SUITE_P( + JetsonVideoCompressorH265Combo, TestH265Compressor, + ::testing::Combine( + // Available profiles + ::testing::Values("MAIN", "MAIN10"), + // Available levels + ::testing::Values( + "1_0_MAIN_TIER", "1_0_HIGH_TIER", "2_0_MAIN_TIER", "2_0_HIGH_TIER", "2_1_MAIN_TIER", + "2_1_HIGH_TIER", "3_0_MAIN_TIER", "3_0_HIGH_TIER", "3_1_MAIN_TIER", "3_1_HIGH_TIER", + "4_0_MAIN_TIER", "4_0_HIGH_TIER", "4_1_MAIN_TIER", "4_1_HIGH_TIER", "5_0_MAIN_TIER", + "5_0_HIGH_TIER", "5_1_MAIN_TIER", "5_1_HIGH_TIER", "5_2_MAIN_TIER", "5_2_HIGH_TIER", + "6_0_MAIN_TIER", "6_0_HIGH_TIER", "6_1_MAIN_TIER", "6_1_HIGH_TIER", "6_2_MAIN_TIER", + "6_2_HIGH_TIER"), + // lossy or lossless + ::testing::Values("lossy", "lossless"))); + +} // namespace accelerated_image_processor::compression +#else +TEST(JetsonVideoCompressorH265Skip, JetsonUnavailable) +{ + GTEST_SKIP() << "Jetson not available. Skipping JetsonVideoCompressorH265 tests."; +} +#endif +int main(int argc, char ** argv) +{ + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/src/accelerated_image_processor_compression/test/test_utility.hpp b/src/accelerated_image_processor_compression/test/test_utility.hpp index 0789953..60d2d4b 100644 --- a/src/accelerated_image_processor_compression/test/test_utility.hpp +++ b/src/accelerated_image_processor_compression/test/test_utility.hpp @@ -18,6 +18,9 @@ #include +#include +#include +#include #include namespace accelerated_image_processor::compression @@ -72,4 +75,101 @@ class TestJPEGCompressor : public ::testing::Test private: common::Image image_; }; + +template +class TestVideoCompressor : public ::testing::TestWithParam +{ +public: + static constexpr int NUM_FRAMES = 20; + + void SetUp() override + { + for (uint32_t i = 0; i < images_.size(); i++) { + auto & image = images_[i]; + image.frame_id = frame_id; + image.timestamp = + timestamp + (i * 100 * 100'000'000ULL); // + (i * 100ms), which emurate 10fps + image.width = width; + image.height = height; + image.step = step; + image.encoding = encoding; + image.format = format; + image.data.resize(image.step * image.height); + + for (uint32_t y = 0; y < image.height; ++y) { + for (uint32_t x = 0; x < image.width; ++x) { + image.data[y * image.step + x * 3 + 0] = static_cast((x + i) % 256); + image.data[y * image.step + x * 3 + 1] = static_cast((y + i) % 256); + image.data[y * image.step + x * 3 + 2] = static_cast((x + y + i) % 256); + } + } + } + + index_ = 0; + image_size_ = images_[0].data.size(); // Save representative image size + frame_from_first_i_frame_ = std::nullopt; + num_received_frame_ = 0; + } + + const std::string frame_id = "camera"; + const int64_t timestamp = 123456789; + const uint32_t width = 1920; + const uint32_t height = 1080; + const uint32_t step = width * 3; + const common::ImageEncoding encoding = common::ImageEncoding::RGB; + const common::ImageFormat format = common::ImageFormat::RAW; + + const common::Image & get_image() + { + // Yield image one by one + if (index_ < images_.size()) { + return images_[index_++]; + } + throw std::out_of_range("TestVideoCompressor's generator exhausted."); + } + + template + void check(const common::Image & result) + { + EXPECT_EQ(result.frame_id, frame_id); + EXPECT_EQ(result.timestamp, timestamp + (num_received_frame_++ * 100 * 100'000'000ULL)); + EXPECT_EQ(result.height, height); + EXPECT_EQ(result.width, width); + EXPECT_EQ(result.format, Fmt); + // expect the compressed data size to be smaller than the original image data size, but not 0 + EXPECT_GT(result.data.size(), 0U); + EXPECT_LE(result.data.size(), image_size_); + // expect the pts field has values larger than zero + EXPECT_TRUE(result.pts.has_value()); + EXPECT_GT(result.pts.value(), 0); + // expect flag should be 0 or 1 + EXPECT_TRUE(result.flags.has_value()); + if (result.flags.value() == 1 && frame_from_first_i_frame_ == std::nullopt) { + // The first I frame is observed. start counting + frame_from_first_i_frame_ = 0; + } else { + (*frame_from_first_i_frame_)++; + } + + if (!frame_from_first_i_frame_) { + // The first I frame has not been observed. + EXPECT_EQ(result.flags.value(), 0); + } else if ( + frame_from_first_i_frame_.has_value() && + frame_from_first_i_frame_.value() % IFrameInterval != 0) { + // The first I frame has been observed, and this frame should not be I frames + EXPECT_EQ(result.flags.value(), 0); + } else { + // This should be the I frame + EXPECT_EQ(result.flags.value(), 1); + } + } + +private: + std::array images_; + size_t index_; + size_t image_size_; + std::optional frame_from_first_i_frame_; + uint64_t num_received_frame_; +}; } // namespace accelerated_image_processor::compression diff --git a/src/accelerated_image_processor_ros/include/accelerated_image_processor_ros/conversion.hpp b/src/accelerated_image_processor_ros/include/accelerated_image_processor_ros/conversion.hpp index 0659422..04eba83 100644 --- a/src/accelerated_image_processor_ros/include/accelerated_image_processor_ros/conversion.hpp +++ b/src/accelerated_image_processor_ros/include/accelerated_image_processor_ros/conversion.hpp @@ -17,6 +17,7 @@ #include #include +#include #include #include #include @@ -146,4 +147,20 @@ std::string to_ros_distortion_model(common::DistortionModel model); * @return sensor_msgs::msg::RegionOfInterest */ sensor_msgs::msg::RegionOfInterest to_ros_roi(const common::Roi & roi); + +/// --- From common::Image to ffmpeg_image_transport_msgs::msg::FFMPEGPacket --- + +/** + * @brief Convert common::Image to ffmpeg_image_transport_msgs::msg::FFMPEGPacket. + * @param image common::Image message + * @return ffmpeg_image_transport_msgs::msg::FFMPEGPacket + */ +ffmpeg_image_transport_msgs::msg::FFMPEGPacket to_ros_ffmpeg(const common::Image & image); + +/** + * @brief Convert common::ImageFormat to ffmpeg_image_transport_msgs::msg::FFMPEGPacket encoding. + * @param format Format of common::Image. + * @return std::string + */ +std::string to_ros_ffmpeg_encoding(common::ImageFormat format); } // namespace accelerated_image_processor::ros diff --git a/src/accelerated_image_processor_ros/package.xml b/src/accelerated_image_processor_ros/package.xml index 19f2ed2..4d97365 100644 --- a/src/accelerated_image_processor_ros/package.xml +++ b/src/accelerated_image_processor_ros/package.xml @@ -13,6 +13,7 @@ accelerated_image_processor_compression accelerated_image_processor_pipeline builtin_interfaces + ffmpeg_image_transport_msgs rclcpp rclcpp_components sensor_msgs diff --git a/src/accelerated_image_processor_ros/src/conversion.cpp b/src/accelerated_image_processor_ros/src/conversion.cpp index abe36d4..e772682 100644 --- a/src/accelerated_image_processor_ros/src/conversion.cpp +++ b/src/accelerated_image_processor_ros/src/conversion.cpp @@ -202,4 +202,33 @@ sensor_msgs::msg::RegionOfInterest to_ros_roi(const common::Roi & roi) .width(roi.width) .do_rectify(roi.do_rectify); } + +/// --- From common::Image to ffmpeg_image_transport_msgs::msg::FFMPEGPacket --- + +std::string to_ros_ffmpeg_encoding(common::ImageFormat format) +{ + switch (format) { + case common::ImageFormat::H264: + return "h264"; + case common::ImageFormat::H265: + return "hevc"; + case common::ImageFormat::AV1: + return "av1"; + default: + throw std::runtime_error("Unsupported format: " + std::to_string(static_cast(format))); + } +} + +ffmpeg_image_transport_msgs::msg::FFMPEGPacket to_ros_ffmpeg(const common::Image & image) +{ + return ffmpeg_image_transport_msgs::build() + .header(to_ros_header(image.timestamp, image.frame_id)) + .width(image.width) + .height(image.height) + .encoding(to_ros_ffmpeg_encoding(image.format)) + .pts(image.pts.value()) + .flags(image.flags.value()) + .is_bigendian(image.is_bigendian) + .data(image.data); +} } // namespace accelerated_image_processor::ros diff --git a/src/accelerated_image_processor_ros/src/imgproc_node.cpp b/src/accelerated_image_processor_ros/src/imgproc_node.cpp index c1bf739..514a3f8 100644 --- a/src/accelerated_image_processor_ros/src/imgproc_node.cpp +++ b/src/accelerated_image_processor_ros/src/imgproc_node.cpp @@ -31,6 +31,9 @@ ImgProcNode::ImgProcNode(const rclcpp::NodeOptions & options) : Node("imgproc_no auto compression_type = this->declare_parameter("compressor.type"); auto do_rectify = this->declare_parameter("rectifier.do_rectify"); + use_jpeg_compression_ = + compression::to_compression_type(compression_type) == compression::CompressionType::JPEG; + // raw compressor { raw_compressor_ = compression::create_compressor( @@ -78,8 +81,13 @@ void ImgProcNode::determine_qos(const bool do_rectify, const int max_task_length image_topic, image_qos, [this](const sensor_msgs::msg::Image::ConstSharedPtr msg) { this->on_image(msg); }); - compressed_publisher_ = - this->create_publisher("image_raw/compressed", image_qos); + if (use_jpeg_compression_) { + compressed_publisher_ = + this->create_publisher("image_raw/compressed", image_qos); + } else { + compressed_publisher_ = this->create_publisher( + "image_raw/compressed", image_qos); + } compression_worker_.emplace(max_task_length); @@ -93,8 +101,14 @@ void ImgProcNode::determine_qos(const bool do_rectify, const int max_task_length this->create_publisher("image_rect", image_qos); rectified_info_publisher_ = this->create_publisher("image_rect/camera_info", info_qos); - rectified_compressed_publisher_ = - this->create_publisher("image_rect/compressed", image_qos); + if (use_jpeg_compression_) { + rectified_compressed_publisher_ = this->create_publisher( + "image_rect/compressed", image_qos); + } else { + rectified_compressed_publisher_ = + this->create_publisher( + "image_rect/compressed", image_qos); + } rectification_worker_.emplace(max_task_length); } @@ -136,8 +150,17 @@ void ImgProcNode::on_camera_info(const sensor_msgs::msg::CameraInfo::ConstShared void ImgProcNode::publish_compressed(const common::Image & image) { - auto compressed = to_ros_compressed(image); - compressed_publisher_->publish(std::move(compressed)); + if (use_jpeg_compression_) { + auto compressed = to_ros_compressed(image); + std::dynamic_pointer_cast>( + compressed_publisher_) + ->publish(std::move(compressed)); + } else { + auto compressed = to_ros_ffmpeg(image); + std::dynamic_pointer_cast>( + compressed_publisher_) + ->publish(std::move(compressed)); + } } void ImgProcNode::publish_rectified_raw(const common::Image & image) @@ -150,8 +173,17 @@ void ImgProcNode::publish_rectified_raw(const common::Image & image) void ImgProcNode::publish_rectified_compressed(const common::Image & image) { - auto compressed = to_ros_compressed(image); - rectified_compressed_publisher_->publish(std::move(compressed)); + if (use_jpeg_compression_) { + auto compressed = to_ros_compressed(image); + std::dynamic_pointer_cast>( + rectified_compressed_publisher_) + ->publish(std::move(compressed)); + } else { + auto compressed = to_ros_ffmpeg(image); + std::dynamic_pointer_cast>( + rectified_compressed_publisher_) + ->publish(std::move(compressed)); + } } } // namespace accelerated_image_processor::ros diff --git a/src/accelerated_image_processor_ros/src/imgproc_node.hpp b/src/accelerated_image_processor_ros/src/imgproc_node.hpp index ce12ba5..e84502c 100644 --- a/src/accelerated_image_processor_ros/src/imgproc_node.hpp +++ b/src/accelerated_image_processor_ros/src/imgproc_node.hpp @@ -19,8 +19,10 @@ #include #include #include +#include #include +#include #include #include #include @@ -83,15 +85,17 @@ class ImgProcNode : public rclcpp::Node rclcpp::Subscription::SharedPtr image_subscription_; rclcpp::Subscription::SharedPtr info_subscription_; - rclcpp::Publisher::SharedPtr compressed_publisher_; + rclcpp::PublisherBase::SharedPtr compressed_publisher_; rclcpp::Publisher::SharedPtr rectified_raw_publisher_; - rclcpp::Publisher::SharedPtr rectified_compressed_publisher_; + rclcpp::PublisherBase::SharedPtr rectified_compressed_publisher_; rclcpp::Publisher::SharedPtr rectified_info_publisher_; rclcpp::TimerBase::SharedPtr qos_request_timer_; std::optional compression_worker_; std::optional rectification_worker_; + + bool use_jpeg_compression_; }; } // namespace accelerated_image_processor::ros diff --git a/src/accelerated_image_processor_ros/test/conversion.cpp b/src/accelerated_image_processor_ros/test/conversion.cpp index 3a615e6..85fca45 100644 --- a/src/accelerated_image_processor_ros/test/conversion.cpp +++ b/src/accelerated_image_processor_ros/test/conversion.cpp @@ -356,6 +356,64 @@ TEST(TestConversionToRosFormat, Png) EXPECT_EQ(result, "png"); } +TEST(TestConversionToRosFFmpeg, CopyFieldsAndVideo) +{ + common::Image image; + image.frame_id = "camera"; + image.timestamp = 123'000'000'000LL + 456LL; + image.height = 480; + image.width = 640; + image.format = common::ImageFormat::H264; + image.data = {9, 8, 7, 6}; + image.pts = 123456789ULL; + image.flags = 1; + image.is_bigendian = true; + + auto pkt = to_ros_ffmpeg(image); + + EXPECT_EQ(pkt.header.frame_id, image.frame_id); + EXPECT_EQ(pkt.header.stamp.sec, 123); + EXPECT_EQ(pkt.header.stamp.nanosec, 456); + EXPECT_EQ(pkt.width, image.width); + EXPECT_EQ(pkt.height, image.height); + EXPECT_EQ(pkt.encoding, "h264"); + EXPECT_EQ(pkt.pts, image.pts.value()); + EXPECT_EQ(pkt.flags, image.flags.value()); + EXPECT_EQ(pkt.is_bigendian, image.is_bigendian); + EXPECT_EQ(pkt.data, image.data); +} + +TEST(TestConversionToRosFFmpegEncoding, ValidEncodings) +{ + struct + { + common::ImageFormat fmt; + std::string expected; + } cases[] = { + {common::ImageFormat::RAW, "raw"}, {common::ImageFormat::JPEG, "jpeg"}, + {common::ImageFormat::PNG, "png"}, {common::ImageFormat::H264, "h264"}, + {common::ImageFormat::H265, "hevc"}, {common::ImageFormat::AV1, "av1"}, + }; + + auto is_supported_encoding = [](const common::ImageFormat & fmt) { + if ( + fmt == common::ImageFormat::H264 || fmt == common::ImageFormat::H265 || + fmt == common::ImageFormat::AV1) { + return true; + } else { + return false; + } + }; + + for (const auto & c : cases) { + if (is_supported_encoding(c.fmt)) { + EXPECT_EQ(to_ros_ffmpeg_encoding(c.fmt), c.expected); + } else { + EXPECT_THROW(to_ros_ffmpeg_encoding(c.fmt), std::runtime_error); + } + } +} + TEST(TestConversionToRosInfo, CopyFields) { common::CameraInfo info;