diff --git a/be/test/util/block_compression_test.cpp b/be/test/util/block_compression_test.cpp index f430a0274e5f93..870345e6978c7f 100644 --- a/be/test/util/block_compression_test.cpp +++ b/be/test/util/block_compression_test.cpp @@ -22,6 +22,7 @@ #include #include +#include #include #include "gtest/gtest_pred_impl.h" @@ -68,10 +69,10 @@ void test_single_slice(segment_v2::CompressionTypePB type) { st = codec->decompress(compressed_slice, &uncompressed_slice); EXPECT_TRUE(st.ok()); - EXPECT_STREQ(orig.c_str(), uncompressed.c_str()); + EXPECT_EQ(orig, uncompressed); } // buffer not enough for decompress - // snappy has no return value if given buffer is not enough + // Snappy's capacity validation is covered in snappy_invalid_input. // NOTE: For ZLIB, we even get OK with a insufficient output // when uncompressed size is 1 if ((type == segment_v2::CompressionTypePB::ZLIB && uncompressed.size() > 1) && @@ -134,7 +135,7 @@ void test_multi_slices(segment_v2::CompressionTypePB type) { st = codec->decompress(compressed_slice, &uncompressed_slice); EXPECT_TRUE(st.ok()); - EXPECT_STREQ(orig.c_str(), uncompressed.c_str()); + EXPECT_EQ(orig, uncompressed); } } } @@ -148,4 +149,85 @@ TEST_F(BlockCompressionTest, multi) { test_multi_slices(segment_v2::CompressionTypePB::ZSTD); } +static void check_snappy_decompression(BlockCompressionCodec* codec, const faststring& compressed, + const std::string& original) { + std::string restored(original.size(), '\0'); + Slice output(restored); + ASSERT_TRUE(codec->decompress(Slice(compressed), &output).ok()); + EXPECT_EQ(original.size(), output.size); + EXPECT_EQ(original, restored); +} + +TEST_F(BlockCompressionTest, snappy_binary_and_block_boundaries) { + BlockCompressionCodec* codec = nullptr; + ASSERT_TRUE(get_block_compression_codec(segment_v2::CompressionTypePB::SNAPPY, &codec).ok()); + std::mt19937 random(0); + // Snappy compresses in 64 KiB blocks. Exercise both sides of that boundary, + // including binary data whose embedded NULs must not truncate comparisons. + for (size_t size : {0, 1, 63, 64, 65, 65535, 65536, 65537, 1048576}) { + SCOPED_TRACE(size); + for (bool compressible : {false, true}) { + SCOPED_TRACE(compressible); + std::string original(size, '\0'); + for (size_t i = 0; i < size; ++i) { + original[i] = static_cast(compressible ? i % 8 : random() & 0xff); + } + faststring compressed; + ASSERT_TRUE(codec->compress(original, &compressed).ok()); + check_snappy_decompression(codec, compressed, original); + + // Empty slices at every position and uneven boundaries force the + // Source adapter to advance across slices within a Snappy block. + size_t split = size / 3; + std::vector slices = {Slice(), Slice(original.data(), split), Slice(), + Slice(original.data() + split, size - split), Slice()}; + ASSERT_TRUE(codec->compress(slices, size, &compressed).ok()); + check_snappy_decompression(codec, compressed, original); + } + } +} + +TEST_F(BlockCompressionTest, snappy_invalid_input) { + BlockCompressionCodec* codec = nullptr; + ASSERT_TRUE(get_block_compression_codec(segment_v2::CompressionTypePB::SNAPPY, &codec).ok()); + const std::string original = "snappy block compression"; + faststring compressed; + ASSERT_TRUE(codec->compress(original, &compressed).ok()); + std::string restored(original.size(), '\0'); + for (size_t size = 0; size < compressed.size(); ++size) { + SCOPED_TRACE(size); + Slice output(restored); + EXPECT_FALSE(codec->decompress(Slice(compressed.data(), size), &output).ok()); + } + restored.assign(original.size(), '\0'); + Slice output(restored.data(), restored.size() - 1); + EXPECT_FALSE(codec->decompress(Slice(compressed), &output).ok()); + EXPECT_EQ(std::string(original.size(), '\0'), restored); + + // An unterminated length varint and a copy with no preceding literal. + for (const auto& invalid : {std::string("\x80", 1), std::string("\x04\x01\x01", 3)}) { + output = Slice(restored); + EXPECT_FALSE(codec->decompress(invalid, &output).ok()); + } +} + +TEST_F(BlockCompressionTest, snappy_1_1_10_compatibility) { + BlockCompressionCodec* codec = nullptr; + ASSERT_TRUE(get_block_compression_codec(segment_v2::CompressionTypePB::SNAPPY, &codec).ok()); + // Frozen output of Snappy 1.1.10 RawCompress, including a literal containing + // NUL and 0xff, and a copy tag. Do not regenerate with the linked library. + const char compressed[] = + "\x41\x30\x73\x6e\x61\x70\x70\x79\x00\x62\x6c\x6f\x63\x6b" + "\xff\xce\x0d\x00"; + std::string expected; + for (int i = 0; i < 5; ++i) { + expected.append("snappy\0block\xff", 13); + } + std::string restored(expected.size() + 16, '\0'); + Slice output(restored); + ASSERT_TRUE(codec->decompress(Slice(compressed, sizeof(compressed) - 1), &output).ok()); + EXPECT_EQ(expected.size(), output.size); + EXPECT_EQ(expected, restored.substr(0, output.size)); +} + } // namespace doris diff --git a/thirdparty/CHANGELOG.md b/thirdparty/CHANGELOG.md index 9d9f55f27bb7de..f56ce806730827 100644 --- a/thirdparty/CHANGELOG.md +++ b/thirdparty/CHANGELOG.md @@ -2,6 +2,14 @@ This file contains version of the third-party dependency libraries in the build-env image. The docker build-env image is apache/doris, and the tag is `build-env-${version}` +## 20260909 + +- Modified: snappy 1.1.10 -> 1.2.1. Enable x86 SSE4.2 paths and AVX2 by default; + use `USE_AVX2=0` or `OFF` when building third-party libraries for a non-AVX2 BE. + Enable ARM CRC32 hashing through `ARM_MARCH` (default `armv8-a+crc`). + Retain RTTI for `SnappySlicesSource` and remove the sign-compare patch already + included upstream. + ## 20260824 - Modified: thrift 0.16.0 -> 0.24.0 diff --git a/thirdparty/build-thirdparty.sh b/thirdparty/build-thirdparty.sh index 5fd78684cffd95..6329b1276fcd59 100755 --- a/thirdparty/build-thirdparty.sh +++ b/thirdparty/build-thirdparty.sh @@ -616,12 +616,28 @@ build_snappy() { sed -i 's/-fno-rtti/-frtti/g' CMakeLists.txt fi + local snappy_cxx_flags="-O3" + case "$(uname -m)" in + x86_64) + # Match the BE's SSE4.2 baseline and optional AVX2 target. + snappy_cxx_flags+=" -msse4.2" + case "${USE_AVX2:-ON}" in + 0 | OFF | off | FALSE | false | NO | no) ;; + *) snappy_cxx_flags+=" -mavx2" ;; + esac + ;; + aarch64 | arm64) + # Match the BE ARM baseline so Snappy can use NEON CRC32 hashing. + snappy_cxx_flags+=" -march=${ARM_MARCH:-armv8-a+crc}" + ;; + esac + mkdir -p "${BUILD_DIR}" cd "${BUILD_DIR}" rm -rf CMakeCache.txt CMakeFiles/ - CFLAGS="-O3" CXXFLAGS="-O3" "${CMAKE_CMD}" -G "${GENERATOR}" -DCMAKE_INSTALL_PREFIX="${TP_INSTALL_DIR}" \ + CFLAGS="-O3" CXXFLAGS="${snappy_cxx_flags}" "${CMAKE_CMD}" -G "${GENERATOR}" -DCMAKE_INSTALL_PREFIX="${TP_INSTALL_DIR}" \ -DCMAKE_POLICY_VERSION_MINIMUM=3.5 \ -DCMAKE_POSITION_INDEPENDENT_CODE=ON \ -DCMAKE_INSTALL_INCLUDEDIR="${TP_INCLUDE_DIR}"/snappy \ diff --git a/thirdparty/download-thirdparty.sh b/thirdparty/download-thirdparty.sh index eb4aed22823e63..7beb47291f87f6 100755 --- a/thirdparty/download-thirdparty.sh +++ b/thirdparty/download-thirdparty.sh @@ -397,17 +397,6 @@ if [[ " ${TP_ARCHIVES[*]} " =~ " GLOG " ]]; then echo "Finished patching ${GLOG_SOURCE}" fi -# snappy patch to fix sign-compare warning -if [[ " ${TP_ARCHIVES[*]} " =~ " SNAPPY " ]]; then - cd "${TP_SOURCE_DIR}/${SNAPPY_SOURCE}" - if [[ ! -f "${PATCHED_MARK}" ]]; then - patch -p1 <"${TP_PATCH_DIR}/snappy-1.1.10-sign-compare.patch" - touch "${PATCHED_MARK}" - fi - cd - - echo "Finished patching ${SNAPPY_SOURCE}" -fi - # mysql patch if [[ " ${TP_ARCHIVES[*]} " =~ " MYSQL " ]]; then cd "${TP_SOURCE_DIR}/${MYSQL_SOURCE}" diff --git a/thirdparty/patches/snappy-1.1.10-sign-compare.patch b/thirdparty/patches/snappy-1.1.10-sign-compare.patch deleted file mode 100644 index 2ef772777429a4..00000000000000 --- a/thirdparty/patches/snappy-1.1.10-sign-compare.patch +++ /dev/null @@ -1,11 +0,0 @@ ---- a/snappy.cc -+++ b/snappy.cc -@@ -1290,7 +1290,7 @@ inline bool Copy64BytesWithPatternExtension(ptrdiff_t dst, size_t offset) { - DeferMemCopy(&deferred_src, &deferred_length, from, len); - } - } while (ip < ip_limit_min_slop && -- (op + deferred_length) < op_limit_min_slop); -+ static_cast(op + deferred_length) < op_limit_min_slop); - exit: - ip--; - assert(ip <= ip_limit); diff --git a/thirdparty/vars.sh b/thirdparty/vars.sh index 886a0eeff9f858..3bf11ff80cf7ce 100644 --- a/thirdparty/vars.sh +++ b/thirdparty/vars.sh @@ -99,10 +99,10 @@ GTEST_SOURCE=googletest-release-1.12.1 GTEST_MD5SUM="e82199374acdfda3f425331028eb4e2a" # snappy -SNAPPY_DOWNLOAD="https://github.com/google/snappy/archive/1.1.10.tar.gz" -SNAPPY_NAME=snappy-1.1.10.tar.gz -SNAPPY_SOURCE=snappy-1.1.10 -SNAPPY_MD5SUM="70153395ebe6d72febe2cf2e40026a44" +SNAPPY_DOWNLOAD="https://github.com/google/snappy/archive/1.2.1.tar.gz" +SNAPPY_NAME=snappy-1.2.1.tar.gz +SNAPPY_SOURCE=snappy-1.2.1 +SNAPPY_MD5SUM="dd6f9b667e69491e1dbf7419bdf68823" # gperftools GPERFTOOLS_DOWNLOAD="https://github.com/gperftools/gperftools/releases/download/gperftools-2.10/gperftools-2.10.tar.gz"