CI validation: COVER multi-key quicksort, rebased (do not merge)

.github/workflows/cmake-tests.yml

@@ -125,6 +125,12 @@ jobs:
             name: "Clang-CL"
             runner: "windows-2022"
             cmake_extra_flags: "-DCMAKE_COMPILE_WARNING_AS_ERROR=ON -DZSTD_BUILD_TESTS=ON"
+          - generator: "Visual Studio 17 2022"
+            flags: "-T ClangCL"
+            name: "Clang-CL (No ZSTD_BUILD_TESTS)"
+            runner: "windows-2022"
+            # Intentionally omit ZSTD_BUILD_TESTS: C++ is not enabled, so only the C compiler id is set
+            cmake_extra_flags: "-DCMAKE_COMPILE_WARNING_AS_ERROR=ON"
           - generator: "Visual Studio 17 2022"
             flags: "-T ClangCL -A x64 -DCMAKE_C_FLAGS=/arch:AVX2"
             name: "Clang-CL AVX2"
@@ -154,7 +160,7 @@ jobs:
   # Uses native ARM64 hardware for optimal performance and compatibility testing
   cmake-macos-arm64:
     name: "CMake macOS ARM64 (Apple Silicon)"
-    runs-on: macos-14  # ARM64 runner
+    runs-on: macos-latest  # ARM64 runner
     steps:
     - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # tag=v7.0.1
     - name: "CMake build and test (ARM64)"

.github/workflows/dev-short-tests.yml

@@ -637,7 +637,7 @@ jobs:
           sudo apt install -y musl-tools
       - name: Compile with musl-gcc and test-zstd
         run: |
-          CC=musl-gcc CFLAGS="-Werror -O3" CPPFLAGS=-DZDICT_QSORT=ZDICT_QSORT_C90 make -j -C tests test-zstd V=1
+          CC=musl-gcc CFLAGS="-Werror -O3" make -j -C tests test-zstd V=1
 
   intel-cet-compatibility:
     runs-on: ubuntu-latest

build/cmake/CMakeModules/AddZstdCompilationFlags.cmake

@@ -23,7 +23,7 @@ function(EnableCompilerFlag _flag _C _CXX _LD)
             set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${_flag}" PARENT_SCOPE)
         endif ()
     endif ()
-    if (_CXX AND CMAKE_CXX_COMPILER)
+    if (_CXX AND CMAKE_CXX_COMPILER_LOADED)
         CHECK_CXX_COMPILER_FLAG(${_flag} CXX_FLAG_${varname})
         if (CXX_FLAG_${varname})
             set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${_flag}" PARENT_SCOPE)
@@ -56,42 +56,42 @@ macro(ADD_ZSTD_COMPILATION_FLAGS _C _CXX _LD)
     # compiler flags to mark the stack as non-executable.
     set(ZSTD_HAS_NOEXECSTACK false)
 
-    if (CMAKE_CXX_COMPILER_ID MATCHES "GNU|Clang" OR MINGW) #Not only UNIX but also WIN32 for MinGW
+    if (CMAKE_C_COMPILER_ID MATCHES "GNU|Clang" OR CMAKE_CXX_COMPILER_ID MATCHES "GNU|Clang" OR MINGW) #Not only UNIX but also WIN32 for MinGW
         # It's possible to select the exact standard used for compilation.
         # It's not necessary, but can be employed for specific purposes.
         # Note that zstd source code is compatible with both C++98 and above
         # and C-gnu90 (c90 + long long + variadic macros ) and above
         # EnableCompilerFlag("-std=c++11" false true) # Set C++ compilation to c++11 standard
         # EnableCompilerFlag("-std=c99" true false)   # Set C compilation to c99 standard
-        if (CMAKE_CXX_COMPILER_ID MATCHES "Clang" AND MSVC)
+        if ((CMAKE_C_COMPILER_ID MATCHES "Clang" OR CMAKE_CXX_COMPILER_ID MATCHES "Clang") AND MSVC)
             # clang-cl normally maps -Wall to -Weverything.
-            EnableCompilerFlag("/clang:-Wall" _C _CXX false)
+            EnableCompilerFlag("/clang:-Wall" ${_C} ${_CXX} false)
         else ()
-            EnableCompilerFlag("-Wall" _C _CXX false)
+            EnableCompilerFlag("-Wall" ${_C} ${_CXX} false)
         endif ()
-        EnableCompilerFlag("-Wextra" _C _CXX false)
-        EnableCompilerFlag("-Wundef" _C _CXX false)
-        EnableCompilerFlag("-Wshadow" _C _CXX false)
-        EnableCompilerFlag("-Wcast-align" _C _CXX false)
-        EnableCompilerFlag("-Wcast-qual" _C _CXX false)
-        EnableCompilerFlag("-Wstrict-prototypes" _C false false)
+        EnableCompilerFlag("-Wextra" ${_C} ${_CXX} false)
+        EnableCompilerFlag("-Wundef" ${_C} ${_CXX} false)
+        EnableCompilerFlag("-Wshadow" ${_C} ${_CXX} false)
+        EnableCompilerFlag("-Wcast-align" ${_C} ${_CXX} false)
+        EnableCompilerFlag("-Wcast-qual" ${_C} ${_CXX} false)
+        EnableCompilerFlag("-Wstrict-prototypes" ${_C} false false)
         # Enable asserts in Debug mode
         if (CMAKE_BUILD_TYPE MATCHES "Debug")
-            EnableCompilerFlag("-DDEBUGLEVEL=1" _C _CXX false)
+            EnableCompilerFlag("-DDEBUGLEVEL=1" ${_C} ${_CXX} false)
         endif ()
         # Add noexecstack flags
         # LDFLAGS
-        EnableCompilerFlag("-Wl,-z,noexecstack" false false _LD)
+        EnableCompilerFlag("-Wl,-z,noexecstack" false false ${_LD})
         # CFLAGS & CXXFLAGS
-        EnableCompilerFlag("-Qunused-arguments" _C _CXX false)
-        EnableCompilerFlag("-Wa,--noexecstack" _C _CXX false)
+        EnableCompilerFlag("-Qunused-arguments" ${_C} ${_CXX} false)
+        EnableCompilerFlag("-Wa,--noexecstack" ${_C} ${_CXX} false)
         # NOTE: Using 3 nested ifs because the variables are sometimes
         # empty if the condition is false, and sometimes equal to false.
         # This implicitly converts them to truthy values. There may be
         # a better way to do this, but this reliably works.
-        if (${LD_FLAG_WL_Z_NOEXECSTACK})
-            if (${C_FLAG_WA_NOEXECSTACK})
-                if (${CXX_FLAG_WA_NOEXECSTACK})
+        if (LD_FLAG_WL_Z_NOEXECSTACK)
+            if (C_FLAG_WA_NOEXECSTACK)
+                if (NOT CMAKE_CXX_COMPILER_LOADED OR CXX_FLAG_WA_NOEXECSTACK)
                     # We've succeeded in marking the stack as non-executable
                     set(ZSTD_HAS_NOEXECSTACK true)
                 endif()

build/cmake/lib/CMakeLists.txt

@@ -98,7 +98,7 @@ if (ZSTD_LEGACY_SUPPORT)
             ${LIBRARY_LEGACY_DIR}/zstd_v07.h)
 endif ()
 
-if (MSVC AND NOT (CMAKE_CXX_COMPILER_ID STREQUAL "Clang"))
+if (MSVC AND NOT (CMAKE_C_COMPILER_ID STREQUAL "Clang"))
     set(MSVC_RESOURCE_DIR ${ZSTD_SOURCE_DIR}/build/VS2010/libzstd-dll)
     set(PlatformDependResources ${MSVC_RESOURCE_DIR}/libzstd-dll.rc)
 else()
@@ -189,7 +189,7 @@ if (MSVC)
 endif ()
 
 # With MSVC static library needs to be renamed to avoid conflict with import library
-if (MSVC OR (WIN32 AND CMAKE_CXX_COMPILER_ID STREQUAL "Clang" AND NOT MINGW))
+if (MSVC OR (WIN32 AND CMAKE_C_COMPILER_ID STREQUAL "Clang" AND NOT MINGW))
     set(STATIC_LIBRARY_BASE_NAME zstd_static)
 else ()
     set(STATIC_LIBRARY_BASE_NAME zstd)

contrib/linux-kernel/linux_zstd.h

@@ -677,7 +677,8 @@ typedef ZSTD_Sequence zstd_sequence;
  * @in_seqs_size: The number of sequences in in_seqs.
  * @literals: The literals associated to the sequences to be compressed.
  * @lit_size: The size of the literals in the literals buffer.
- * @lit_capacity: The size of the literals buffer.
+ * @lit_capacity: The size of the literals buffer, which must be at least
+ *                8 bytes larger than lit_size.
  * @decompressed_size: The size of the input data
  *
  * Return: The compressed size or an error, which can be checked using

contrib/seekable_format/tests/seekable_tests.c

@@ -6,6 +6,7 @@
 #include <string.h>
 
 #include "../zstd_seekable.h"
+#include "zstd_errors.h"
 
 
 /* ZSTD_seekable_customFile implementation that reads/seeks a buffer while keeping track of total bytes read */
@@ -365,6 +366,84 @@ int main(int argc, const char** argv)
     }
     printf("Success!\n");
 
+    printf("Test %u - reject seek table with numFrames beyond format max: ", testNb++);
+    {   /* A corrupt footer claiming numFrames > ZSTD_SEEKABLE_MAXFRAMES must be
+         * rejected up front: such a value overflows the seek table size math. */
+        const uint8_t compressed_data[17] = {
+            0x5e, 0x2a, 0x4d, 0x18,       /* skippable frame magic (| 0xE) */
+            0x09, 0x00, 0x00, 0x00,       /* skippable frame size = 9 */
+            0x01, 0x00, 0x00, 0x08,       /* footer: numFrames = 0x08000001 */
+            0x00,                         /* footer: seek table descriptor */
+            0xb1, 0xea, 0x92, 0x8f,       /* seekable magic number */
+        };
+
+        ZSTD_seekable* const stream = ZSTD_seekable_create();
+        assert(stream != NULL);
+        {   size_t const status = ZSTD_seekable_initBuff(stream, compressed_data, sizeof(compressed_data));
+            if (ZSTD_getErrorCode(status) != ZSTD_error_corruption_detected) {
+                ZSTD_seekable_free(stream);
+                goto _test_error;
+        }   }
+        ZSTD_seekable_free(stream);
+    }
+    printf("Success!\n");
+
+    printf("Test %u - decompress at and past the end of the data: ", testNb++);
+    {   char const inBuffer[] = "Lorem ipsum dolor sit amet";
+        size_t const inSize = sizeof(inBuffer);
+
+        size_t const seekCapacity = 5000;
+        void* const seekBuffer = malloc(seekCapacity);
+        assert(seekBuffer != NULL);
+        size_t seekSize;
+
+        char outBuffer[16];
+
+        ZSTD_seekable_CStream* const zscs = ZSTD_seekable_createCStream();
+        assert(zscs != NULL);
+        { size_t const initStatus = ZSTD_seekable_initCStream(zscs, 9, 0 /* checksumFlag */, 0 /* maxFrameSize */);
+          assert(!ZSTD_isError(initStatus));
+        }
+
+        {   ZSTD_outBuffer outb = { .dst=seekBuffer, .pos=0, .size=seekCapacity };
+            ZSTD_inBuffer inb = { .src=inBuffer, .pos=0, .size=inSize };
+
+            size_t const cStatus = ZSTD_seekable_compressStream(zscs, &outb, &inb);
+            assert(!ZSTD_isError(cStatus));
+            assert(inb.pos == inb.size);
+
+            size_t const endStatus = ZSTD_seekable_endStream(zscs, &outb);
+            assert(!ZSTD_isError(endStatus));
+            seekSize = outb.pos;
+        }
+
+        ZSTD_seekable* const stream = ZSTD_seekable_create();
+        assert(stream != NULL);
+        { size_t const initStatus = ZSTD_seekable_initBuff(stream, seekBuffer, seekSize);
+          assert(!ZSTD_isError(initStatus)); }
+
+        {   size_t const atEnd = ZSTD_seekable_decompress(stream, outBuffer, sizeof(outBuffer), inSize);
+            size_t const pastEnd = ZSTD_seekable_decompress(stream, outBuffer, sizeof(outBuffer), inSize + 5000);
+            int failed = atEnd != 0 || !ZSTD_isError(pastEnd);
+            if (!failed) {
+                /* offset + len wraps around to 0 */
+                size_t const wrapped = ZSTD_seekable_decompress(stream, outBuffer, sizeof(outBuffer), 0ULL - sizeof(outBuffer));
+                failed = !ZSTD_isError(wrapped);
+            }
+            if (failed) {
+                free(seekBuffer);
+                ZSTD_seekable_freeCStream(zscs);
+                ZSTD_seekable_free(stream);
+                goto _test_error;
+            }
+        }
+
+        free(seekBuffer);
+        ZSTD_seekable_freeCStream(zscs);
+        ZSTD_seekable_free(stream);
+    }
+    printf("Success!\n");
+
     /* TODO: Add more tests */
     printf("Finished tests\n");
     return 0;

contrib/seekable_format/zstd_seekable.h

@@ -152,6 +152,9 @@ ZSTDLIB_API size_t ZSTD_seekable_writeSeekTable(ZSTD_frameLog* fl, ZSTD_outBuffe
 *  consecutive range of data, it will efficiently retain the decompressor object
 *  and avoid redecompressing frame prefixes.  The return value is the number of
 *  bytes decompressed, or an error code checkable with ZSTD_isError().
+*  It is smaller than `dstSize` only when the data ends before
+*  `offset + dstSize`, and it is 0 when `offset` is the end of the data.
+*  An `offset` past the end of the data is an error.
 *
 *  The seek table access functions can be used to obtain the data contained
 *  in the seek table.  If frameIndex is larger than the value returned by

contrib/seekable_format/zstdseek_decompress.c

@@ -397,6 +397,13 @@ static size_t ZSTD_seekable_loadSeekTable(ZSTD_seekable* zs)
         U32 const frameSize = tableSize + ZSTD_seekTableFooterSize + ZSTD_SKIPPABLEHEADERSIZE;
 
         U32 remaining = frameSize - ZSTD_seekTableFooterSize; /* don't need to re-read footer */
+
+        /* numFrames is bounded by the format (the compressor never writes more
+         * than ZSTD_SEEKABLE_MAXFRAMES). A larger value can only come from a
+         * corrupt footer and overflows the tableSize/frameSize math above, so
+         * reject it before those values are used to seek or size the table. */
+        if (numFrames > ZSTD_SEEKABLE_MAXFRAMES) return ERROR(corruption_detected);
+
         {   U32 const toRead = MIN(remaining, SEEKABLE_BUFF_SIZE);
             CHECK_IO(src.seek(src.opaque, -(S64)frameSize, SEEK_END));
             CHECK_IO(src.read(src.opaque, zs->inBuff, toRead));
@@ -489,8 +496,9 @@ size_t ZSTD_seekable_initAdvanced(ZSTD_seekable* zs, ZSTD_seekable_customFile sr
 size_t ZSTD_seekable_decompress(ZSTD_seekable* zs, void* dst, size_t len, unsigned long long offset)
 {
     unsigned long long const eos = zs->seekTable.entries[zs->seekTable.tableLen].dOffset;
-    if (offset + len > eos) {
-        len = eos - offset;
+    if (offset > eos) return ERROR(parameter_outOfBound);
+    if (len > eos - offset) {
+        len = (size_t)(eos - offset);
     }
 
     U32 targetFrame = ZSTD_seekable_offsetToFrameIndex(zs, offset);

doc/zstd_compression_format.md

@@ -489,7 +489,7 @@ __`Size_Format`__
 - For `Raw_Literals_Block` and `RLE_Literals_Block`,
   it's only necessary to decode `Regenerated_Size`.
   There is no `Compressed_Size` field.
-- For `Compressed_Block` and `Treeless_Literals_Block`,
+- For `Compressed_Literals_Block` and `Treeless_Literals_Block`,
   it's required to decode both `Compressed_Size`
   and `Regenerated_Size` (the decompressed size).
   It's also necessary to decode the number of streams (1 or 4).

lib/README.md

@@ -208,11 +208,6 @@ The file structure is designed to make this selection manually achievable for an
   and assembly decoding loops. You may want to use this macro if these loops are
   slower on your platform.
 
-- The macro `ZDICT_QSORT` can enforce selection of a specific sorting variant,
-  which is useful when autodetection fails, for example with older versions of `musl`.
-  For this scenario, it can be set as `ZDICT_QSORT=ZDICT_QSORT_C90`.
-  Other selectable suffixes are `_GNU`, `_APPLE`, `_MSVC` and `_C11`.
-
 #### Windows : using MinGW+MSYS to create DLL
 
 DLL can be created using MinGW+MSYS with the `make libzstd` command.

lib/common/zstd_deps.h

@@ -24,18 +24,6 @@
 #ifndef ZSTD_DEPS_COMMON
 #define ZSTD_DEPS_COMMON
 
-/* Even though we use qsort_r only for the dictionary builder, the macro
- * _GNU_SOURCE has to be declared *before* the inclusion of any standard
- * header and the script 'combine.sh' combines the whole zstd source code
- * in a single file.
- */
-#if defined(__linux) || defined(__linux__) || defined(linux) || defined(__gnu_linux__) || \
-    defined(__CYGWIN__) || defined(__MSYS__)
-#if !defined(_GNU_SOURCE) && !defined(__ANDROID__) /* NDK doesn't ship qsort_r(). */
-#define _GNU_SOURCE
-#endif
-#endif
-
 #include <limits.h>
 #include <stddef.h>
 #include <string.h>