From 4f5dce98917adf9405af93b79f83e39258715150 Mon Sep 17 00:00:00 2001 From: Nick Terrell Date: Tue, 10 Nov 2026 10:12:19 -0800 Subject: [PATCH] compress: validate repcodes against retained dictionary content Full-format dictionaries may contain repeat offsets into content discarded by the short-cache CDict limit or the general index limit. Revalidate all three repcodes after loading the retained suffix and enforce the same bound before attaching a CDict, preventing wrapped dictMatchState indices and wild reads. Add regression coverage for 20 MiB dictionaries, every repcode slot, fast/dfast levels, deferred and streaming dictionary loading, the short-cache boundary, and valid non-truncating controls. --- lib/compress/zstd_compress.c | 88 ++++++++-- lib/compress/zstd_double_fast.c | 4 +- lib/compress/zstd_fast.c | 4 +- lib/compress/zstd_lazy.c | 4 +- tests/fuzzer.c | 286 ++++++++++++++++++++++++++++++++ 5 files changed, 363 insertions(+), 23 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 38c3fc0d4cd..ae0679cf567 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -1264,6 +1264,13 @@ static int ZSTD_dedicatedDictSearch_isSupported( const ZSTD_compressionParameters* cParams); static void ZSTD_dedicatedDictSearch_revertCParams( ZSTD_compressionParameters* cParams); +static size_t ZSTD_createCDict_advanced2_internal( + ZSTD_CDict** cdictPtr, + const void* dict, size_t dictSize, + ZSTD_dictLoadMethod_e dictLoadMethod, + ZSTD_dictContentType_e dictContentType, + const ZSTD_CCtx_params* originalCctxParams, + ZSTD_customMem customMem); /** * Initializes the local dictionary using requested parameters. @@ -1289,14 +1296,14 @@ static size_t ZSTD_initLocalDict(ZSTD_CCtx* cctx) assert(cctx->cdict == NULL); assert(cctx->prefixDict.dict == NULL); - dl->cdict = ZSTD_createCDict_advanced2( + FORWARD_IF_ERROR(ZSTD_createCDict_advanced2_internal( + &dl->cdict, dl->dict, dl->dictSize, ZSTD_dlm_byRef, dl->dictContentType, &cctx->requestedParams, - cctx->customMem); - RETURN_ERROR_IF(!dl->cdict, memory_allocation, "ZSTD_createCDict_advanced failed"); + cctx->customMem), ""); cctx->cdict = dl->cdict; return 0; } @@ -2350,6 +2357,20 @@ static int ZSTD_shouldAttachDict(const ZSTD_CDict* cdict, * handled in _enforceMaxDist */ } +/* Check against the addressable content, which may be smaller than the full + * dictionary when only a suffix is loaded. */ +static size_t ZSTD_checkDictionaryRepCodes(const U32 rep[ZSTD_REP_NUM], size_t dictContentSize) +{ + U32 u; + for (u = 0; u < ZSTD_REP_NUM; ++u) { + RETURN_ERROR_IF(rep[u] == 0 || rep[u] > dictContentSize, + dictionary_corrupted, + "repcode %u exceeds dictionary content (%zu bytes)", + rep[u], dictContentSize); + } + return 0; +} + static size_t ZSTD_resetCCtx_byAttachingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict, @@ -2389,6 +2410,9 @@ ZSTD_resetCCtx_byAttachingCDict(ZSTD_CCtx* cctx, /* don't even attach dictionaries with no contents */ DEBUGLOG(4, "skipping attaching empty dictionary"); } else { + /* Enforce the dictMatchState block compressors' repcode bounds + * before allowing them to search this dictionary. */ + FORWARD_IF_ERROR(ZSTD_checkDictionaryRepCodes(cdict->cBlockState.rep, cdictLen), ""); DEBUGLOG(4, "attaching dictionary into context"); cctx->blockState.matchState.dictMatchState = &cdict->matchState; @@ -5168,12 +5192,9 @@ size_t ZSTD_loadCEntropy(ZSTD_compressedBlockState_t* bs, void* workspace, /* All offset values <= dictContentSize + 128 KB must be representable for a valid table */ bs->entropy.fse.offcode_repeatMode = ZSTD_dictNCountRepeat(offcodeNCount, offcodeMaxValue, MIN(offcodeMax, MaxOff)); - /* All repCodes must be <= dictContentSize and != 0 */ - { U32 u; - for (u=0; u<3; u++) { - RETURN_ERROR_IF(bs->rep[u] == 0, dictionary_corrupted, ""); - RETURN_ERROR_IF(bs->rep[u] > dictContentSize, dictionary_corrupted, ""); - } } } + /* The format permits nonzero repcodes up to the full content size. */ + FORWARD_IF_ERROR(ZSTD_checkDictionaryRepCodes(bs->rep, dictContentSize), ""); + } return (size_t)(dictPtr - (const BYTE*)dict); } @@ -5214,6 +5235,15 @@ static size_t ZSTD_loadZstdDictionary(ZSTD_compressedBlockState_t* bs, FORWARD_IF_ERROR(ZSTD_loadDictionaryContent( ms, NULL, ws, params, dictPtr, dictContentSize, dtlm, tfp), ""); } + if (tfp == ZSTD_tfp_forCDict) { + /* The matchfinder may retain only a suffix of a large dictionary. + * In particular, short-cache CDicts retain at most (1 << 24) - 2 + * bytes. Recheck the repcodes against the actual window so that + * dictMatchState searches cannot underflow their match indices. */ + U32 const loadedDictSize = (U32)(ms->window.nextSrc - ms->window.base) - ms->window.lowLimit; + assert((U32)(ms->window.nextSrc - ms->window.base) >= ms->window.lowLimit); + FORWARD_IF_ERROR(ZSTD_checkDictionaryRepCodes(bs->rep, loadedDictSize), ""); + } return dictID; } @@ -5686,7 +5716,10 @@ ZSTD_CDict* ZSTD_createCDict_advanced(const void* dictBuffer, size_t dictSize, &cctxParams, customMem); } -ZSTD_CDict* ZSTD_createCDict_advanced2( +/* Preserve initialization errors for internal callers. The public constructors + * retain their pointer-or-NULL API. On failure, *cdictPtr remains NULL. */ +static size_t ZSTD_createCDict_advanced2_internal( + ZSTD_CDict** cdictPtr, const void* dict, size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictContentType_e dictContentType, @@ -5697,8 +5730,10 @@ ZSTD_CDict* ZSTD_createCDict_advanced2( ZSTD_compressionParameters cParams; ZSTD_CDict* cdict; + *cdictPtr = NULL; DEBUGLOG(3, "ZSTD_createCDict_advanced2, dictSize=%u, mode=%u", (unsigned)dictSize, (unsigned)dictContentType); - if (!customMem.customAlloc ^ !customMem.customFree) return NULL; + RETURN_ERROR_IF((!customMem.customAlloc) ^ (!customMem.customFree), + parameter_combination_unsupported, "Invalid custom allocator"); if (cctxParams.enableDedicatedDictSearch) { cParams = ZSTD_dedicatedDictSearch_getCParams( @@ -5725,14 +5760,33 @@ ZSTD_CDict* ZSTD_createCDict_advanced2( cctxParams.useRowMatchFinder, cctxParams.enableDedicatedDictSearch, customMem); - if (!cdict || ZSTD_isError( ZSTD_initCDict_internal(cdict, - dict, dictSize, - dictLoadMethod, dictContentType, - cctxParams) )) { - ZSTD_freeCDict(cdict); - return NULL; + RETURN_ERROR_IF(!cdict, memory_allocation, "Could not allocate CDict"); + { size_t const err = ZSTD_initCDict_internal(cdict, + dict, dictSize, + dictLoadMethod, dictContentType, + cctxParams); + if (ZSTD_isError(err)) { + ZSTD_freeCDict(cdict); + return err; + } } + *cdictPtr = cdict; + return 0; +} + +ZSTD_CDict* ZSTD_createCDict_advanced2( + const void* dict, size_t dictSize, + ZSTD_dictLoadMethod_e dictLoadMethod, + ZSTD_dictContentType_e dictContentType, + const ZSTD_CCtx_params* originalCctxParams, + ZSTD_customMem customMem) +{ + ZSTD_CDict* cdict; + size_t const err = ZSTD_createCDict_advanced2_internal( + &cdict, dict, dictSize, dictLoadMethod, dictContentType, + originalCctxParams, customMem); + if (ZSTD_isError(err)) return NULL; return cdict; } diff --git a/lib/compress/zstd_double_fast.c b/lib/compress/zstd_double_fast.c index 1a266e7d955..568df6f7204 100644 --- a/lib/compress/zstd_double_fast.c +++ b/lib/compress/zstd_double_fast.c @@ -375,8 +375,8 @@ size_t ZSTD_compressBlock_doubleFast_dictMatchState_generic( /* init */ ip += (dictAndPrefixLength == 0); - /* dictMatchState repCode checks don't currently handle repCode == 0 - * disabling. */ + /* Dictionary repcodes are validated when the CDict is attached. + * dictMatchState repCode checks don't handle repCode == 0 disabling. */ assert(offset_1 <= dictAndPrefixLength); assert(offset_2 <= dictAndPrefixLength); diff --git a/lib/compress/zstd_fast.c b/lib/compress/zstd_fast.c index ee25bcbac8d..8130fc961ff 100644 --- a/lib/compress/zstd_fast.c +++ b/lib/compress/zstd_fast.c @@ -532,8 +532,8 @@ size_t ZSTD_compressBlock_fast_dictMatchState_generic( /* init */ DEBUGLOG(5, "ZSTD_compressBlock_fast_dictMatchState_generic"); ip0 += (dictAndPrefixLength == 0); - /* dictMatchState repCode checks don't currently handle repCode == 0 - * disabling. */ + /* Dictionary repcodes are validated when the CDict is attached. + * dictMatchState repCode checks don't handle repCode == 0 disabling. */ assert(offset_1 <= dictAndPrefixLength); assert(offset_2 <= dictAndPrefixLength); diff --git a/lib/compress/zstd_lazy.c b/lib/compress/zstd_lazy.c index 18b7b43948f..03c9992d0db 100644 --- a/lib/compress/zstd_lazy.c +++ b/lib/compress/zstd_lazy.c @@ -1601,8 +1601,8 @@ size_t ZSTD_compressBlock_lazy_generic( if (offset_1 > maxRep) offsetSaved1 = offset_1, offset_1 = 0; } if (isDxS) { - /* dictMatchState repCode checks don't currently handle repCode == 0 - * disabling. */ + /* Dictionary repcodes are validated when the CDict is attached. + * dictMatchState repCode checks don't handle repCode == 0 disabling. */ assert(offset_1 <= dictAndPrefixLength); assert(offset_2 <= dictAndPrefixLength); } diff --git a/tests/fuzzer.c b/tests/fuzzer.c index 3d3cd66abf8..218bbe23543 100644 --- a/tests/fuzzer.c +++ b/tests/fuzzer.c @@ -1096,6 +1096,284 @@ static unsigned test_get1BlockSummary(unsigned testNb) /* ============================================================= */ +typedef struct { + int fail; + size_t outstanding; +} FUZ_dictAllocState; + +static void* FUZ_dictMalloc(void* opaque, size_t size) +{ + FUZ_dictAllocState* const state = (FUZ_dictAllocState*)opaque; + void* allocation; + if (state->fail) return NULL; + allocation = malloc(size); + if (allocation != NULL) ++state->outstanding; + return allocation; +} + +static void FUZ_dictFree(void* opaque, void* allocation) +{ + FUZ_dictAllocState* const state = (FUZ_dictAllocState*)opaque; + if (allocation != NULL) { + CHECK(state->outstanding != 0); + --state->outstanding; + } + free(allocation); +} + +static void test_dictionaryLoadErrors(const void* dict, size_t dictSize) +{ + FUZ_dictAllocState state = { 0, 0 }; + ZSTD_customMem const customMem = { FUZ_dictMalloc, FUZ_dictFree, &state }; + ZSTD_CCtx* const cctx = ZSTD_createCCtx_advanced(customMem); + ZSTD_CCtx_params cdictParams; + BYTE const src[32] = { 0 }; + BYTE compressed[ZSTD_COMPRESSBOUND(sizeof(src))]; + size_t const baseline = state.outstanding; + size_t cSize; + + CHECK(cctx != NULL); + CHECK_Z(ZSTD_CCtxParams_init(&cdictParams, 1)); + CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 1)); + CHECK_Z(ZSTD_CCtx_loadDictionary_byReference(cctx, dict, dictSize)); + + /* Actual allocation failures must keep their distinct error code. */ + state.fail = 1; + CHECK(ZSTD_createCDict_advanced2(dict, dictSize, ZSTD_dlm_byRef, + ZSTD_dct_fullDict, &cdictParams, customMem) == NULL); + CHECK_EQ(ZSTD_getErrorCode(ZSTD_compress2(cctx, compressed, sizeof(compressed), + src, sizeof(src))), ZSTD_error_memory_allocation); + CHECK_EQ(state.outstanding, baseline); + state.fail = 0; + + /* A failed CDict initialization must free its partial allocation and + * leave the CCtx able to retry. The first eight bytes retain the magic + * number and dictionary ID, but omit the required entropy tables. */ + CHECK(dictSize > 8); + CHECK_Z(ZSTD_CCtx_loadDictionary_byReference(cctx, dict, 8)); + CHECK(ZSTD_createCDict_advanced2(dict, 8, ZSTD_dlm_byRef, + ZSTD_dct_fullDict, &cdictParams, customMem) == NULL); + CHECK_EQ(state.outstanding, baseline); + CHECK_EQ(ZSTD_getErrorCode(ZSTD_compress2(cctx, compressed, sizeof(compressed), + src, sizeof(src))), ZSTD_error_dictionary_corrupted); + CHECK_EQ(state.outstanding, baseline); + CHECK_EQ(ZSTD_getErrorCode(ZSTD_compress2(cctx, compressed, sizeof(compressed), + src, sizeof(src))), ZSTD_error_dictionary_corrupted); + CHECK_EQ(state.outstanding, baseline); + + CHECK_Z(ZSTD_CCtx_loadDictionary_byReference(cctx, dict, dictSize)); + CHECK_VAR(cSize, ZSTD_compress2(cctx, compressed, sizeof(compressed), src, sizeof(src))); + { ZSTD_DCtx* const dctx = ZSTD_createDCtx(); + BYTE decoded[sizeof(src)]; + CHECK(dctx != NULL); + { CHECK_NEWV(dSize, ZSTD_decompress_usingDict(dctx, decoded, sizeof(decoded), + compressed, cSize, dict, dictSize)); + CHECK(dSize == sizeof(src) && memcmp(src, decoded, sizeof(src)) == 0); + } + ZSTD_freeDCtx(dctx); + } + ZSTD_freeCCtx(cctx); + CHECK_EQ(state.outstanding, 0); +} + +#if ZSTD_WINDOW_OVERFLOW_CORRECT_FREQUENTLY +/* dict has a valid full-dictionary header followed by at least 3 MB + 64 bytes of content. */ +static void test_dictionaryRepcodesAfterOverflow(BYTE* dict, size_t headerSize) +{ + /* Reaching the normal overflow threshold requires compressing a few GB. + * Only run this regression in frequent-correction mode, where a smaller + * dictionary and forceMaxWindow reproduce the same state cheaply. Loading + * the dictionary into the reused CCtx retains only 3 MB of its content. */ + size_t const contentSize = 3 MB + 64; + size_t const preloadSize = 256 KB - 64; + size_t const fullDictSize = headerSize + contentSize; + U32 const rep = (U32)contentSize - 16; + size_t const compressedCapacity = ZSTD_compressBound(128 KB); + void* const compressed = malloc(compressedCapacity); + ZSTD_CCtx* const fresh = ZSTD_createCCtx(); + ZSTD_CCtx* const reused = ZSTD_createCCtx(); + ZSTD_DCtx* const dctx = ZSTD_createDCtx(); + ZSTD_parameters params = ZSTD_getParams(1, ZSTD_CONTENTSIZE_UNKNOWN, 0); + ZSTD_CCtx_params cctxParams; + BYTE src[4 KB]; + BYTE decoded[sizeof(src)]; + size_t remaining = preloadSize; + U32 u; + + CHECK(compressed != NULL && fresh != NULL && reused != NULL && dctx != NULL); + for (u = 0; u < ZSTD_REP_NUM; ++u) + MEM_writeLE32(dict + headerSize - 12 + 4 * u, repStartValue[u]); + MEM_writeLE32(dict + headerSize - 12, rep); + memcpy(src, dict + fullDictSize - rep, sizeof(src)); + + params.cParams.windowLog = 20; + params.cParams.hashLog = 18; + params.cParams.chainLog = 17; + CHECK_Z(ZSTD_CCtxParams_init_advanced(&cctxParams, params)); + CHECK_Z(ZSTD_CCtxParams_setParameter(&cctxParams, ZSTD_c_forceMaxWindow, 1)); + + /* First prove that exactly this dictionary and these parameters work. */ + CHECK_Z(ZSTD_compressBegin_advanced_internal(fresh, dict, fullDictSize, + ZSTD_dct_fullDict, ZSTD_dtlm_fast, NULL, &cctxParams, ZSTD_CONTENTSIZE_UNKNOWN)); + CHECK_EQ(fresh->blockState.matchState.window.nbOverflowCorrections, 0); + { CHECK_NEWV(cSize, ZSTD_compressEnd(fresh, compressed, compressedCapacity, src, sizeof(src))); + { CHECK_NEWV(dSize, ZSTD_decompress_usingDict(dctx, decoded, sizeof(decoded), + compressed, cSize, dict, fullDictSize)); + CHECK(dSize == sizeof(src) && memcmp(src, decoded, sizeof(src)) == 0); + } + } + + /* Advance the indices using actual compression, without replacing the + * workspace or manufacturing internal match-state indices. */ + CHECK_Z(ZSTD_compressBegin_advanced_internal(reused, NULL, 0, + ZSTD_dct_auto, ZSTD_dtlm_fast, NULL, &cctxParams, ZSTD_CONTENTSIZE_UNKNOWN)); + while (remaining != 0) { + size_t const chunkSize = MIN(remaining, 128 KB); + CHECK_Z(ZSTD_compressContinue(reused, compressed, compressedCapacity, + dict + headerSize, chunkSize)); + remaining -= chunkSize; + } + CHECK_Z(ZSTD_compressEnd(reused, compressed, compressedCapacity, NULL, 0)); + CHECK_EQ(reused->blockState.matchState.window.nbOverflowCorrections, 0); + CHECK_EQ((size_t)(reused->blockState.matchState.window.nextSrc + - reused->blockState.matchState.window.base), + preloadSize + ZSTD_WINDOW_START_INDEX); + + { size_t const result = ZSTD_compressBegin_advanced_internal(reused, dict, fullDictSize, + ZSTD_dct_fullDict, ZSTD_dtlm_fast, NULL, &cctxParams, ZSTD_CONTENTSIZE_UNKNOWN); + ZSTD_window_t const* const window = &reused->blockState.matchState.window; + U32 const loadedDictSize = (U32)(window->nextSrc - window->base) - window->lowLimit; + CHECK_EQ(window->nbOverflowCorrections, 1); + CHECK_EQ(loadedDictSize, 3 MB); + CHECK_LT(loadedDictSize, rep); + CHECK_Z(result); + } + { CHECK_NEWV(cSize, ZSTD_compressEnd(reused, compressed, compressedCapacity, src, sizeof(src))); + { CHECK_NEWV(dSize, ZSTD_decompress_usingDict(dctx, decoded, sizeof(decoded), + compressed, cSize, dict, fullDictSize)); + CHECK(dSize == sizeof(src) && memcmp(src, decoded, sizeof(src)) == 0); + } + } + + ZSTD_freeDCtx(dctx); + ZSTD_freeCCtx(reused); + ZSTD_freeCCtx(fresh); + free(compressed); +} +#endif + +static void test_largeDictionaryRepcodes(const void* dict, size_t dictSize) +{ + U32 const shortCacheMaxDictSize = (1U << (32 - ZSTD_SHORT_CACHE_TAG_BITS)) - ZSTD_WINDOW_START_INDEX; + struct DictCase { size_t contentSize; U32 rep; }; + struct DictCase const cases[] = { + { 20 MB, shortCacheMaxDictSize + 1 }, + { 20 MB, 20 MB - 16 }, + { 20 MB, shortCacheMaxDictSize }, + { 10 MB, 10 MB - 16 }, + }; + int const levels[] = { -5, 1, 2, 3, 4, 5 }; + size_t const headerSize = ZDICT_getDictHeaderSize(dict, dictSize); + BYTE* largeDict; + ZSTD_CCtx* const cctx = ZSTD_createCCtx(); + ZSTD_DCtx* const dctx = ZSTD_createDCtx(); + BYTE src[4 KB]; + BYTE compressed[8 KB]; + BYTE decoded[sizeof(src)]; + size_t ci, li; + U32 r; + + CHECK_Z(headerSize); + CHECK(headerSize >= 12); + largeDict = (BYTE*)malloc(headerSize + 20 MB); + CHECK(largeDict != NULL && cctx != NULL && dctx != NULL); + memcpy(largeDict, dict, headerSize); + RDG_genBuffer(largeDict + headerSize, 20 MB, 0.5, 0., 0); + +#if ZSTD_WINDOW_OVERFLOW_CORRECT_FREQUENTLY + test_dictionaryRepcodesAfterOverflow(largeDict, headerSize); +#endif + + for (ci = 0; ci < COUNTOF(cases); ++ci) { + size_t const largeDictSize = headerSize + cases[ci].contentSize; + /* Make the large repcode a possible match in the full dictionary. */ + memcpy(src, largeDict + largeDictSize - cases[ci].rep, sizeof(src)); + for (li = 0; li < COUNTOF(levels); ++li) { + int const level = levels[li]; + ZSTD_CCtx_params cdictParams; + ZSTD_compressionParameters cParams; + int reject; + CHECK_Z(ZSTD_CCtxParams_init(&cdictParams, level)); + cParams = ZSTD_getCParamsFromCCtxParams(&cdictParams, + ZSTD_CONTENTSIZE_UNKNOWN, largeDictSize, ZSTD_cpm_createCDict); + reject = (cParams.strategy == ZSTD_fast || cParams.strategy == ZSTD_dfast) + && cases[ci].rep > shortCacheMaxDictSize; + for (r = 0; r < ZSTD_REP_NUM; ++r) { + ZSTD_CDict* cdict; + U32 u; + size_t cSize; + ZSTD_inBuffer in = { src, sizeof(src), 0 }; + ZSTD_outBuffer out = { compressed, sizeof(compressed), 0 }; + for (u = 0; u < ZSTD_REP_NUM; ++u) + MEM_writeLE32(largeDict + headerSize - 12 + 4 * u, repStartValue[u]); + MEM_writeLE32(largeDict + headerSize - 12 + 4 * r, cases[ci].rep); + + cdict = ZSTD_createCDict_byReference(largeDict, largeDictSize, level); + if (reject) { + CHECK(cdict == NULL); + /* The copy constructor and advanced constructor share the check. */ + CHECK(ZSTD_createCDict(largeDict, largeDictSize, level) == NULL); + CHECK(ZSTD_createCDict_advanced(largeDict, largeDictSize, + ZSTD_dlm_byRef, ZSTD_dct_fullDict, + cParams, ZSTD_defaultCMem) == NULL); + + /* Local CDict creation is deferred until compression starts. */ + CHECK_Z(ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters)); + CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, level)); + CHECK_Z(ZSTD_CCtx_loadDictionary(cctx, largeDict, largeDictSize)); + CHECK_EQ(ZSTD_getErrorCode(ZSTD_compress2(cctx, compressed, + sizeof(compressed), src, sizeof(src))), ZSTD_error_dictionary_corrupted); + CHECK_Z(ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters)); + CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, level)); + CHECK_Z(ZSTD_CCtx_loadDictionary_byReference(cctx, largeDict, largeDictSize)); + CHECK_EQ(ZSTD_getErrorCode(ZSTD_compressStream2(cctx, &out, &in, + ZSTD_e_continue)), ZSTD_error_dictionary_corrupted); + + /* The dictionary is format-valid. The non-CDict path retains + * all of its content, so it must still round-trip. */ + CHECK_VAR(cSize, ZSTD_compress_usingDict(cctx, compressed, sizeof(compressed), + src, sizeof(src), largeDict, largeDictSize, level)); + } else { + CHECK(cdict != NULL); + CHECK_VAR(cSize, ZSTD_compress_usingCDict(cctx, compressed, sizeof(compressed), + src, sizeof(src), cdict)); + { CHECK_NEWV(dSize, ZSTD_decompress_usingDict(dctx, decoded, sizeof(decoded), + compressed, cSize, largeDict, largeDictSize)); + CHECK(dSize == sizeof(src) && memcmp(src, decoded, sizeof(src)) == 0); + } + + /* Start with unknown input size to exercise attached streaming. */ + CHECK_Z(ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters)); + CHECK_Z(ZSTD_CCtx_refCDict(cctx, cdict)); + CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_continue)); + CHECK(in.pos == in.size); + CHECK(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end) == 0); + cSize = out.pos; + ZSTD_freeCDict(cdict); + } + { CHECK_NEWV(dSize, ZSTD_decompress_usingDict(dctx, decoded, sizeof(decoded), + compressed, cSize, largeDict, largeDictSize)); + CHECK(dSize == sizeof(src) && memcmp(src, decoded, sizeof(src)) == 0); + } + } + } + } + + ZSTD_freeDCtx(dctx); + ZSTD_freeCCtx(cctx); + free(largeDict); +} + static int basicUnitTests(U32 const seed, double compressibility) { size_t const CNBuffSize = 5 MB; @@ -3289,6 +3567,14 @@ static int basicUnitTests(U32 const seed, double compressibility) } DISPLAYLEVEL(3, "OK : %u \n", (unsigned)dictHeaderSize); + DISPLAYLEVEL(3, "test%3i : Dictionary creation errors : ", testNb++); + test_dictionaryLoadErrors(dictBuffer, dictSize); + DISPLAYLEVEL(3, "OK \n"); + + DISPLAYLEVEL(3, "test%3i : Large dictionary repcodes : ", testNb++); + test_largeDictionaryRepcodes(dictBuffer, dictSize); + DISPLAYLEVEL(3, "OK \n"); + DISPLAYLEVEL(3, "test%3i : compress with dictionary : ", testNb++); cSize = ZSTD_compress_usingDict(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize,