From 87106dc77ef39c0661a2ad4b7e5f6efb2b86c208 Mon Sep 17 00:00:00 2001 From: Sunrisepeak Date: Sat, 7 Nov 2026 06:03:54 +0900 Subject: [PATCH 1/4] zstd-m 1.5.7: Zstandard as C++23 modules (mcpplibs.mwrap.zstd) Co-authored-by: speak-agent <248744407+speak-agent@users.noreply.github.com> --- .gitattributes | 2 + .github/workflows/ci.yml | 88 ++++++++ .github/workflows/release.yml | 49 +++++ .github/workflows/templates.yml | 28 +++ .gitignore | 6 + .xlings.json | 5 + NOTICE | 14 ++ README.zh.md | 75 +++++++ docs/usage.md | 55 +++++ include/mwrap/zstd/macros.hpp | 15 ++ mcpp.toml | 47 +++++ src/macros.cppm | 61 ++++++ src/zstd.cppm | 200 ++++++++++++++++++ templates/squeeze/mcpp.toml.in | 10 + templates/squeeze/src/main.cpp.in | 39 ++++ templates/squeeze/template.toml | 4 + tests/api_test.cpp | 93 ++++++++ tests/header_mix_test.cpp | 15 ++ tests/macros.cpp | 76 +++++++ tools/api.py | 129 ++++++++++++ tools/check_mwrap_release.py | 50 +++++ tools/generate.py | 196 +++++++++++++++++ tools/run_conformance.py | 119 +++++++++++ tools/surface.toml | 18 ++ tools/test_template.py | 83 ++++++++ upstream/documents.json | 22 ++ upstream/zstd/COPYING | 339 ++++++++++++++++++++++++++++++ upstream/zstd/LICENSE | 30 +++ upstream/zstd/README.md | 237 +++++++++++++++++++++ 29 files changed, 2105 insertions(+) create mode 100644 .gitattributes create mode 100644 .github/workflows/ci.yml create mode 100644 .github/workflows/release.yml create mode 100644 .github/workflows/templates.yml create mode 100644 .gitignore create mode 100644 .xlings.json create mode 100644 NOTICE create mode 100644 README.zh.md create mode 100644 docs/usage.md create mode 100644 include/mwrap/zstd/macros.hpp create mode 100644 mcpp.toml create mode 100644 src/macros.cppm create mode 100644 src/zstd.cppm create mode 100644 templates/squeeze/mcpp.toml.in create mode 100644 templates/squeeze/src/main.cpp.in create mode 100644 templates/squeeze/template.toml create mode 100644 tests/api_test.cpp create mode 100644 tests/header_mix_test.cpp create mode 100644 tests/macros.cpp create mode 100644 tools/api.py create mode 100644 tools/check_mwrap_release.py create mode 100644 tools/generate.py create mode 100644 tools/run_conformance.py create mode 100644 tools/surface.toml create mode 100644 tools/test_template.py create mode 100644 upstream/documents.json create mode 100644 upstream/zstd/COPYING create mode 100644 upstream/zstd/LICENSE create mode 100644 upstream/zstd/README.md diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..8a6c241 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,2 @@ +# Preserve upstream document bytes and original whitespace on every platform. +upstream/** -text -whitespace diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..1b4654b --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,88 @@ +name: CI + +on: + push: + branches: [main] + pull_request: + branches: [main] + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: ci-${{ github.ref }} + cancel-in-progress: true + +jobs: + mcpp: + name: mcpp (${{ matrix.os }}, ${{ matrix.toolchain }}${{ matrix.target && format(', {0}', matrix.target) || '' }}) + strategy: + fail-fast: false + matrix: + include: + - { os: ubuntu-24.04, toolchain: 'gcc@16.1.0' } + - { os: ubuntu-24.04, toolchain: 'gcc@16.1.0', target: x86_64-linux-musl } + - { os: ubuntu-24.04, toolchain: 'llvm@23.1.3' } + - { os: macos-15, toolchain: 'llvm@23.1.3' } + - { os: windows-2025, toolchain: 'llvm@23.1.3' } + runs-on: ${{ matrix.os }} + timeout-minutes: 60 + defaults: + run: + shell: bash + env: + XLINGS_NON_INTERACTIVE: 1 + TARGET: ${{ matrix.target && format('--target {0}', matrix.target) || '' }} + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Mwrap identity and bilingual README contract + run: python3 tools/check_mwrap_release.py + + - name: Install Xlings (Linux/macOS) + if: runner.os != 'Windows' + run: | + curl -fsSL https://raw.githubusercontent.com/openxlings/xlings/refs/heads/main/tools/other/quick_install.sh | bash + echo "$HOME/.xlings/subos/current/bin" >> "$GITHUB_PATH" + + - name: Install Xlings (Windows) + if: runner.os == 'Windows' + shell: pwsh + run: | + irm https://raw.githubusercontent.com/openxlings/xlings/main/tools/other/quick_install.ps1 | iex + "$env:USERPROFILE\.xlings\subos\current\bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append + + - name: Install project tools (.xlings.json) + run: | + for attempt in 1 2 3; do + xlings update > /dev/null 2>&1 || true + if xlings install; then break; fi + [ "$attempt" = 3 ] && exit 1 + sleep 30 + done + mcpp --version + + - name: Select the toolchain + run: | + name="${{ matrix.toolchain }}" + mcpp toolchain install "${name%@*}" "${name#*@}" + mcpp toolchain default "$name" + + - name: Build the modules + run: mcpp build $TARGET + + - name: Tests (one shot, parameters and checksums, streams, dictionaries, errors, the constants, header mixing) + run: mcpp test $TARGET + + - name: Export audit (the headers as clang reads them on this host) + if: startsWith(matrix.toolchain, 'llvm') + run: python3 tools/generate.py --check + + - name: Upstream's examples over the same library + if: ${{ !matrix.target }} + run: python3 tools/run_conformance.py --toolchain "${{ matrix.toolchain }}" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..f0c6447 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,49 @@ +name: Release +on: + push: + tags: ["mwrap-*"] +permissions: + contents: read +jobs: + publish: + runs-on: ubuntu-latest + timeout-minutes: 30 + permissions: + contents: write + actions: read + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - name: Check tag, upstream version and bilingual entry pages + run: python3 tools/check_mwrap_release.py --tag "$GITHUB_REF_NAME" + - name: Require successful source CI at this exact commit + env: + GH_TOKEN: ${{ github.token }} + shell: bash + run: | + set -euo pipefail + result=$(gh run list --repo "$GITHUB_REPOSITORY" --workflow ci.yml --commit "$GITHUB_SHA" --limit 1 --json status,conclusion --jq '.[0] | .status == "completed" and .conclusion == "success"') + test "$result" = true || { echo 'No successful source CI at the tag commit'; exit 1; } + - name: Build immutable source asset and publish + env: + GH_TOKEN: ${{ github.token }} + shell: bash + run: | + set -euo pipefail + version="${GITHUB_REF_NAME#mwrap-}" + repo="${GITHUB_REPOSITORY#*/}" + asset="${repo}-${version}.tar.gz" + git archive --format=tar.gz --prefix="${repo}-${version}/" "$GITHUB_SHA" > "$RUNNER_TEMP/$asset" + (cd "$RUNNER_TEMP" && sha256sum "$asset" > "$asset.sha256") + if gh release view "$GITHUB_REF_NAME" --repo "$GITHUB_REPOSITORY" >/dev/null 2>&1; then + echo 'Release already exists; refusing to change immutable assets'; exit 1 + fi + extra=() + if [[ "$version" == *-rc* ]]; then extra+=(--prerelease); fi + gh release create "$GITHUB_REF_NAME" "$RUNNER_TEMP/$asset" "$RUNNER_TEMP/$asset.sha256" \ + --repo "$GITHUB_REPOSITORY" --verify-tag --title "$repo $version" \ + --notes "Module wrapper for upstream $version. New mwrap package identities become installable after mcpp-index publication; see the bilingual README for the tested support scope." "${extra[@]}" diff --git a/.github/workflows/templates.yml b/.github/workflows/templates.yml new file mode 100644 index 0000000..4bcfe74 --- /dev/null +++ b/.github/workflows/templates.yml @@ -0,0 +1,28 @@ +name: Actual project templates +on: + push: + branches: [main] + pull_request: + workflow_dispatch: +permissions: + contents: read +concurrency: + group: templates-${{ github.ref }} + cancel-in-progress: true +env: + MCPP_VERSION: "2026.11.5.1" +jobs: + templates: + runs-on: ubuntu-24.04 + timeout-minutes: 60 + steps: + - uses: actions/checkout@v4 + - name: Install the pinned mcpp + shell: bash + run: | + archive="mcpp-${MCPP_VERSION}-linux-x86_64.tar.gz" + curl -L -fsS -o "$archive" "https://github.com/mcpp-community/mcpp/releases/download/v${MCPP_VERSION}/$archive" + tar -xzf "$archive" + echo "$PWD/mcpp-${MCPP_VERSION}-linux-x86_64/bin" >> "$GITHUB_PATH" + - name: Actual scaffold, strict build and run of the default template + run: python3 tools/test_template.py --toolchain gcc@16.1.0 diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..7c7b405 --- /dev/null +++ b/.gitignore @@ -0,0 +1,6 @@ +target/ +.mcpp/ +mcpp.lock +compile_commands.json +__pycache__/ +.xlings/ diff --git a/.xlings.json b/.xlings.json new file mode 100644 index 0000000..0ee91fd --- /dev/null +++ b/.xlings.json @@ -0,0 +1,5 @@ +{ + "workspace": { + "xim:mcpp": "2026.11.5.1" + } +} diff --git a/NOTICE b/NOTICE new file mode 100644 index 0000000..ef7e570 --- /dev/null +++ b/NOTICE @@ -0,0 +1,14 @@ +zstd-m: C++ module wrapper +Copyright (c) 2026 mcpp-community contributors + +The wrapper contributions are licensed under the Apache License, Version 2.0. +Upstream code, copied interfaces and dependencies retain their original +licenses and copyright notices. Zstandard is distributed under the BSD license +or, at the recipient's choice, the GPLv2: + + Copyright (c) Meta Platforms, Inc. and affiliates. All rights reserved. + +Its original README and both licenses are retained in upstream/zstd/. The +library itself is provided by compat.zstd from the same release, and the macro +of include/mwrap/zstd/macros.hpp is upstream's definition. Provenance and +checksums are recorded in upstream/documents.json. diff --git a/README.zh.md b/README.zh.md new file mode 100644 index 0000000..3391adc --- /dev/null +++ b/README.zh.md @@ -0,0 +1,75 @@ +# zstd + +> Zstandard——快速无损压缩库，以 C++23 模块提供。 + +[English](README.md) · **简体中文** · [上游](https://github.com/facebook/zstd) · [发布](https://github.com/mcpplibs/zstd-m/releases) · [mcpp](https://github.com/mcpp-community/mcpp) · [索引](https://github.com/mcpplibs/mcpp-index) + +[![C++23](https://img.shields.io/badge/C%2B%2B-23-blue.svg)](https://en.cppreference.com/w/cpp/23) +[![License](https://img.shields.io/badge/license-Apache--2.0-blue.svg)](LICENSE) +[![mcpp-community](https://img.shields.io/badge/mcpp--community-mwrap-blue)](https://github.com/mcpp-community) +[![CI](https://github.com/mcpplibs/zstd-m/actions/workflows/ci.yml/badge.svg?branch=main)](https://github.com/mcpplibs/zstd-m/actions/workflows/ci.yml) + +上游 API 不变：`ZSTD_compress`、`ZSTD_compress2`、`ZSTD_decompressStream`、`ZSTD_CCtx`、`ZSTD_cParameter` 及其枚举值、 +`` 的错误码都是上游实体，位于上游声明它们的全局命名空间。库本身是 `compat.zstd` 1.5.7；常量宏映射为 +C++ 常量。每次 CI 都在同一份库上运行上游自己的示例并完成往返。 + +## 模块 + +- `import mcpplibs.mwrap.zstd;`：`` 的稳定 API 与 `` 的错误码，并重新导出宏模块 +- `import mcpplibs.mwrap.zstd.macros;`：`ZSTD_CLEVEL_DEFAULT`、`ZSTD_CONTENTSIZE_ERROR`、`ZSTD_MAGICNUMBER`、`ZSTD_VERSION_NUMBER` 等，位于 `mcpplibs::mwrap::zstd::macros` +- `#include `：`ZSTD_COMPRESSBOUND(size)`，以常量表达式给出上界 + +> 更多模块 API -> [用法与 API](docs/usage.md) + +## 快速开始 + +```sh +mcpp new squeeze --template mcpplibs.mwrap.zstd@1.5.7 +cd squeeze +mcpp run +``` + +默认模板以两个级别带校验和压缩一段文本，再还原并打印大小。 + +```toml +[dependencies] +"mcpplibs.mwrap.zstd" = "1.5.7" +``` + +```cpp +import std; +import mcpplibs.mwrap.zstd; + +namespace z = mcpplibs::mwrap::zstd::macros; + +int main() { + std::string text(10000, 'z'), frame(ZSTD_compressBound(text.size()), '\0'); + const auto n = ZSTD_compress(frame.data(), frame.size(), text.data(), text.size(), z::ZSTD_CLEVEL_DEFAULT); + std::println("{} -> {} bytes", text.size(), ZSTD_isError(n) ? 0 : n); +} +``` + +## 范围与宏 + +模块导出稳定 API。`` 的实验部分（`ZSTD_STATIC_LINKING_ONLY` 之后）可能在版本之间变化，不导出。`` +也不导出：`compat.zstd` 不构建字典训练器。原始内容字典可经 `ZSTD_createCDict` 与 `ZSTD_createDDict` 使用。 + +命名模块不能导出宏。常量映射为 `mcpplibs::mwrap::zstd::macros` 中同名的 `inline constexpr`。`ZSTD_COMPRESSBOUND` +带参数，保留为 `` 中的宏；它用到 `size_t`，所以该翻译单元导入 `std.compat`。可见性与弃用宏只用于 +书写声明。宏在限定名中也会展开。同时包含 `` 的翻译单元经头文件的宏读取常量（[docs/usage.md](docs/usage.md)）。 + +## 构建与测试 + +Linux、macOS、Windows 的 CI 用 GCC 与 LLVM 构建模块并运行测试：一次调用的往返、带校验和的 19 级帧及其损坏检测、分段 +输入的流、原始内容字典、无效输入的错误码、版本与上界同其宏相等、每个常量同其宏相等、头文件混用。CI 还运行： + +- 导出审计（[tools/generate.py](tools/generate.py) `--check`）：按 clang 读到的头文件重新生成模块、宏映射、宏头文件与 + 常量测试并逐字节比较；在 macOS 与 Windows 上检查头文件在该平台声明的每个名字都已导出； +- 在同一份库上运行上游的四个示例（[tools/run_conformance.py](tools/run_conformance.py)）。 + +另一个作业把默认模板当作已安装的包生成项目并运行。 + +## 许可 + +包装部分为 [Apache-2.0](LICENSE)，作者 mcpp-community（[NOTICE](NOTICE)）。Zstandard 仍遵循 BSD 许可或 GPLv2 +（[upstream/zstd/LICENSE](upstream/zstd/LICENSE)、[COPYING](upstream/zstd/COPYING)、[README](upstream/zstd/README.md)）。 diff --git a/docs/usage.md b/docs/usage.md new file mode 100644 index 0000000..2331cab --- /dev/null +++ b/docs/usage.md @@ -0,0 +1,55 @@ +# Usage and API + +`mcpplibs.mwrap.zstd` 1.5.7 exports Zstandard's stable API, upstream's names +unchanged. This page lists what each module holds and the places where a module +and a header differ. [tools/generate.py](../tools/generate.py) writes the lists +from the headers, and CI checks them on every target. + +## Modules + +| Module | Holds | +|---|---| +| `mcpplibs.mwrap.zstd` | the stable API of ``: one-shot `ZSTD_compress`, `ZSTD_decompress`, `ZSTD_compressBound`, `ZSTD_getFrameContentSize`, `ZSTD_findFrameCompressedSize`; explicit contexts (`ZSTD_CCtx`, `ZSTD_DCtx` and their `create`, `free`, `compressCCtx`, `decompressDCtx`); the advanced API (`ZSTD_cParameter`, `ZSTD_dParameter` and their enumerators, `ZSTD_CCtx_setParameter`, `ZSTD_compress2`, `ZSTD_cParam_getBounds`, `ZSTD_strategy`, `ZSTD_ResetDirective`); streaming (`ZSTD_inBuffer`, `ZSTD_outBuffer`, `ZSTD_EndDirective`, `ZSTD_compressStream2`, `ZSTD_decompressStream`, the `CStream`/`DStream` forms); dictionaries (`ZSTD_CDict`, `ZSTD_DDict`, `*_usingDict`, `*_usingCDict`, `*_usingDDict`, `ZSTD_CCtx_loadDictionary`, `refPrefix`, `getDictID_*`); `ZSTD_sizeof_*`; `ZSTD_versionNumber`, `ZSTD_versionString`, `ZSTD_isError`, `ZSTD_getErrorName`, levels. From ``: `ZSTD_ErrorCode` with every `ZSTD_error_*`, `ZSTD_getErrorCode`, `ZSTD_getErrorString`. Re-exports the macros module | +| `mcpplibs.mwrap.zstd.macros` | in `mcpplibs::mwrap::zstd::macros`: `ZSTD_VERSION_MAJOR`, `_MINOR`, `_RELEASE`, `_NUMBER`, `_STRING`; `ZSTD_CLEVEL_DEFAULT`; `ZSTD_MAGICNUMBER`, `ZSTD_MAGIC_DICTIONARY`, `ZSTD_MAGIC_SKIPPABLE_START`, `ZSTD_MAGIC_SKIPPABLE_MASK`; `ZSTD_BLOCKSIZELOG_MAX`, `ZSTD_BLOCKSIZE_MAX`; `ZSTD_CONTENTSIZE_UNKNOWN`, `ZSTD_CONTENTSIZE_ERROR`; `ZSTD_MAX_INPUT_SIZE` | + +## What is not exported + +- **The experimental API.** The part of `` behind + `ZSTD_STATIC_LINKING_ONLY` is declared only when a unit defines that macro + before the header, and upstream may change it in any release. A program that + needs it includes `` with the macro and links the same + `compat.zstd`. +- **``.** `compat.zstd` builds `lib/common`, `lib/compress` and + `lib/decompress`, not `lib/dictBuilder`; the `ZDICT_*` functions would not + link. A raw-content dictionary (any bytes) works with `ZSTD_createCDict`. + +## Macros + +- **Constants.** Each object-like macro whose expansion is a constant + expression is an `inline constexpr` of the same name and value; + tests/macros.cpp compares every one with its macro. +- **`ZSTD_COMPRESSBOUND(srcSize)`** is a constant expression where + `ZSTD_compressBound` is a call, so it stays a macro in + ``, reading `ZSTD_MAX_INPUT_SIZE` from the macros + module. It casts to `size_t`: import `std.compat`, which declares `::size_t`. +- **Declaration syntax.** `ZSTDLIB_API`, `ZSTDLIB_VISIBLE`, `ZSTDLIB_HIDDEN`, + their `ZSTDERRORLIB_*` counterparts, `ZSTD_DEPRECATED`, and the + stringification steps `ZSTD_QUOTE`, `ZSTD_EXPAND_AND_QUOTE`, + `ZSTD_LIB_VERSION` (whose result is `ZSTD_VERSION_STRING`) are not offered. + tools/surface.toml lists them. + +## A header beside the module + +`#include ` and `import mcpplibs.mwrap.zstd;` may appear in one unit: +they declare the same entities. A macro expands even inside a qualified name, +so `mcpplibs::mwrap::zstd::macros::ZSTD_CLEVEL_DEFAULT` written after the header +is `...::macros::3`. In such a unit use the header's macros, or read their values +under other names and `#undef` the macros before naming the constants +(tests/macros.cpp does this). + +## Linking + +The package depends on `compat.zstd` 1.5.7, which builds Zstandard from the +release sources with the target's toolchain, without assembly +(`ZSTD_DISABLE_ASM`). A program that imports the module links that library; it +does not look for a system zstd. diff --git a/include/mwrap/zstd/macros.hpp b/include/mwrap/zstd/macros.hpp new file mode 100644 index 0000000..fbe41e5 --- /dev/null +++ b/include/mwrap/zstd/macros.hpp @@ -0,0 +1,15 @@ +// --- the macros of the headers that are neither a constant +// nor a type: they take arguments, or call a function. A named module cannot export a +// macro: include this header before `import mcpplibs.mwrap.zstd;`. Each definition is +// upstream's (Zstandard 1.5.7, BSD license or GPLv2), made only where the upstream header has +// not defined it, with the constants it names read from mcpplibs::mwrap::zstd::macros. +// The upstream header included after this one redefines these macros. +// Written by tools/generate.py. +#ifndef MWRAP_ZSTD_MACROS_HPP +#define MWRAP_ZSTD_MACROS_HPP + +#ifndef ZSTD_COMPRESSBOUND +#define ZSTD_COMPRESSBOUND(srcSize) (((size_t)(srcSize) >= ::mcpplibs::mwrap::zstd::macros::ZSTD_MAX_INPUT_SIZE) ? 0 : (srcSize) + ((srcSize)>>8) + (((srcSize) < (128<<10)) ? (((128<<10) - (srcSize)) >> 11) : 0)) +#endif + +#endif // MWRAP_ZSTD_MACROS_HPP diff --git a/mcpp.toml b/mcpp.toml new file mode 100644 index 0000000..3e03033 --- /dev/null +++ b/mcpp.toml @@ -0,0 +1,47 @@ +# Module wrapper; upstream API preserved. Policy: mcpp-index/docs/mwrap.md. +[package] +namespace = "mcpplibs.mwrap" +name = "zstd" +version = "1.5.7" +description = "Zstandard 1.5.7 C++ modules for mcpp (import mcpplibs.mwrap.zstd;): one-shot and streaming compression and decompression, parameters, dictionaries and error codes over compat.zstd, with the constants of zstd.h as C++ constants" +license = "Apache-2.0" +authors = ["mcpp-community"] +repo = "https://github.com/mcpplibs/zstd-m" +platforms = ["linux", "macos", "windows"] + +[targets.zstd] +kind = "lib" + +[build] +sources = ["src/*.cppm"] +# : the macros that take arguments. +include_dirs = ["include"] + +[dependencies] +compat.zstd = "1.5.7" + +[test] +discover = ["tests/*.cpp"] + +[package.metadata.mwrap] +upstream-version = "1.5.7" + +[[runtime.deploy]] +from = "LICENSE" +to = "licenses/zstd" + +[[runtime.deploy]] +from = "NOTICE" +to = "licenses/zstd" + +[[runtime.deploy]] +from = "upstream/zstd/README.md" +to = "licenses/zstd/upstream/zstd" + +[[runtime.deploy]] +from = "upstream/zstd/LICENSE" +to = "licenses/zstd/upstream/zstd" + +[[runtime.deploy]] +from = "upstream/zstd/COPYING" +to = "licenses/zstd/upstream/zstd" diff --git a/src/macros.cppm b/src/macros.cppm new file mode 100644 index 0000000..c8e9d52 --- /dev/null +++ b/src/macros.cppm @@ -0,0 +1,61 @@ +// mcpplibs.mwrap.zstd.macros --- the macros of the headers that are a constant or a +// type, under their original names in mcpplibs::mwrap::zstd::macros, each equal to +// the macro where the module is built (a named module cannot export a macro). The +// other macros are in . Written by tools/generate.py. +module; +#include +#include + +// The values, taken under other names while the macros are defined. +namespace mwrap_zstd_capture { +inline constexpr auto value_ZSTD_BLOCKSIZELOG_MAX = (ZSTD_BLOCKSIZELOG_MAX); +inline constexpr auto value_ZSTD_BLOCKSIZE_MAX = (ZSTD_BLOCKSIZE_MAX); +inline constexpr auto value_ZSTD_CLEVEL_DEFAULT = (ZSTD_CLEVEL_DEFAULT); +inline constexpr auto value_ZSTD_CONTENTSIZE_ERROR = (ZSTD_CONTENTSIZE_ERROR); +inline constexpr auto value_ZSTD_CONTENTSIZE_UNKNOWN = (ZSTD_CONTENTSIZE_UNKNOWN); +inline constexpr auto value_ZSTD_MAGICNUMBER = (ZSTD_MAGICNUMBER); +inline constexpr auto value_ZSTD_MAGIC_DICTIONARY = (ZSTD_MAGIC_DICTIONARY); +inline constexpr auto value_ZSTD_MAGIC_SKIPPABLE_MASK = (ZSTD_MAGIC_SKIPPABLE_MASK); +inline constexpr auto value_ZSTD_MAGIC_SKIPPABLE_START = (ZSTD_MAGIC_SKIPPABLE_START); +inline constexpr auto value_ZSTD_MAX_INPUT_SIZE = (ZSTD_MAX_INPUT_SIZE); +inline constexpr auto value_ZSTD_VERSION_MAJOR = (ZSTD_VERSION_MAJOR); +inline constexpr auto value_ZSTD_VERSION_MINOR = (ZSTD_VERSION_MINOR); +inline constexpr auto value_ZSTD_VERSION_NUMBER = (ZSTD_VERSION_NUMBER); +inline constexpr auto value_ZSTD_VERSION_RELEASE = (ZSTD_VERSION_RELEASE); +inline constexpr auto value_ZSTD_VERSION_STRING = (ZSTD_VERSION_STRING); +} +#undef ZSTD_BLOCKSIZELOG_MAX +#undef ZSTD_BLOCKSIZE_MAX +#undef ZSTD_CLEVEL_DEFAULT +#undef ZSTD_CONTENTSIZE_ERROR +#undef ZSTD_CONTENTSIZE_UNKNOWN +#undef ZSTD_MAGICNUMBER +#undef ZSTD_MAGIC_DICTIONARY +#undef ZSTD_MAGIC_SKIPPABLE_MASK +#undef ZSTD_MAGIC_SKIPPABLE_START +#undef ZSTD_MAX_INPUT_SIZE +#undef ZSTD_VERSION_MAJOR +#undef ZSTD_VERSION_MINOR +#undef ZSTD_VERSION_NUMBER +#undef ZSTD_VERSION_RELEASE +#undef ZSTD_VERSION_STRING + +export module mcpplibs.mwrap.zstd.macros; + +export namespace mcpplibs::mwrap::zstd::macros { +inline constexpr auto ZSTD_BLOCKSIZELOG_MAX = mwrap_zstd_capture::value_ZSTD_BLOCKSIZELOG_MAX; +inline constexpr auto ZSTD_BLOCKSIZE_MAX = mwrap_zstd_capture::value_ZSTD_BLOCKSIZE_MAX; +inline constexpr auto ZSTD_CLEVEL_DEFAULT = mwrap_zstd_capture::value_ZSTD_CLEVEL_DEFAULT; +inline constexpr auto ZSTD_CONTENTSIZE_ERROR = mwrap_zstd_capture::value_ZSTD_CONTENTSIZE_ERROR; +inline constexpr auto ZSTD_CONTENTSIZE_UNKNOWN = mwrap_zstd_capture::value_ZSTD_CONTENTSIZE_UNKNOWN; +inline constexpr auto ZSTD_MAGICNUMBER = mwrap_zstd_capture::value_ZSTD_MAGICNUMBER; +inline constexpr auto ZSTD_MAGIC_DICTIONARY = mwrap_zstd_capture::value_ZSTD_MAGIC_DICTIONARY; +inline constexpr auto ZSTD_MAGIC_SKIPPABLE_MASK = mwrap_zstd_capture::value_ZSTD_MAGIC_SKIPPABLE_MASK; +inline constexpr auto ZSTD_MAGIC_SKIPPABLE_START = mwrap_zstd_capture::value_ZSTD_MAGIC_SKIPPABLE_START; +inline constexpr auto ZSTD_MAX_INPUT_SIZE = mwrap_zstd_capture::value_ZSTD_MAX_INPUT_SIZE; +inline constexpr auto ZSTD_VERSION_MAJOR = mwrap_zstd_capture::value_ZSTD_VERSION_MAJOR; +inline constexpr auto ZSTD_VERSION_MINOR = mwrap_zstd_capture::value_ZSTD_VERSION_MINOR; +inline constexpr auto ZSTD_VERSION_NUMBER = mwrap_zstd_capture::value_ZSTD_VERSION_NUMBER; +inline constexpr auto ZSTD_VERSION_RELEASE = mwrap_zstd_capture::value_ZSTD_VERSION_RELEASE; +inline constexpr auto ZSTD_VERSION_STRING = mwrap_zstd_capture::value_ZSTD_VERSION_STRING; +} diff --git a/src/zstd.cppm b/src/zstd.cppm new file mode 100644 index 0000000..5cb240e --- /dev/null +++ b/src/zstd.cppm @@ -0,0 +1,200 @@ +// mcpplibs.mwrap.zstd --- Zstandard 1.5.7 as a C++ module. Every exported name is +// the upstream entity of the headers below, unchanged, in the global namespace where +// upstream declares it; the root also re-exports the macro mapping. The implementation +// is compat.zstd 1.5.7. Written by tools/generate.py from the headers. +module; +#include +#include + +export module mcpplibs.mwrap.zstd; + +export import mcpplibs.mwrap.zstd.macros; + +export { +using ::ZSTD_CCtx; +using ::ZSTD_CCtx_loadDictionary; +using ::ZSTD_CCtx_refCDict; +using ::ZSTD_CCtx_refPrefix; +using ::ZSTD_CCtx_reset; +using ::ZSTD_CCtx_s; +using ::ZSTD_CCtx_setParameter; +using ::ZSTD_CCtx_setPledgedSrcSize; +using ::ZSTD_CDict; +using ::ZSTD_CDict_s; +using ::ZSTD_CStream; +using ::ZSTD_CStreamInSize; +using ::ZSTD_CStreamOutSize; +using ::ZSTD_DCtx; +using ::ZSTD_DCtx_loadDictionary; +using ::ZSTD_DCtx_refDDict; +using ::ZSTD_DCtx_refPrefix; +using ::ZSTD_DCtx_reset; +using ::ZSTD_DCtx_s; +using ::ZSTD_DCtx_setParameter; +using ::ZSTD_DDict; +using ::ZSTD_DDict_s; +using ::ZSTD_DStream; +using ::ZSTD_DStreamInSize; +using ::ZSTD_DStreamOutSize; +using ::ZSTD_EndDirective; +using ::ZSTD_ErrorCode; +using ::ZSTD_ResetDirective; +using ::ZSTD_bounds; +using ::ZSTD_btlazy2; +using ::ZSTD_btopt; +using ::ZSTD_btultra; +using ::ZSTD_btultra2; +using ::ZSTD_cParam_getBounds; +using ::ZSTD_cParameter; +using ::ZSTD_c_chainLog; +using ::ZSTD_c_checksumFlag; +using ::ZSTD_c_compressionLevel; +using ::ZSTD_c_contentSizeFlag; +using ::ZSTD_c_dictIDFlag; +using ::ZSTD_c_enableLongDistanceMatching; +using ::ZSTD_c_experimentalParam1; +using ::ZSTD_c_experimentalParam10; +using ::ZSTD_c_experimentalParam11; +using ::ZSTD_c_experimentalParam12; +using ::ZSTD_c_experimentalParam13; +using ::ZSTD_c_experimentalParam14; +using ::ZSTD_c_experimentalParam15; +using ::ZSTD_c_experimentalParam16; +using ::ZSTD_c_experimentalParam17; +using ::ZSTD_c_experimentalParam18; +using ::ZSTD_c_experimentalParam19; +using ::ZSTD_c_experimentalParam2; +using ::ZSTD_c_experimentalParam20; +using ::ZSTD_c_experimentalParam3; +using ::ZSTD_c_experimentalParam4; +using ::ZSTD_c_experimentalParam5; +using ::ZSTD_c_experimentalParam7; +using ::ZSTD_c_experimentalParam8; +using ::ZSTD_c_experimentalParam9; +using ::ZSTD_c_hashLog; +using ::ZSTD_c_jobSize; +using ::ZSTD_c_ldmBucketSizeLog; +using ::ZSTD_c_ldmHashLog; +using ::ZSTD_c_ldmHashRateLog; +using ::ZSTD_c_ldmMinMatch; +using ::ZSTD_c_minMatch; +using ::ZSTD_c_nbWorkers; +using ::ZSTD_c_overlapLog; +using ::ZSTD_c_searchLog; +using ::ZSTD_c_strategy; +using ::ZSTD_c_targetCBlockSize; +using ::ZSTD_c_targetLength; +using ::ZSTD_c_windowLog; +using ::ZSTD_compress; +using ::ZSTD_compress2; +using ::ZSTD_compressBound; +using ::ZSTD_compressCCtx; +using ::ZSTD_compressStream; +using ::ZSTD_compressStream2; +using ::ZSTD_compress_usingCDict; +using ::ZSTD_compress_usingDict; +using ::ZSTD_createCCtx; +using ::ZSTD_createCDict; +using ::ZSTD_createCStream; +using ::ZSTD_createDCtx; +using ::ZSTD_createDDict; +using ::ZSTD_createDStream; +using ::ZSTD_dParam_getBounds; +using ::ZSTD_dParameter; +using ::ZSTD_d_experimentalParam1; +using ::ZSTD_d_experimentalParam2; +using ::ZSTD_d_experimentalParam3; +using ::ZSTD_d_experimentalParam4; +using ::ZSTD_d_experimentalParam5; +using ::ZSTD_d_experimentalParam6; +using ::ZSTD_d_windowLogMax; +using ::ZSTD_decompress; +using ::ZSTD_decompressDCtx; +using ::ZSTD_decompressStream; +using ::ZSTD_decompress_usingDDict; +using ::ZSTD_decompress_usingDict; +using ::ZSTD_defaultCLevel; +using ::ZSTD_dfast; +using ::ZSTD_e_continue; +using ::ZSTD_e_end; +using ::ZSTD_e_flush; +using ::ZSTD_endStream; +using ::ZSTD_error_GENERIC; +using ::ZSTD_error_cannotProduce_uncompressedBlock; +using ::ZSTD_error_checksum_wrong; +using ::ZSTD_error_corruption_detected; +using ::ZSTD_error_dictionaryCreation_failed; +using ::ZSTD_error_dictionary_corrupted; +using ::ZSTD_error_dictionary_wrong; +using ::ZSTD_error_dstBuffer_null; +using ::ZSTD_error_dstBuffer_wrong; +using ::ZSTD_error_dstSize_tooSmall; +using ::ZSTD_error_externalSequences_invalid; +using ::ZSTD_error_frameIndex_tooLarge; +using ::ZSTD_error_frameParameter_unsupported; +using ::ZSTD_error_frameParameter_windowTooLarge; +using ::ZSTD_error_init_missing; +using ::ZSTD_error_literals_headerWrong; +using ::ZSTD_error_maxCode; +using ::ZSTD_error_maxSymbolValue_tooLarge; +using ::ZSTD_error_maxSymbolValue_tooSmall; +using ::ZSTD_error_memory_allocation; +using ::ZSTD_error_noForwardProgress_destFull; +using ::ZSTD_error_noForwardProgress_inputEmpty; +using ::ZSTD_error_no_error; +using ::ZSTD_error_parameter_combination_unsupported; +using ::ZSTD_error_parameter_outOfBound; +using ::ZSTD_error_parameter_unsupported; +using ::ZSTD_error_prefix_unknown; +using ::ZSTD_error_seekableIO; +using ::ZSTD_error_sequenceProducer_failed; +using ::ZSTD_error_srcBuffer_wrong; +using ::ZSTD_error_srcSize_wrong; +using ::ZSTD_error_stabilityCondition_notRespected; +using ::ZSTD_error_stage_wrong; +using ::ZSTD_error_tableLog_tooLarge; +using ::ZSTD_error_version_unsupported; +using ::ZSTD_error_workSpace_tooSmall; +using ::ZSTD_fast; +using ::ZSTD_findFrameCompressedSize; +using ::ZSTD_flushStream; +using ::ZSTD_freeCCtx; +using ::ZSTD_freeCDict; +using ::ZSTD_freeCStream; +using ::ZSTD_freeDCtx; +using ::ZSTD_freeDDict; +using ::ZSTD_freeDStream; +using ::ZSTD_getDecompressedSize; +using ::ZSTD_getDictID_fromCDict; +using ::ZSTD_getDictID_fromDDict; +using ::ZSTD_getDictID_fromDict; +using ::ZSTD_getDictID_fromFrame; +using ::ZSTD_getErrorCode; +using ::ZSTD_getErrorName; +using ::ZSTD_getErrorString; +using ::ZSTD_getFrameContentSize; +using ::ZSTD_greedy; +using ::ZSTD_inBuffer; +using ::ZSTD_inBuffer_s; +using ::ZSTD_initCStream; +using ::ZSTD_initDStream; +using ::ZSTD_isError; +using ::ZSTD_lazy; +using ::ZSTD_lazy2; +using ::ZSTD_maxCLevel; +using ::ZSTD_minCLevel; +using ::ZSTD_outBuffer; +using ::ZSTD_outBuffer_s; +using ::ZSTD_reset_parameters; +using ::ZSTD_reset_session_and_parameters; +using ::ZSTD_reset_session_only; +using ::ZSTD_sizeof_CCtx; +using ::ZSTD_sizeof_CDict; +using ::ZSTD_sizeof_CStream; +using ::ZSTD_sizeof_DCtx; +using ::ZSTD_sizeof_DDict; +using ::ZSTD_sizeof_DStream; +using ::ZSTD_strategy; +using ::ZSTD_versionNumber; +using ::ZSTD_versionString; +} diff --git a/templates/squeeze/mcpp.toml.in b/templates/squeeze/mcpp.toml.in new file mode 100644 index 0000000..a637560 --- /dev/null +++ b/templates/squeeze/mcpp.toml.in @@ -0,0 +1,10 @@ +[package] +name = "{{project.name}}" +version = "0.1.0" + +[targets.{{project.name}}] +kind = "bin" +main = "src/main.cpp" + +[dependencies] +"{{self.name}}" = "{{self.version}}" diff --git a/templates/squeeze/src/main.cpp.in b/templates/squeeze/src/main.cpp.in new file mode 100644 index 0000000..3023215 --- /dev/null +++ b/templates/squeeze/src/main.cpp.in @@ -0,0 +1,39 @@ +// {{project.name}}: Zstandard through import mcpplibs.mwrap.zstd. The functions, +// types and enumerators are Zstandard's own names; the constants are in +// mcpplibs::mwrap::zstd::macros. +import std; +import mcpplibs.mwrap.zstd; + +namespace z = mcpplibs::mwrap::zstd::macros; + +int main(int argc, char** argv) +{ + std::string text; + if (argc > 1) { + std::ifstream file(argv[1], std::ios::binary); + text.assign(std::istreambuf_iterator(file), {}); + } else { + for (int i = 0; i < 200; ++i) text += std::format("Zstandard through a C++ module, entry {}\n", i % 10); + } + + ZSTD_CCtx* cctx = ZSTD_createCCtx(); + ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1); + std::string frame; + for (int level : {z::ZSTD_CLEVEL_DEFAULT, 19}) { + ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, level); + frame.resize(ZSTD_compressBound(text.size())); + const auto size = ZSTD_compress2(cctx, frame.data(), frame.size(), text.data(), text.size()); + if (ZSTD_isError(size)) { + std::println("level {}: {}", level, ZSTD_getErrorName(size)); + return 1; + } + frame.resize(size); + std::println("level {:2}: {} bytes -> {} bytes", level, text.size(), size); + } + ZSTD_freeCCtx(cctx); + + std::string back(ZSTD_getFrameContentSize(frame.data(), frame.size()), '\0'); + const auto n = ZSTD_decompress(back.data(), back.size(), frame.data(), frame.size()); + std::println("zstd {}: restored {}", ZSTD_versionString(), !ZSTD_isError(n) && back == text ? "intact" : "DIFFERENT"); + return !ZSTD_isError(n) && back == text ? 0 : 1; +} diff --git a/templates/squeeze/template.toml b/templates/squeeze/template.toml new file mode 100644 index 0000000..60b6cd8 --- /dev/null +++ b/templates/squeeze/template.toml @@ -0,0 +1,4 @@ +[template] +description = "Compression with import mcpplibs.mwrap.zstd: a text compressed at two levels with a checksum, then restored" +default = true +post_message = "Run `mcpp run`, or `mcpp run -- ` to compress a file of your own." diff --git a/tests/api_test.cpp b/tests/api_test.cpp new file mode 100644 index 0000000..85dcf63 --- /dev/null +++ b/tests/api_test.cpp @@ -0,0 +1,93 @@ +// The module's functions over compat.zstd: a one-shot round trip, a frame +// with a checksum through the advanced parameters, a stream, a raw-content +// dictionary, an error code, and the version against its macros. +#include +import std.compat; +import mcpplibs.mwrap.zstd; + +namespace m = mcpplibs::mwrap::zstd::macros; + +int main() +{ + int failures = 0; + auto check = [&](bool ok, std::string_view what) { + std::println("{} {}", ok ? "ok " : "FAIL", what); + failures += ok ? 0 : 1; + }; + + std::string text; + for (int i = 0; i < 2000; ++i) text += std::format("record {} of a repetitive log\n", i % 23); + + // One shot. + std::string frame(ZSTD_compressBound(text.size()), '\0'); + const auto size = ZSTD_compress(frame.data(), frame.size(), text.data(), text.size(), m::ZSTD_CLEVEL_DEFAULT); + check(!ZSTD_isError(size) && size < text.size() / 10, "ZSTD_compress shrinks the text"); + frame.resize(size); + check(ZSTD_getFrameContentSize(frame.data(), frame.size()) == text.size(), "the frame states its content size"); + std::string back(text.size(), '\0'); + check(ZSTD_decompress(back.data(), back.size(), frame.data(), frame.size()) == text.size() && back == text, + "ZSTD_decompress restores it"); + + // The advanced parameters: a level and a checksum. + ZSTD_CCtx* cctx = ZSTD_createCCtx(); + check(!ZSTD_isError(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 19)) + && !ZSTD_isError(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1)), "ZSTD_CCtx_setParameter"); + std::string checked(ZSTD_compressBound(text.size()), '\0'); + const auto checked_size = ZSTD_compress2(cctx, checked.data(), checked.size(), text.data(), text.size()); + check(!ZSTD_isError(checked_size), "ZSTD_compress2 with a checksum"); + checked.resize(checked_size); + checked[checked.size() - 1] ^= 1; // the last byte is the checksum's + const auto damaged = ZSTD_decompress(back.data(), back.size(), checked.data(), checked.size()); + check(ZSTD_isError(damaged) && ZSTD_getErrorCode(damaged) == ZSTD_error_checksum_wrong, + "a damaged checksum is ZSTD_error_checksum_wrong"); + + // A stream, fed in pieces and flushed at the end. + check(!ZSTD_isError(ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters)), "ZSTD_CCtx_reset"); + std::string streamed(ZSTD_compressBound(text.size()), '\0'); + ZSTD_outBuffer out{streamed.data(), streamed.size(), 0}; + for (std::size_t at = 0; at < text.size(); at += 4096) { + const auto n = std::min(4096, text.size() - at); + ZSTD_inBuffer in{text.data() + at, n, 0}; + const bool last = at + n == text.size(); + std::size_t left; + do left = ZSTD_compressStream2(cctx, &out, &in, last ? ZSTD_e_end : ZSTD_e_continue); + while (!ZSTD_isError(left) && (last ? left != 0 : in.pos != in.size)); + } + ZSTD_freeCCtx(cctx); + ZSTD_DCtx* dctx = ZSTD_createDCtx(); + std::string unstreamed(text.size(), '\0'); + ZSTD_inBuffer in{streamed.data(), out.pos, 0}; + ZSTD_outBuffer whole{unstreamed.data(), unstreamed.size(), 0}; + const auto rest = ZSTD_decompressStream(dctx, &whole, &in); + check(rest == 0 && whole.pos == text.size() && unstreamed == text, "ZSTD_decompressStream restores the stream"); + ZSTD_freeDCtx(dctx); + + // A raw-content dictionary: the second text compresses against the first. + const std::string dictionary = text.substr(0, 4096); + ZSTD_CDict* cdict = ZSTD_createCDict(dictionary.data(), dictionary.size(), 3); + ZSTD_DDict* ddict = ZSTD_createDDict(dictionary.data(), dictionary.size()); + cctx = ZSTD_createCCtx(); + dctx = ZSTD_createDCtx(); + const std::string sample = text.substr(100, 600); + std::string small(ZSTD_compressBound(sample.size()), '\0'); + const auto with = ZSTD_compress_usingCDict(cctx, small.data(), small.size(), sample.data(), sample.size(), cdict); + const auto without = ZSTD_compress(frame.data(), frame.size(), sample.data(), sample.size(), 3); + check(!ZSTD_isError(with) && with < without, "the dictionary helps"); + std::string sample_back(sample.size(), '\0'); + check(ZSTD_decompress_usingDDict(dctx, sample_back.data(), sample_back.size(), small.data(), with, ddict) + == sample.size() && sample_back == sample, "ZSTD_decompress_usingDDict restores it"); + ZSTD_freeCCtx(cctx); + ZSTD_freeDCtx(dctx); + ZSTD_freeCDict(cdict); + ZSTD_freeDDict(ddict); + + // Errors and the version. + const auto garbage = ZSTD_decompress(back.data(), back.size(), "not a frame", 11); + check(ZSTD_isError(garbage) && ZSTD_getErrorCode(garbage) == ZSTD_error_prefix_unknown, + std::format("garbage is ZSTD_error_prefix_unknown ({})", ZSTD_getErrorName(garbage))); + check(ZSTD_versionNumber() == m::ZSTD_VERSION_NUMBER && std::string_view(ZSTD_versionString()) == m::ZSTD_VERSION_STRING, + "the library is the headers' version"); + check(ZSTD_COMPRESSBOUND(100000) == ZSTD_compressBound(100000), "ZSTD_COMPRESSBOUND is ZSTD_compressBound"); + std::println("{} failures", failures); + return failures != 0; +} diff --git a/tests/header_mix_test.cpp b/tests/header_mix_test.cpp new file mode 100644 index 0000000..871f2a0 --- /dev/null +++ b/tests/header_mix_test.cpp @@ -0,0 +1,15 @@ +// The header and the module in one translation unit: the same entities, so a +// context created through one spelling is freed through the other. +#include +import std; +import mcpplibs.mwrap.zstd; + +int main() +{ + ZSTD_CCtx* c = ::ZSTD_createCCtx(); // the header's declaration + const bool set = !ZSTD_isError(ZSTD_CCtx_setParameter(c, ZSTD_c_compressionLevel, ZSTD_CLEVEL_DEFAULT)); + const bool freed = ZSTD_freeCCtx(c) == 0; // the module's + const bool same = &::ZSTD_compress == &ZSTD_compress; + std::println("{}", set && freed && same ? "header and module agree" : "FAIL"); + return set && freed && same ? 0 : 1; +} diff --git a/tests/macros.cpp b/tests/macros.cpp new file mode 100644 index 0000000..bb1d65a --- /dev/null +++ b/tests/macros.cpp @@ -0,0 +1,76 @@ +// Each constant and type of mcpplibs.mwrap.zstd.macros equals its macro. A macro +// expands even in a qualified name, so the values of the headers are taken under other +// names and the macros undefined before the mapped names are used. Written by +// tools/generate.py. +#include +#include + +namespace header { +inline constexpr auto value_ZSTD_BLOCKSIZELOG_MAX = (ZSTD_BLOCKSIZELOG_MAX); +inline constexpr auto value_ZSTD_BLOCKSIZE_MAX = (ZSTD_BLOCKSIZE_MAX); +inline constexpr auto value_ZSTD_CLEVEL_DEFAULT = (ZSTD_CLEVEL_DEFAULT); +inline constexpr auto value_ZSTD_CONTENTSIZE_ERROR = (ZSTD_CONTENTSIZE_ERROR); +inline constexpr auto value_ZSTD_CONTENTSIZE_UNKNOWN = (ZSTD_CONTENTSIZE_UNKNOWN); +inline constexpr auto value_ZSTD_MAGICNUMBER = (ZSTD_MAGICNUMBER); +inline constexpr auto value_ZSTD_MAGIC_DICTIONARY = (ZSTD_MAGIC_DICTIONARY); +inline constexpr auto value_ZSTD_MAGIC_SKIPPABLE_MASK = (ZSTD_MAGIC_SKIPPABLE_MASK); +inline constexpr auto value_ZSTD_MAGIC_SKIPPABLE_START = (ZSTD_MAGIC_SKIPPABLE_START); +inline constexpr auto value_ZSTD_MAX_INPUT_SIZE = (ZSTD_MAX_INPUT_SIZE); +inline constexpr auto value_ZSTD_VERSION_MAJOR = (ZSTD_VERSION_MAJOR); +inline constexpr auto value_ZSTD_VERSION_MINOR = (ZSTD_VERSION_MINOR); +inline constexpr auto value_ZSTD_VERSION_NUMBER = (ZSTD_VERSION_NUMBER); +inline constexpr auto value_ZSTD_VERSION_RELEASE = (ZSTD_VERSION_RELEASE); +inline constexpr auto value_ZSTD_VERSION_STRING = (ZSTD_VERSION_STRING); +} +#undef ZSTD_BLOCKSIZELOG_MAX +#undef ZSTD_BLOCKSIZE_MAX +#undef ZSTD_CLEVEL_DEFAULT +#undef ZSTD_CONTENTSIZE_ERROR +#undef ZSTD_CONTENTSIZE_UNKNOWN +#undef ZSTD_MAGICNUMBER +#undef ZSTD_MAGIC_DICTIONARY +#undef ZSTD_MAGIC_SKIPPABLE_MASK +#undef ZSTD_MAGIC_SKIPPABLE_START +#undef ZSTD_MAX_INPUT_SIZE +#undef ZSTD_VERSION_MAJOR +#undef ZSTD_VERSION_MINOR +#undef ZSTD_VERSION_NUMBER +#undef ZSTD_VERSION_RELEASE +#undef ZSTD_VERSION_STRING + +import std; +import mcpplibs.mwrap.zstd.macros; + +namespace m = mcpplibs::mwrap::zstd::macros; + + +template +bool same(A a, B b) +{ + if constexpr (std::is_convertible_v && std::is_convertible_v) + return std::string_view(a) == std::string_view(b); + else + return a == b; +} + +int main() +{ + int differ = 0; + if (!same(m::ZSTD_BLOCKSIZELOG_MAX, header::value_ZSTD_BLOCKSIZELOG_MAX)) { std::println("FAIL ZSTD_BLOCKSIZELOG_MAX"); ++differ; } + if (!same(m::ZSTD_BLOCKSIZE_MAX, header::value_ZSTD_BLOCKSIZE_MAX)) { std::println("FAIL ZSTD_BLOCKSIZE_MAX"); ++differ; } + if (!same(m::ZSTD_CLEVEL_DEFAULT, header::value_ZSTD_CLEVEL_DEFAULT)) { std::println("FAIL ZSTD_CLEVEL_DEFAULT"); ++differ; } + if (!same(m::ZSTD_CONTENTSIZE_ERROR, header::value_ZSTD_CONTENTSIZE_ERROR)) { std::println("FAIL ZSTD_CONTENTSIZE_ERROR"); ++differ; } + if (!same(m::ZSTD_CONTENTSIZE_UNKNOWN, header::value_ZSTD_CONTENTSIZE_UNKNOWN)) { std::println("FAIL ZSTD_CONTENTSIZE_UNKNOWN"); ++differ; } + if (!same(m::ZSTD_MAGICNUMBER, header::value_ZSTD_MAGICNUMBER)) { std::println("FAIL ZSTD_MAGICNUMBER"); ++differ; } + if (!same(m::ZSTD_MAGIC_DICTIONARY, header::value_ZSTD_MAGIC_DICTIONARY)) { std::println("FAIL ZSTD_MAGIC_DICTIONARY"); ++differ; } + if (!same(m::ZSTD_MAGIC_SKIPPABLE_MASK, header::value_ZSTD_MAGIC_SKIPPABLE_MASK)) { std::println("FAIL ZSTD_MAGIC_SKIPPABLE_MASK"); ++differ; } + if (!same(m::ZSTD_MAGIC_SKIPPABLE_START, header::value_ZSTD_MAGIC_SKIPPABLE_START)) { std::println("FAIL ZSTD_MAGIC_SKIPPABLE_START"); ++differ; } + if (!same(m::ZSTD_MAX_INPUT_SIZE, header::value_ZSTD_MAX_INPUT_SIZE)) { std::println("FAIL ZSTD_MAX_INPUT_SIZE"); ++differ; } + if (!same(m::ZSTD_VERSION_MAJOR, header::value_ZSTD_VERSION_MAJOR)) { std::println("FAIL ZSTD_VERSION_MAJOR"); ++differ; } + if (!same(m::ZSTD_VERSION_MINOR, header::value_ZSTD_VERSION_MINOR)) { std::println("FAIL ZSTD_VERSION_MINOR"); ++differ; } + if (!same(m::ZSTD_VERSION_NUMBER, header::value_ZSTD_VERSION_NUMBER)) { std::println("FAIL ZSTD_VERSION_NUMBER"); ++differ; } + if (!same(m::ZSTD_VERSION_RELEASE, header::value_ZSTD_VERSION_RELEASE)) { std::println("FAIL ZSTD_VERSION_RELEASE"); ++differ; } + if (!same(m::ZSTD_VERSION_STRING, header::value_ZSTD_VERSION_STRING)) { std::println("FAIL ZSTD_VERSION_STRING"); ++differ; } + std::println("15 constants and 0 types compared, {} differ", differ); + return differ != 0; +} diff --git a/tools/api.py b/tools/api.py new file mode 100644 index 0000000..1627a90 --- /dev/null +++ b/tools/api.py @@ -0,0 +1,129 @@ +#!/usr/bin/env python3 +"""The public API of a set of C headers, as the compiler reads them. + +`extract` returns the functions, variables, types and enumerators that the +owned headers declare, and the macros that they define, as the compiler's AST and +preprocessor see them in a C++23 translation unit including the headers. +`constant_macros` and `type_macros` select the object-like macros whose +expansion is a constant expression or a type. tools/generate.py writes the +module from these sets and audits it against them. +""" +import json +from pathlib import Path +import re +import subprocess + + +def _run(cmd, stdin): + return subprocess.run(cmd, input=stdin, capture_output=True, text=True, check=True).stdout + + +def _owned(path, owned): + path = path.replace('\\', '/') + return any(path == name or path.endswith('/' + name) for name in owned) + + +def extract(compiler, flags, headers, owned=None): + """`flags`: the compiler's include, define and target flags for the module; + `owned`: the header files whose declarations are the API (default: `headers`).""" + owned = list(owned or headers) + source = ''.join(f'#include <{h}>\n' for h in headers) + flags = ['-x', 'c++', '-std=c++23', *flags] + ast = json.loads(_run([compiler, *flags, '-fsyntax-only', '-Xclang', '-ast-dump=json', '-'], source)) + api = {'functions': set(), 'internal': set(), 'variables': set(), 'types': set(), 'enumerators': set()} + current = [''] + + def mine(node): + # The AST names a file only where it changes from the previous node. + loc = node.get('loc', {}) + for where in (loc, loc.get('spellingLoc', {}), loc.get('expansionLoc', {})): + if 'file' in where: + current[0] = where['file'] + return _owned(current[0], owned) + + def walk(nodes): + for node in nodes: + kind = node.get('kind') + if kind == 'LinkageSpecDecl': + mine(node) + walk(node.get('inner', [])) + continue + if not mine(node) or node.get('isImplicit'): + continue + name = node.get('name') + if kind == 'EnumDecl': + api['enumerators'] |= {e['name'] for e in node.get('inner', []) if e.get('kind') == 'EnumConstantDecl'} + if not name: + continue + if kind == 'FunctionDecl': + api['internal' if node.get('storageClass') == 'static' else 'functions'].add(name) + elif kind == 'VarDecl' and node.get('storageClass') == 'extern': + api['variables'].add(name) + elif kind in ('TypedefDecl', 'TypeAliasDecl', 'RecordDecl', 'CXXRecordDecl', 'EnumDecl'): + api['types'].add(name) + walk(ast.get('inner', [])) + api['functions'] -= api['internal'] + + macros = {} + current = '' + for line in _run([compiler, *flags, '-E', '-dD', '-'], source).splitlines(): + marker = re.match(r'#\s*(?:line\s+)?\d+\s+"([^"]*)"', line) + if marker: + current = marker.group(1) + continue + define = re.match(r'#define\s+([A-Za-z_][A-Za-z_0-9]*)(\([^)]*\))?\s*(.*)$', line) + if define and _owned(current, owned): + macros[define.group(1)] = {'params': define.group(2), 'body': define.group(3).strip(), 'file': current} + undefine = re.match(r'#undef\s+([A-Za-z_][A-Za-z_0-9]*)', line) + if undefine: + macros.pop(undefine.group(1), None) + return api, macros + + +def guards(macros): + """The include guards among `macros`: an empty macro that the first + conditional directive of its file tests (`#ifndef G` or `#if !defined(G)`).""" + found = set() + for name, macro in macros.items(): + if macro['params'] is not None or macro['body']: + continue + try: + text = Path(macro['file']).read_text(encoding='utf-8', errors='replace') + except OSError: + continue + text = re.sub(r'/\*.*?\*/', ' ', text, flags=re.S) + first = re.search(r'^\s*#\s*if.*$', text, flags=re.M) + if first and re.fullmatch(r'\s*#\s*(?:ifndef\s+' + name + r'|if\s+!\s*defined\s*\(?\s*' + name + r'\s*\)?)\s*', + first.group(0)): + found.add(name) + return found + + +def _accepted(compiler, flags, headers, names, form): + """The names for which `form` (with {i} and {n}) compiles after the headers: + one translation unit, dropping the lines with an error until none remain.""" + head = ''.join(f'#include <{h}>\n' for h in headers) + pending = sorted(names) + while pending: + probe = head + '#line 1 "mwrap-probe"\n' + ''.join(form.format(i=i, n=n) + '\n' for i, n in enumerate(pending)) + result = subprocess.run([compiler, '-x', 'c++', '-std=c++23', *flags, '-fsyntax-only', '-ferror-limit=0', '-'], + input=probe, capture_output=True, text=True) + if result.returncode == 0: + return set(pending) + bad = {int(m.group(1)) - 1 for m in re.finditer(r'mwrap-probe:(\d+):\d+: error', result.stderr)} + if not bad: + raise RuntimeError(result.stderr[-2000:]) + pending = [n for i, n in enumerate(pending) if i not in bad] + return set() + + +def constant_macros(compiler, flags, headers, macros): + """The object-like macros among `macros` whose expansion is a constant expression.""" + names = [n for n, m in macros.items() if m['params'] is None and m['body']] + return _accepted(compiler, flags, headers, names, 'inline constexpr auto mwrap_probe_{i} = ({n});') + + +def type_macros(compiler, flags, headers, macros): + """The object-like macros among `macros` whose expansion is a type.""" + names = [n for n, m in macros.items() if m['params'] is None and m['body']] + return _accepted(compiler, flags, headers, names, 'using mwrap_probe_{i} = {n};') diff --git a/tools/check_mwrap_release.py b/tools/check_mwrap_release.py new file mode 100644 index 0000000..9bedddb --- /dev/null +++ b/tools/check_mwrap_release.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 +"""Check the immutable mwrap release coordinate and bilingual entry pages.""" +import argparse +import hashlib +import json +from pathlib import Path +import re +import tomllib + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--tag', help='release tag; omitted during PR checks') + args = parser.parse_args() + root = Path(__file__).resolve().parents[1] + package = tomllib.loads((root / 'mcpp.toml').read_text(encoding="utf-8"))['package'] + assert package['namespace'] == 'mcpplibs.mwrap', package + assert re.fullmatch(r'[A-Za-z_][A-Za-z_0-9]*', package['name']), package['name'] + upstream = package['metadata']['mwrap']['upstream-version'] + version = package['version'] + assert version == upstream or re.fullmatch(re.escape(upstream) + r'\.[1-9][0-9]*', version), (version, upstream) + if args.tag: + assert args.tag == 'mwrap-' + version, (args.tag, version) + assert package['license'] == 'Apache-2.0', package['license'] + assert package['authors'] == ['mcpp-community'], package.get('authors') + license_text = (root / 'LICENSE').read_text(encoding='utf-8') + assert 'Apache License' in license_text and 'Version 2.0' in license_text + assert 'mcpp-community' in (root / 'NOTICE').read_text(encoding='utf-8') + documents = json.loads((root / 'upstream/documents.json').read_text(encoding='utf-8'))['documents'] + assert any(Path(d['path']).name.startswith('README') for d in documents) + assert any(Path(d['path']).name.startswith(('LICENSE', 'COPYING')) for d in documents) + deployed = {entry['from'] for entry in tomllib.loads((root / 'mcpp.toml').read_text(encoding='utf-8')).get('runtime', {}).get('deploy', [])} + assert {'LICENSE', 'NOTICE'} <= deployed + for document in documents: + path = Path(document['path']) + assert not path.is_absolute() and '..' not in path.parts and path.parts[0] == 'upstream' + assert hashlib.sha256((root / path).read_bytes()).hexdigest() == document['sha256'], path + assert document['source'] and path.as_posix() in deployed, path + identity = package['namespace'] + '.' + package['name'] + for name, other in [('README.md', 'README.zh.md'), ('README.zh.md', 'README.md')]: + text = (root / name).read_text(encoding="utf-8") + assert len(text.splitlines()) <= 150, name + assert '](' + other + ')' in '\n'.join(text.splitlines()[:12]), name + assert identity in text and version in text, name + assert text.count('```') % 2 == 0, name + print(identity + '@' + version) + + +if __name__ == '__main__': + main() diff --git a/tools/generate.py b/tools/generate.py new file mode 100644 index 0000000..a17ef0d --- /dev/null +++ b/tools/generate.py @@ -0,0 +1,196 @@ +#!/usr/bin/env python3 +"""Write, or with --check audit, the parts of the wrapper that follow from the headers. + +tools/surface.toml names the headers and classifies what the compiler cannot: +the macros that are how the headers are written (`syntax`), and the names that +only some targets declare (`conditional`). Everything else follows from the +headers as the compiler reads them (tools/api.py), with the include, define +and target flags of the last `mcpp build` (target/**/compile_commands.json): + + src/.cppm every function, variable, type and enumerator, + `export using ::name` + src/macros.cppm every object-like macro that is a constant + expression or a type, under its own name in + mcpplibs::mwrap::::macros + include/mwrap//macros.hpp every other macro, upstream's definition + tests/macros.cpp each mapped constant and type against its macro + +The include guards are recognised and omitted. --check renders the same files +in memory: on the host named by `generated-on` they must be byte-identical; +on every host, each name the headers declare there must be exported, mapped, +kept or classified, and each unconditional export must be declared there. +The compiler is the clang of the build; run it after `mcpp build` with LLVM. +""" +import argparse +import json +from pathlib import Path +import re +import sys +import tomllib + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +import api # noqa: E402 + +ROOT = Path(__file__).resolve().parents[1] +KEEP_JOINED = ('-I', '-isystem', '-idirafter', '-iquote', '-D', '-U', '--sysroot=', '-isysroot', '--target=', + '-target', '-include', '-imsvc', '-fms-compatibility-version=', '-fms-runtime-lib=', '-fms-extensions') +KEEP_ALONE = ('-nostdinc++', '-nostdlibinc', '-nostdinc', '--no-default-config') +TAKES_VALUE = ('-I', '-isystem', '-idirafter', '-iquote', '-D', '-U', '-isysroot', '-target', '-include', '-imsvc') + + +def build_flags(name): + """The clang of the newest LLVM build and its preprocessing and target flags for src/.cppm.""" + databases = sorted((ROOT / 'target').glob('**/compile_commands.json'), key=lambda p: p.stat().st_mtime) + for database in reversed(databases): + for entry in json.loads(database.read_text(encoding='utf-8')): + words = entry.get('arguments') or entry['command'].split() + if Path(entry['file']).as_posix().endswith(f'src/{name}.cppm') and 'clang' in Path(words[0]).name: + flags, rest = [], iter(words[1:]) + for word in rest: + if word in TAKES_VALUE: + flags += [word, next(rest)] + elif word in KEEP_ALONE or word.startswith(KEEP_JOINED): + flags.append(word) + return words[0], flags + sys.exit(f'no LLVM build of src/{name}.cppm; run `mcpp build --toolchain llvm@23.1.3` first') + + +def qualify(body, mapped, scope): + return re.sub(r'(?' for h in cfg['headers']) + declared = surface['functions'] | surface['variables'] | surface['types'] | surface['enumerators'] + exported = sorted(declared - set(conditional)) + cap = f'mwrap_{name}_capture' + + module = [f"// mcpplibs.mwrap.{name} --- {cfg['title']} as a C++ module. Every exported name is", + "// the upstream entity of the headers below, unchanged, in the global namespace where", + f"// upstream declares it; the root also re-exports the macro mapping. The implementation", + f"// is compat.{cfg['compat']} {cfg['version']}. Written by tools/generate.py from the headers.", + 'module;', includes, '', f'export module mcpplibs.mwrap.{name};', '', + f'export import mcpplibs.mwrap.{name}.macros;', '', 'export {'] + module += [f'using ::{n};' for n in exported] + module.append('}') + for condition in sorted(set(conditional.values())): + module += ['', f'#if {condition}', 'export {'] + module += [f'using ::{n};' for n in sorted(conditional) if conditional[n] == condition] + module += ['}', '#endif'] + + mapped = sorted(constants | types) + macros = [f"// mcpplibs.mwrap.{name}.macros --- the macros of the headers that are a constant or a", + "// type, under their original names in " + scope + "::macros, each equal to", + "// the macro where the module is built (a named module cannot export a macro). The", + f"// other macros are in . Written by tools/generate.py.", + 'module;', includes, '', '// The values, taken under other names while the macros are defined.', + f'namespace {cap} {{'] + macros += [f'inline constexpr auto value_{n} = ({n});' if n in constants else f'using type_{n} = {n};' + for n in mapped] + macros += ['}'] + [f'#undef {n}' for n in mapped] + macros += ['', f'export module mcpplibs.mwrap.{name}.macros;', '', f'export namespace {scope}::macros {{'] + macros += [f'inline constexpr auto {n} = {cap}::value_{n};' if n in constants else f'using {n} = {cap}::type_{n};' + for n in mapped] + macros.append('}') + + guard = f'MWRAP_{name.upper()}_MACROS_HPP' + header = [f'// --- the macros of the headers that are neither a constant', + '// nor a type: they take arguments, or call a function. A named module cannot export a', + f'// macro: include this header before `import mcpplibs.mwrap.{name};`. Each definition is', + f"// upstream's ({cfg['title']}, {cfg['license']}), made only where the upstream header has", + f'// not defined it, with the constants it names read from {scope}::macros.', + '// The upstream header included after this one redefines these macros.', + '// Written by tools/generate.py.', + f'#ifndef {guard}', f'#define {guard}'] + for n in sorted(kept): + body = qualify(kept[n]['body'], set(mapped), f'{scope}::macros') + header += ['', f'#ifndef {n}', f"#define {n}{kept[n]['params'] or ''} {body}".rstrip(), '#endif'] + header += ['', f'#endif // {guard}'] + + test = [f'// Each constant and type of mcpplibs.mwrap.{name}.macros equals its macro. A macro', + '// expands even in a qualified name, so the values of the headers are taken under other', + '// names and the macros undefined before the mapped names are used. Written by', + '// tools/generate.py.', includes, '', 'namespace header {'] + test += [f'inline constexpr auto value_{n} = ({n});' if n in constants else f'using type_{n} = {n};' for n in mapped] + test += ['}'] + [f'#undef {n}' for n in mapped] + test += ['', 'import std;', f'import mcpplibs.mwrap.{name}.macros;', '', + f'namespace m = {scope}::macros;', ''] + test += [f'static_assert(std::is_same_v, "{n}");' for n in sorted(types)] + test += ['', 'template ', 'bool same(A a, B b)', '{', + ' if constexpr (std::is_convertible_v && std::is_convertible_v)', + ' return std::string_view(a) == std::string_view(b);', + ' else', + ' return a == b;', + '}', '', 'int main()', '{', ' int differ = 0;'] + test += [f' if (!same(m::{n}, header::value_{n})) {{ std::println("FAIL {n}"); ++differ; }}' for n in sorted(constants)] + test += [f' std::println("{len(constants)} constants and {len(types)} types compared, {{}} differ", differ);', + ' return differ != 0;', '}'] + join = lambda lines: '\n'.join(lines) + '\n' + return {f'src/{name}.cppm': join(module), 'src/macros.cppm': join(macros), + f'include/mwrap/{name}/macros.hpp': join(header), 'tests/macros.cpp': join(test)} + + +def main(): + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument('--check', action='store_true', help='audit instead of writing') + args = parser.parse_args() + cfg = tomllib.loads((ROOT / 'tools/surface.toml').read_text(encoding='utf-8')) + compiler, flags = build_flags(cfg['name']) + surface, macros, guards, constants, types, kept = classify(cfg, compiler, flags) + files = render(cfg, surface, constants, types, kept) + problems = [] + host = {'linux': 'linux', 'darwin': 'macos', 'win32': 'windows'}.get(sys.platform, sys.platform) + if not args.check: + for path, text in files.items(): + target = ROOT / path + target.parent.mkdir(parents=True, exist_ok=True) + if not target.exists() or target.read_text(encoding='utf-8') != text: + target.write_text(text, encoding='utf-8', newline='\n') + elif host == cfg['generated-on']: + problems += [f'differs from the headers: {p}' for p, t in files.items() + if not (ROOT / p).exists() or (ROOT / p).read_text(encoding='utf-8') != t] + else: + # Another host: its declarations against the files written on `generated-on`. + module = (ROOT / f"src/{cfg['name']}.cppm").read_text(encoding='utf-8') + written = set(re.findall(r'using ::(\w+);', module)) + unconditional = set(re.findall(r'using ::(\w+);', module.split('\n#if ')[0])) + declared = surface['functions'] | surface['variables'] | surface['types'] | surface['enumerators'] + mapping = (ROOT / 'src/macros.cppm').read_text(encoding='utf-8') + mapped = set(re.findall(r'^#undef (\w+)$', mapping, flags=re.M)) + kept_written = set(re.findall(r'^#ifndef (\w+)$', (ROOT / f"include/mwrap/{cfg['name']}/macros.hpp") + .read_text(encoding='utf-8'), flags=re.M)) + problems += [f'declared here, not exported: {n}' for n in sorted(declared - written)] + problems += [f'exported, not declared here: {n}' for n in sorted(unconditional - declared)] + problems += [f'macro here, neither mapped, kept nor classified: {n}' for n in sorted( + set(macros) - guards - set(cfg.get('syntax', [])) - mapped - kept_written)] + problems += [f'mapped, not a constant or type here: {n}' for n in sorted(mapped - constants - types)] + problems += [f'internal linkage, cannot be exported: {n}' for n in sorted(surface['internal']) + if n not in cfg.get('adapted', [])] + for problem in problems: + print(problem) + print(f"{host}: {len(surface['functions'])} functions, {len(surface['variables'])} variables, " + f"{len(surface['types'])} types, {len(surface['enumerators'])} enumerators; {len(constants)} constant and " + f"{len(types)} type macros mapped, {len(kept)} kept, {len(guards)} guards, " + f"{len(set(cfg.get('syntax', [])) & set(macros))} syntax; {len(problems)} problems") + return 1 if problems else 0 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/tools/run_conformance.py b/tools/run_conformance.py new file mode 100644 index 0000000..b23e94a --- /dev/null +++ b/tools/run_conformance.py @@ -0,0 +1,119 @@ +#!/usr/bin/env python3 +"""Run Zstandard's own examples (of the release) against compat.zstd 1.5.7, the +library this wrapper exports. + +The release tarball is downloaded once into target/upstream and checked +against upstream/documents.json. simple_compression.c, simple_decompression.c, +streaming_compression.c, streaming_decompression.c and their common.h are +copied unchanged into target/conformance/examples, and a project there builds +them as C over compat.zstd (a C program includes ; the module's own +tests are tests/*.cpp). The release's README.md is compressed in one shot and +decoded in memory, then compressed as a stream and decoded as a stream; the +decoded bytes must equal the original. +""" +import argparse +import hashlib +import json +from pathlib import Path +import shutil +import subprocess +import sys +import tarfile +import urllib.request + +MANIFEST = '''[package] +name = "zstd-conformance" +version = "1.5.7" +description = "Zstandard 1.5.7's examples over compat.zstd (tools/run_conformance.py)" + +[targets.simple_compression] +kind = "bin" +main = "examples/simple_compression.c" + +[targets.simple_decompression] +kind = "bin" +main = "examples/simple_decompression.c" + +[targets.streaming_compression] +kind = "bin" +main = "examples/streaming_compression.c" + +[targets.streaming_decompression] +kind = "bin" +main = "examples/streaming_decompression.c" + +[build] +c_standard = "c11" +sources = [] +# common.h reads file sizes with stat() and off_t, which ISO C does not declare. +cflags = ["-D_POSIX_C_SOURCE=200809L"] + +[dependencies] +compat.zstd = "1.5.7" +''' + + +def fetch(root: Path) -> Path: + pins = json.loads((root / 'upstream/documents.json').read_text(encoding='utf-8')) + cache = root / 'target/upstream' + cache.mkdir(parents=True, exist_ok=True) + archive = cache / 'zstd-release.tar.gz' + if not archive.exists() or hashlib.sha256(archive.read_bytes()).hexdigest() != pins['upstream_sha256']: + print('+ fetch', pins['upstream'], flush=True) + with urllib.request.urlopen(pins['upstream'], timeout=120) as response: + archive.write_bytes(response.read()) + digest = hashlib.sha256(archive.read_bytes()).hexdigest() + if digest != pins['upstream_sha256']: + sys.exit(f'upstream archive checksum {digest} != {pins["upstream_sha256"]}') + with tarfile.open(archive) as tar: + top = tar.getmembers()[0].name.split('/')[0] + if not (cache / top).exists(): + tar.extractall(cache, filter='data') + return cache / top + + +def run(command, cwd, **kwargs): + print('+', ' '.join(map(str, command)), flush=True) + return subprocess.run(command, cwd=cwd, check=True, **kwargs) + + +EXAMPLES = ('simple_compression', 'simple_decompression', 'streaming_compression', 'streaming_decompression') + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--mcpp', default='mcpp') + parser.add_argument('--toolchain') + args = parser.parse_args() + root = Path(__file__).resolve().parents[1] + upstream = fetch(root) + work = root / 'target/conformance' + shutil.rmtree(work, ignore_errors=True) + (work / 'examples').mkdir(parents=True) + for name in [e + '.c' for e in EXAMPLES] + ['common.h']: + shutil.copyfile(upstream / 'examples' / name, work / 'examples' / name) + (work / 'mcpp.toml').write_text(MANIFEST, encoding='utf-8') + build = [args.mcpp, 'build', '--release'] + if args.toolchain: + build += ['--toolchain', args.toolchain] + run(build, work) + binaries = {p.stem: p for p in (work / 'target').rglob('*') if p.is_file() and p.stem in EXAMPLES + and p.suffix in ('', '.exe')} + missing = [e for e in EXAMPLES if e not in binaries] + if missing: + sys.exit(f'did not build: {missing}') + sample = work / 'sample.txt' + shutil.copyfile(upstream / 'README.md', sample) + run([binaries['simple_compression'], sample.name], work) + run([binaries['simple_decompression'], sample.name + '.zst'], work) + run([binaries['streaming_compression'], sample.name], work) + decoded = run([binaries['streaming_decompression'], sample.name + '.zst'], work, capture_output=True).stdout + if sys.platform == 'win32': # the example writes to stdout, which Windows opens in text mode + decoded = decoded.replace(b'\r\n', b'\n') + if decoded != sample.read_bytes(): + sys.exit('streaming_decompression did not restore the sample') + print('Zstandard 1.5.7: the examples round-trip over compat.zstd', flush=True) + + +if __name__ == '__main__': + main() diff --git a/tools/surface.toml b/tools/surface.toml new file mode 100644 index 0000000..2f815e6 --- /dev/null +++ b/tools/surface.toml @@ -0,0 +1,18 @@ +# What tools/generate.py cannot read from the headers; everything else it can. +name = "zstd" +title = "Zstandard 1.5.7" +compat = "zstd" +version = "1.5.7" +license = "BSD license or GPLv2" +generated-on = "linux" +headers = ["zstd.h", "zstd_errors.h"] +owned = ["zstd.h", "zstd_errors.h"] + +# How the headers write the declarations, not what Zstandard offers: symbol +# visibility (ZSTDLIB_*, ZSTDERRORLIB_*), the deprecation attribute, and the +# stringification steps that build ZSTD_VERSION_STRING, which is mapped. +syntax = [ + "ZSTDLIB_API", "ZSTDLIB_HIDDEN", "ZSTDLIB_VISIBLE", + "ZSTDERRORLIB_API", "ZSTDERRORLIB_HIDDEN", "ZSTDERRORLIB_VISIBLE", + "ZSTD_DEPRECATED", "ZSTD_QUOTE", "ZSTD_EXPAND_AND_QUOTE", "ZSTD_LIB_VERSION", +] diff --git a/tools/test_template.py b/tools/test_template.py new file mode 100644 index 0000000..81ea805 --- /dev/null +++ b/tools/test_template.py @@ -0,0 +1,83 @@ +#!/usr/bin/env python3 +"""Render an installed candidate's default mcpp template, then build and run it.""" +import argparse +import json +import os +from pathlib import Path +import shutil +import subprocess +import tempfile +import tomllib + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--mcpp", default="mcpp") + parser.add_argument("--namespace", help="index namespace for legacy manifests without package.namespace") + parser.add_argument("--toolchain", default="llvm@23.1.3") + parser.add_argument("--template", help="named template; omit for the package default") + parser.add_argument("--run-arg", action="append", default=[]) + parser.add_argument("--index", type=Path, help="candidate SDK/wrapper index for unpublished feature dependencies") + args = parser.parse_args() + root = Path(__file__).resolve().parents[1] + metadata = tomllib.loads((root / "mcpp.toml").read_text())["package"] + namespace = metadata.get("namespace", args.namespace) + assert namespace, "Supply the published index namespace for this legacy manifest" + name, version = metadata["name"], metadata["version"] + identity = namespace + "." + name + with tempfile.TemporaryDirectory(prefix=name + "-template-") as directory: + temporary = Path(directory) + home = temporary / "home" + package = home / "registry/data/xpkgs" / (namespace + "-x-" + name) / version / (name + "-" + version) + shutil.copytree(root, package, ignore=shutil.ignore_patterns( + ".git", "target", ".mcpp", ".xlings", ".candidate-index", "__pycache__", "mcpp.lock", "compile_commands.json", "mcpp-2026.*")) + index = home / "indices/mcpplibs/pkgs/mwrap" + index.mkdir(parents=True) + entry = '{url="https://example.invalid/candidate.tar.gz",sha256="fixture"}' + descriptor = ('package={spec="1",namespace=' + json.dumps(namespace) + ',name=' + json.dumps(name) + + ',description="Installed template fixture",type="package",mcpp="mcpp.toml",xpm={') + descriptor += ",".join(platform + "={[" + json.dumps(version) + "]=" + entry + "}" + for platform in ("linux", "macosx", "windows")) + "}}\n" + (index / (identity + ".lua")).write_text(descriptor) + env = dict(os.environ, MCPP_HOME=str(home)) + selector = identity + "@" + version + for command in ([args.mcpp, "new", "--list-templates", selector], + [args.mcpp, "new", "quickstart", "--template", selector + (":" + args.template if args.template else "")]): + print("+", " ".join(command), flush=True) + subprocess.run(command, cwd=temporary, env=env, check=True) + project = temporary / "quickstart" + manifest = project / "mcpp.toml" + text = manifest.read_text() + dependency = tomllib.loads(text)["dependencies"][identity] + options = {"version": dependency} if isinstance(dependency, str) else dependency + assert options["version"] == version, options + replacement = '{ path = ' + json.dumps(package.as_posix()) + if options.get("features"): + replacement += ', features = ' + json.dumps(options["features"]) + replacement += ' }' + lines = text.splitlines() + matches = [i for i, line in enumerate(lines) if line.startswith(json.dumps(identity) + " =")] + assert len(matches) == 1, "Expected exactly one qualified dependency from self.name" + lines[matches[0]] = json.dumps(identity) + " = " + replacement + manifest.write_text("\n".join(lines) + "\n") + if args.index: + candidate_index = args.index.resolve() + subprocess.run([os.sys.executable, str(candidate_index / "tests/prepare_mwrap_indices.py")], + cwd=candidate_index, check=True) + with manifest.open("a") as out: + out.write("\n[indices]\n") + for namespace in ("compat", "mcpplibs.mwrap"): + view = candidate_index / "tests/mwrap/.indices" / namespace + out.write(json.dumps(namespace) + ' = { path = ' + json.dumps(view.as_posix()) + ' }\n') + command = [args.mcpp, "build", "--toolchain", args.toolchain, "--strict"] + print("+", " ".join(command), flush=True) + subprocess.run(command, cwd=project, check=True) + command = [args.mcpp, "run", "--toolchain", args.toolchain] + if args.run_arg: + command += ["--", *args.run_arg] + print("+", " ".join(command), flush=True) + subprocess.run(command, cwd=project, check=True) + + +if __name__ == "__main__": + main() diff --git a/upstream/documents.json b/upstream/documents.json new file mode 100644 index 0000000..684f781 --- /dev/null +++ b/upstream/documents.json @@ -0,0 +1,22 @@ +{ + "upstream": "https://github.com/facebook/zstd/archive/refs/tags/v1.5.7.tar.gz", + "upstream_sha256": "37d7284556b20954e56e1ca85b80226768902e2edabd3b649e9e72c0c9012ee3", + "pruning": "Nothing of Zstandard is vendored: the library is compat.zstd 1.5.7's, from the same tarball, and the macro of include/mwrap/zstd/macros.hpp is upstream's definition, written out by tools/generate.py. Its README and both licenses are retained here. The conformance run builds four of the release's examples against compat.zstd, from the tarball downloaded at test time.", + "documents": [ + { + "path": "upstream/zstd/README.md", + "source": "https://github.com/facebook/zstd/blob/v1.5.7/README.md", + "sha256": "c97c5752f0cf8e235e5834726968708ff1f4b3bf6bfbfcf59db0229b5889ac35" + }, + { + "path": "upstream/zstd/LICENSE", + "source": "https://github.com/facebook/zstd/blob/v1.5.7/LICENSE", + "sha256": "7055266497633c9025b777c78eb7235af13922117480ed5c674677adc381c9d8" + }, + { + "path": "upstream/zstd/COPYING", + "source": "https://github.com/facebook/zstd/blob/v1.5.7/COPYING", + "sha256": "f9c375a1be4a41f7b70301dd83c91cb89e41567478859b77eef375a52d782505" + } + ] +} diff --git a/upstream/zstd/COPYING b/upstream/zstd/COPYING new file mode 100644 index 0000000..ecbc059 --- /dev/null +++ b/upstream/zstd/COPYING @@ -0,0 +1,339 @@ + GNU GENERAL PUBLIC LICENSE + Version 2, June 1991 + + Copyright (C) 1989, 1991 Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The licenses for most software are designed to take away your +freedom to share and change it. By contrast, the GNU General Public +License is intended to guarantee your freedom to share and change free +software--to make sure the software is free for all its users. This +General Public License applies to most of the Free Software +Foundation's software and to any other program whose authors commit to +using it. (Some other Free Software Foundation software is covered by +the GNU Lesser General Public License instead.) You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +this service if you wish), that you receive source code or can get it +if you want it, that you can change the software or use pieces of it +in new free programs; and that you know you can do these things. + + To protect your rights, we need to make restrictions that forbid +anyone to deny you these rights or to ask you to surrender the rights. +These restrictions translate to certain responsibilities for you if you +distribute copies of the software, or if you modify it. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must give the recipients all the rights that +you have. You must make sure that they, too, receive or can get the +source code. And you must show them these terms so they know their +rights. + + We protect your rights with two steps: (1) copyright the software, and +(2) offer you this license which gives you legal permission to copy, +distribute and/or modify the software. + + Also, for each author's protection and ours, we want to make certain +that everyone understands that there is no warranty for this free +software. If the software is modified by someone else and passed on, we +want its recipients to know that what they have is not the original, so +that any problems introduced by others will not reflect on the original +authors' reputations. + + Finally, any free program is threatened constantly by software +patents. We wish to avoid the danger that redistributors of a free +program will individually obtain patent licenses, in effect making the +program proprietary. To prevent this, we have made it clear that any +patent must be licensed for everyone's free use or not licensed at all. + + The precise terms and conditions for copying, distribution and +modification follow. + + GNU GENERAL PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. This License applies to any program or other work which contains +a notice placed by the copyright holder saying it may be distributed +under the terms of this General Public License. The "Program", below, +refers to any such program or work, and a "work based on the Program" +means either the Program or any derivative work under copyright law: +that is to say, a work containing the Program or a portion of it, +either verbatim or with modifications and/or translated into another +language. (Hereinafter, translation is included without limitation in +the term "modification".) Each licensee is addressed as "you". + +Activities other than copying, distribution and modification are not +covered by this License; they are outside its scope. The act of +running the Program is not restricted, and the output from the Program +is covered only if its contents constitute a work based on the +Program (independent of having been made by running the Program). +Whether that is true depends on what the Program does. + + 1. You may copy and distribute verbatim copies of the Program's +source code as you receive it, in any medium, provided that you +conspicuously and appropriately publish on each copy an appropriate +copyright notice and disclaimer of warranty; keep intact all the +notices that refer to this License and to the absence of any warranty; +and give any other recipients of the Program a copy of this License +along with the Program. + +You may charge a fee for the physical act of transferring a copy, and +you may at your option offer warranty protection in exchange for a fee. + + 2. You may modify your copy or copies of the Program or any portion +of it, thus forming a work based on the Program, and copy and +distribute such modifications or work under the terms of Section 1 +above, provided that you also meet all of these conditions: + + a) You must cause the modified files to carry prominent notices + stating that you changed the files and the date of any change. + + b) You must cause any work that you distribute or publish, that in + whole or in part contains or is derived from the Program or any + part thereof, to be licensed as a whole at no charge to all third + parties under the terms of this License. + + c) If the modified program normally reads commands interactively + when run, you must cause it, when started running for such + interactive use in the most ordinary way, to print or display an + announcement including an appropriate copyright notice and a + notice that there is no warranty (or else, saying that you provide + a warranty) and that users may redistribute the program under + these conditions, and telling the user how to view a copy of this + License. (Exception: if the Program itself is interactive but + does not normally print such an announcement, your work based on + the Program is not required to print an announcement.) + +These requirements apply to the modified work as a whole. If +identifiable sections of that work are not derived from the Program, +and can be reasonably considered independent and separate works in +themselves, then this License, and its terms, do not apply to those +sections when you distribute them as separate works. But when you +distribute the same sections as part of a whole which is a work based +on the Program, the distribution of the whole must be on the terms of +this License, whose permissions for other licensees extend to the +entire whole, and thus to each and every part regardless of who wrote it. + +Thus, it is not the intent of this section to claim rights or contest +your rights to work written entirely by you; rather, the intent is to +exercise the right to control the distribution of derivative or +collective works based on the Program. + +In addition, mere aggregation of another work not based on the Program +with the Program (or with a work based on the Program) on a volume of +a storage or distribution medium does not bring the other work under +the scope of this License. + + 3. You may copy and distribute the Program (or a work based on it, +under Section 2) in object code or executable form under the terms of +Sections 1 and 2 above provided that you also do one of the following: + + a) Accompany it with the complete corresponding machine-readable + source code, which must be distributed under the terms of Sections + 1 and 2 above on a medium customarily used for software interchange; or, + + b) Accompany it with a written offer, valid for at least three + years, to give any third party, for a charge no more than your + cost of physically performing source distribution, a complete + machine-readable copy of the corresponding source code, to be + distributed under the terms of Sections 1 and 2 above on a medium + customarily used for software interchange; or, + + c) Accompany it with the information you received as to the offer + to distribute corresponding source code. (This alternative is + allowed only for noncommercial distribution and only if you + received the program in object code or executable form with such + an offer, in accord with Subsection b above.) + +The source code for a work means the preferred form of the work for +making modifications to it. For an executable work, complete source +code means all the source code for all modules it contains, plus any +associated interface definition files, plus the scripts used to +control compilation and installation of the executable. However, as a +special exception, the source code distributed need not include +anything that is normally distributed (in either source or binary +form) with the major components (compiler, kernel, and so on) of the +operating system on which the executable runs, unless that component +itself accompanies the executable. + +If distribution of executable or object code is made by offering +access to copy from a designated place, then offering equivalent +access to copy the source code from the same place counts as +distribution of the source code, even though third parties are not +compelled to copy the source along with the object code. + + 4. You may not copy, modify, sublicense, or distribute the Program +except as expressly provided under this License. Any attempt +otherwise to copy, modify, sublicense or distribute the Program is +void, and will automatically terminate your rights under this License. +However, parties who have received copies, or rights, from you under +this License will not have their licenses terminated so long as such +parties remain in full compliance. + + 5. You are not required to accept this License, since you have not +signed it. However, nothing else grants you permission to modify or +distribute the Program or its derivative works. These actions are +prohibited by law if you do not accept this License. Therefore, by +modifying or distributing the Program (or any work based on the +Program), you indicate your acceptance of this License to do so, and +all its terms and conditions for copying, distributing or modifying +the Program or works based on it. + + 6. Each time you redistribute the Program (or any work based on the +Program), the recipient automatically receives a license from the +original licensor to copy, distribute or modify the Program subject to +these terms and conditions. You may not impose any further +restrictions on the recipients' exercise of the rights granted herein. +You are not responsible for enforcing compliance by third parties to +this License. + + 7. If, as a consequence of a court judgment or allegation of patent +infringement or for any other reason (not limited to patent issues), +conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot +distribute so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you +may not distribute the Program at all. For example, if a patent +license would not permit royalty-free redistribution of the Program by +all those who receive copies directly or indirectly through you, then +the only way you could satisfy both it and this License would be to +refrain entirely from distribution of the Program. + +If any portion of this section is held invalid or unenforceable under +any particular circumstance, the balance of the section is intended to +apply and the section as a whole is intended to apply in other +circumstances. + +It is not the purpose of this section to induce you to infringe any +patents or other property right claims or to contest validity of any +such claims; this section has the sole purpose of protecting the +integrity of the free software distribution system, which is +implemented by public license practices. Many people have made +generous contributions to the wide range of software distributed +through that system in reliance on consistent application of that +system; it is up to the author/donor to decide if he or she is willing +to distribute software through any other system and a licensee cannot +impose that choice. + +This section is intended to make thoroughly clear what is believed to +be a consequence of the rest of this License. + + 8. If the distribution and/or use of the Program is restricted in +certain countries either by patents or by copyrighted interfaces, the +original copyright holder who places the Program under this License +may add an explicit geographical distribution limitation excluding +those countries, so that distribution is permitted only in or among +countries not thus excluded. In such case, this License incorporates +the limitation as if written in the body of this License. + + 9. The Free Software Foundation may publish revised and/or new versions +of the General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + +Each version is given a distinguishing version number. If the Program +specifies a version number of this License which applies to it and "any +later version", you have the option of following the terms and conditions +either of that version or of any later version published by the Free +Software Foundation. If the Program does not specify a version number of +this License, you may choose any version ever published by the Free Software +Foundation. + + 10. If you wish to incorporate parts of the Program into other free +programs whose distribution conditions are different, write to the author +to ask for permission. For software which is copyrighted by the Free +Software Foundation, write to the Free Software Foundation; we sometimes +make exceptions for this. Our decision will be guided by the two goals +of preserving the free status of all derivatives of our free software and +of promoting the sharing and reuse of software generally. + + NO WARRANTY + + 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY +FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN +OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES +PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED +OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS +TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE +PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, +REPAIR OR CORRECTION. + + 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR +REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, +INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING +OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED +TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY +YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER +PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE +POSSIBILITY OF SUCH DAMAGES. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +convey the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +Also add information on how to contact you by electronic and paper mail. + +If the program is interactive, make it output a short notice like this +when it starts in an interactive mode: + + Gnomovision version 69, Copyright (C) year name of author + Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, the commands you use may +be called something other than `show w' and `show c'; they could even be +mouse-clicks or menu items--whatever suits your program. + +You should also get your employer (if you work as a programmer) or your +school, if any, to sign a "copyright disclaimer" for the program, if +necessary. Here is a sample; alter the names: + + Yoyodyne, Inc., hereby disclaims all copyright interest in the program + `Gnomovision' (which makes passes at compilers) written by James Hacker. + + , 1 April 1989 + Ty Coon, President of Vice + +This General Public License does not permit incorporating your program into +proprietary programs. If your program is a subroutine library, you may +consider it more useful to permit linking proprietary applications with the +library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. \ No newline at end of file diff --git a/upstream/zstd/LICENSE b/upstream/zstd/LICENSE new file mode 100644 index 0000000..7580028 --- /dev/null +++ b/upstream/zstd/LICENSE @@ -0,0 +1,30 @@ +BSD License + +For Zstandard software + +Copyright (c) Meta Platforms, Inc. and affiliates. All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + * Neither the name Facebook, nor Meta, nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/upstream/zstd/README.md b/upstream/zstd/README.md new file mode 100644 index 0000000..d91ef5d --- /dev/null +++ b/upstream/zstd/README.md @@ -0,0 +1,237 @@ +

Zstandard

+ +__Zstandard__, or `zstd` as short version, is a fast lossless compression algorithm, +targeting real-time compression scenarios at zlib-level and better compression ratios. +It's backed by a very fast entropy stage, provided by [Huff0 and FSE library](https://github.com/Cyan4973/FiniteStateEntropy). + +Zstandard's format is stable and documented in [RFC8878](https://datatracker.ietf.org/doc/html/rfc8878). Multiple independent implementations are already available. +This repository represents the reference implementation, provided as an open-source dual [BSD](LICENSE) OR [GPLv2](COPYING) licensed **C** library, +and a command line utility producing and decoding `.zst`, `.gz`, `.xz` and `.lz4` files. +Should your project require another programming language, +a list of known ports and bindings is provided on [Zstandard homepage](https://facebook.github.io/zstd/#other-languages). + +**Development branch status:** + +[![Build Status][travisDevBadge]][travisLink] +[![Build status][CircleDevBadge]][CircleLink] +[![Build status][CirrusDevBadge]][CirrusLink] +[![Fuzzing Status][OSSFuzzBadge]][OSSFuzzLink] + +[travisDevBadge]: https://api.travis-ci.com/facebook/zstd.svg?branch=dev "Continuous Integration test suite" +[travisLink]: https://travis-ci.com/facebook/zstd +[CircleDevBadge]: https://circleci.com/gh/facebook/zstd/tree/dev.svg?style=shield "Short test suite" +[CircleLink]: https://circleci.com/gh/facebook/zstd +[CirrusDevBadge]: https://api.cirrus-ci.com/github/facebook/zstd.svg?branch=dev +[CirrusLink]: https://cirrus-ci.com/github/facebook/zstd +[OSSFuzzBadge]: https://oss-fuzz-build-logs.storage.googleapis.com/badges/zstd.svg +[OSSFuzzLink]: https://bugs.chromium.org/p/oss-fuzz/issues/list?sort=-opened&can=1&q=proj:zstd + +## Benchmarks + +For reference, several fast compression algorithms were tested and compared +on a desktop featuring a Core i7-9700K CPU @ 4.9GHz +and running Ubuntu 20.04 (`Linux ubu20 5.15.0-101-generic`), +using [lzbench], an open-source in-memory benchmark by @inikep +compiled with [gcc] 9.4.0, +on the [Silesia compression corpus]. + +[lzbench]: https://github.com/inikep/lzbench +[Silesia compression corpus]: https://sun.aei.polsl.pl//~sdeor/index.php?page=silesia +[gcc]: https://gcc.gnu.org/ + +| Compressor name | Ratio | Compression| Decompress.| +| --------------- | ------| -----------| ---------- | +| **zstd 1.5.6 -1** | 2.887 | 510 MB/s | 1580 MB/s | +| [zlib] 1.2.11 -1 | 2.743 | 95 MB/s | 400 MB/s | +| brotli 1.0.9 -0 | 2.702 | 395 MB/s | 430 MB/s | +| **zstd 1.5.6 --fast=1** | 2.437 | 545 MB/s | 1890 MB/s | +| **zstd 1.5.6 --fast=3** | 2.239 | 650 MB/s | 2000 MB/s | +| quicklz 1.5.0 -1 | 2.238 | 525 MB/s | 750 MB/s | +| lzo1x 2.10 -1 | 2.106 | 650 MB/s | 825 MB/s | +| [lz4] 1.9.4 | 2.101 | 700 MB/s | 4000 MB/s | +| lzf 3.6 -1 | 2.077 | 420 MB/s | 830 MB/s | +| snappy 1.1.9 | 2.073 | 530 MB/s | 1660 MB/s | + +[zlib]: https://www.zlib.net/ +[lz4]: https://lz4.github.io/lz4/ + +The negative compression levels, specified with `--fast=#`, +offer faster compression and decompression speed +at the cost of compression ratio. + +Zstd can also offer stronger compression ratios at the cost of compression speed. +Speed vs Compression trade-off is configurable by small increments. +Decompression speed is preserved and remains roughly the same at all settings, +a property shared by most LZ compression algorithms, such as [zlib] or lzma. + +The following tests were run +on a server running Linux Debian (`Linux version 4.14.0-3-amd64`) +with a Core i7-6700K CPU @ 4.0GHz, +using [lzbench], an open-source in-memory benchmark by @inikep +compiled with [gcc] 7.3.0, +on the [Silesia compression corpus]. + +Compression Speed vs Ratio | Decompression Speed +---------------------------|-------------------- +![Compression Speed vs Ratio](doc/images/CSpeed2.png "Compression Speed vs Ratio") | ![Decompression Speed](doc/images/DSpeed3.png "Decompression Speed") + +A few other algorithms can produce higher compression ratios at slower speeds, falling outside of the graph. +For a larger picture including slow modes, [click on this link](doc/images/DCspeed5.png). + + +## The case for Small Data compression + +Previous charts provide results applicable to typical file and stream scenarios (several MB). Small data comes with different perspectives. + +The smaller the amount of data to compress, the more difficult it is to compress. This problem is common to all compression algorithms, and reason is, compression algorithms learn from past data how to compress future data. But at the beginning of a new data set, there is no "past" to build upon. + +To solve this situation, Zstd offers a __training mode__, which can be used to tune the algorithm for a selected type of data. +Training Zstandard is achieved by providing it with a few samples (one file per sample). The result of this training is stored in a file called "dictionary", which must be loaded before compression and decompression. +Using this dictionary, the compression ratio achievable on small data improves dramatically. + +The following example uses the `github-users` [sample set](https://github.com/facebook/zstd/releases/tag/v1.1.3), created from [github public API](https://developer.github.com/v3/users/#get-all-users). +It consists of roughly 10K records weighing about 1KB each. + +Compression Ratio | Compression Speed | Decompression Speed +------------------|-------------------|-------------------- +![Compression Ratio](doc/images/dict-cr.png "Compression Ratio") | ![Compression Speed](doc/images/dict-cs.png "Compression Speed") | ![Decompression Speed](doc/images/dict-ds.png "Decompression Speed") + + +These compression gains are achieved while simultaneously providing _faster_ compression and decompression speeds. + +Training works if there is some correlation in a family of small data samples. The more data-specific a dictionary is, the more efficient it is (there is no _universal dictionary_). +Hence, deploying one dictionary per type of data will provide the greatest benefits. +Dictionary gains are mostly effective in the first few KB. Then, the compression algorithm will gradually use previously decoded content to better compress the rest of the file. + +### Dictionary compression How To: + +1. Create the dictionary + + `zstd --train FullPathToTrainingSet/* -o dictionaryName` + +2. Compress with dictionary + + `zstd -D dictionaryName FILE` + +3. Decompress with dictionary + + `zstd -D dictionaryName --decompress FILE.zst` + + +## Build instructions + +`make` is the officially maintained build system of this project. +All other build systems are "compatible" and 3rd-party maintained, +they may feature small differences in advanced options. +When your system allows it, prefer using `make` to build `zstd` and `libzstd`. + +### Makefile + +If your system is compatible with standard `make` (or `gmake`), +invoking `make` in root directory will generate `zstd` cli in root directory. +It will also create `libzstd` into `lib/`. + +Other available options include: +- `make install` : create and install zstd cli, library and man pages +- `make check` : create and run `zstd`, test its behavior on local platform + +The `Makefile` follows the [GNU Standard Makefile conventions](https://www.gnu.org/prep/standards/html_node/Makefile-Conventions.html), +allowing staged install, standard flags, directory variables and command variables. + +For advanced use cases, specialized compilation flags which control binary generation +are documented in [`lib/README.md`](lib/README.md#modular-build) for the `libzstd` library +and in [`programs/README.md`](programs/README.md#compilation-variables) for the `zstd` CLI. + +### cmake + +A `cmake` project generator is provided within `build/cmake`. +It can generate Makefiles or other build scripts +to create `zstd` binary, and `libzstd` dynamic and static libraries. + +By default, `CMAKE_BUILD_TYPE` is set to `Release`. + +#### Support for Fat (Universal2) Output + +`zstd` can be built and installed with support for both Apple Silicon (M1/M2) as well as Intel by using CMake's Universal2 support. +To perform a Fat/Universal2 build and install use the following commands: + +```bash +cmake -B build-cmake-debug -S build/cmake -G Ninja -DCMAKE_OSX_ARCHITECTURES="x86_64;x86_64h;arm64" +cd build-cmake-debug +ninja +sudo ninja install +``` + +### Meson + +A Meson project is provided within [`build/meson`](build/meson). Follow +build instructions in that directory. + +You can also take a look at [`.travis.yml`](.travis.yml) file for an +example about how Meson is used to build this project. + +Note that default build type is **release**. + +### VCPKG +You can build and install zstd [vcpkg](https://github.com/Microsoft/vcpkg/) dependency manager: + + git clone https://github.com/Microsoft/vcpkg.git + cd vcpkg + ./bootstrap-vcpkg.sh + ./vcpkg integrate install + ./vcpkg install zstd + +The zstd port in vcpkg is kept up to date by Microsoft team members and community contributors. +If the version is out of date, please [create an issue or pull request](https://github.com/Microsoft/vcpkg) on the vcpkg repository. + +### Conan + +You can install pre-built binaries for zstd or build it from source using [Conan](https://conan.io/). Use the following command: + +```bash +conan install --requires="zstd/[*]" --build=missing +``` + +The zstd Conan recipe is kept up to date by Conan maintainers and community contributors. +If the version is out of date, please [create an issue or pull request](https://github.com/conan-io/conan-center-index) on the ConanCenterIndex repository. + +### Visual Studio (Windows) + +Going into `build` directory, you will find additional possibilities: +- Projects for Visual Studio 2005, 2008 and 2010. + + VS2010 project is compatible with VS2012, VS2013, VS2015 and VS2017. +- Automated build scripts for Visual compiler by [@KrzysFR](https://github.com/KrzysFR), in `build/VS_scripts`, + which will build `zstd` cli and `libzstd` library without any need to open Visual Studio solution. + +### Buck + +You can build the zstd binary via buck by executing: `buck build programs:zstd` from the root of the repo. +The output binary will be in `buck-out/gen/programs/`. + +### Bazel + +You easily can integrate zstd into your Bazel project by using the module hosted on the [Bazel Central Repository](https://registry.bazel.build/modules/zstd). + +## Testing + +You can run quick local smoke tests by running `make check`. +If you can't use `make`, execute the `playTest.sh` script from the `src/tests` directory. +Two env variables `$ZSTD_BIN` and `$DATAGEN_BIN` are needed for the test script to locate the `zstd` and `datagen` binary. +For information on CI testing, please refer to `TESTING.md`. + +## Status + +Zstandard is currently deployed within Facebook and many other large cloud infrastructures. +It is run continuously to compress large amounts of data in multiple formats and use cases. +Zstandard is considered safe for production environments. + +## License + +Zstandard is dual-licensed under [BSD](LICENSE) OR [GPLv2](COPYING). + +## Contributing + +The `dev` branch is the one where all contributions are merged before reaching `release`. +If you plan to propose a patch, please commit into the `dev` branch, or its own feature branch. +Direct commit to `release` are not permitted. +For more information, please read [CONTRIBUTING](CONTRIBUTING.md). From 78bdbb471a4778efeed2891ea83f20c31389459c Mon Sep 17 00:00:00 2001 From: Sunrisepeak Date: Sat, 7 Nov 2026 06:09:24 +0900 Subject: [PATCH 2/4] generate: link every exported function; reserved names and NULL; compat-unbuilt functions Co-authored-by: speak-agent <248744407+speak-agent@users.noreply.github.com> --- .github/workflows/ci.yml | 2 +- README.md | 4 +- README.zh.md | 4 +- include/mwrap/zstd/macros.hpp | 3 +- tests/linked.cpp | 83 +++++++++++++++++++++++++++++++++++ tools/generate.py | 40 ++++++++++++----- 6 files changed, 120 insertions(+), 16 deletions(-) create mode 100644 tests/linked.cpp diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1b4654b..5286eb4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -76,7 +76,7 @@ jobs: - name: Build the modules run: mcpp build $TARGET - - name: Tests (one shot, parameters and checksums, streams, dictionaries, errors, the constants, header mixing) + - name: Tests (one shot, parameters and checksums, streams, dictionaries, errors, the constants, every function linked, header mixing) run: mcpp test $TARGET - name: Export audit (the headers as clang reads them on this host) diff --git a/README.md b/README.md index c7d30ba..3253695 100644 --- a/README.md +++ b/README.md @@ -75,10 +75,10 @@ Linux, macOS and Windows CI build the modules with GCC and LLVM and run the tests: a one-shot round trip, a level-19 frame with a checksum that detects damage, a stream fed in pieces, a raw-content dictionary, the error code of garbage input, the version and the bound against their macros, every constant -against its macro, and header mixing. CI also runs: +against its macro, every exported function linked, and header mixing. CI also runs: - the export audit ([tools/generate.py](tools/generate.py) `--check`): the - module, the macro mapping, the macro header and the constant test are + module, the macro mapping, the macro header and the generated tests are regenerated from the headers as clang reads them and compared, and on macOS and Windows every name the headers declare there is exported; - four of upstream's examples over the same library diff --git a/README.zh.md b/README.zh.md index 3391adc..49fa4e0 100644 --- a/README.zh.md +++ b/README.zh.md @@ -61,10 +61,10 @@ int main() { ## 构建与测试 Linux、macOS、Windows 的 CI 用 GCC 与 LLVM 构建模块并运行测试：一次调用的往返、带校验和的 19 级帧及其损坏检测、分段 -输入的流、原始内容字典、无效输入的错误码、版本与上界同其宏相等、每个常量同其宏相等、头文件混用。CI 还运行： +输入的流、原始内容字典、无效输入的错误码、版本与上界同其宏相等、每个常量同其宏相等、每个导出函数都能链接、头文件混用。CI 还运行： - 导出审计（[tools/generate.py](tools/generate.py) `--check`）：按 clang 读到的头文件重新生成模块、宏映射、宏头文件与 - 常量测试并逐字节比较；在 macOS 与 Windows 上检查头文件在该平台声明的每个名字都已导出； + 生成的测试并逐字节比较；在 macOS 与 Windows 上检查头文件在该平台声明的每个名字都已导出； - 在同一份库上运行上游的四个示例（[tools/run_conformance.py](tools/run_conformance.py)）。 另一个作业把默认模板当作已安装的包生成项目并运行。 diff --git a/include/mwrap/zstd/macros.hpp b/include/mwrap/zstd/macros.hpp index fbe41e5..d523054 100644 --- a/include/mwrap/zstd/macros.hpp +++ b/include/mwrap/zstd/macros.hpp @@ -2,7 +2,8 @@ // nor a type: they take arguments, or call a function. A named module cannot export a // macro: include this header before `import mcpplibs.mwrap.zstd;`. Each definition is // upstream's (Zstandard 1.5.7, BSD license or GPLv2), made only where the upstream header has -// not defined it, with the constants it names read from mcpplibs::mwrap::zstd::macros. +// not defined it, with the constants it names read from mcpplibs::mwrap::zstd::macros +// and NULL written nullptr. // The upstream header included after this one redefines these macros. // Written by tools/generate.py. #ifndef MWRAP_ZSTD_MACROS_HPP diff --git a/tests/linked.cpp b/tests/linked.cpp new file mode 100644 index 0000000..e20b93f --- /dev/null +++ b/tests/linked.cpp @@ -0,0 +1,83 @@ +// Every function that mcpplibs.mwrap.zstd exports, by address: the program links only +// where compat.zstd defines each one. Written by tools/generate.py. +import std; +import mcpplibs.mwrap.zstd; + +int main() +{ + // volatile: an optimiser may not drop a reference, so each one reaches the link. + const void* volatile functions[] = { + reinterpret_cast(&ZSTD_CCtx_loadDictionary), + reinterpret_cast(&ZSTD_CCtx_refCDict), + reinterpret_cast(&ZSTD_CCtx_refPrefix), + reinterpret_cast(&ZSTD_CCtx_reset), + reinterpret_cast(&ZSTD_CCtx_setParameter), + reinterpret_cast(&ZSTD_CCtx_setPledgedSrcSize), + reinterpret_cast(&ZSTD_CStreamInSize), + reinterpret_cast(&ZSTD_CStreamOutSize), + reinterpret_cast(&ZSTD_DCtx_loadDictionary), + reinterpret_cast(&ZSTD_DCtx_refDDict), + reinterpret_cast(&ZSTD_DCtx_refPrefix), + reinterpret_cast(&ZSTD_DCtx_reset), + reinterpret_cast(&ZSTD_DCtx_setParameter), + reinterpret_cast(&ZSTD_DStreamInSize), + reinterpret_cast(&ZSTD_DStreamOutSize), + reinterpret_cast(&ZSTD_cParam_getBounds), + reinterpret_cast(&ZSTD_compress), + reinterpret_cast(&ZSTD_compress2), + reinterpret_cast(&ZSTD_compressBound), + reinterpret_cast(&ZSTD_compressCCtx), + reinterpret_cast(&ZSTD_compressStream), + reinterpret_cast(&ZSTD_compressStream2), + reinterpret_cast(&ZSTD_compress_usingCDict), + reinterpret_cast(&ZSTD_compress_usingDict), + reinterpret_cast(&ZSTD_createCCtx), + reinterpret_cast(&ZSTD_createCDict), + reinterpret_cast(&ZSTD_createCStream), + reinterpret_cast(&ZSTD_createDCtx), + reinterpret_cast(&ZSTD_createDDict), + reinterpret_cast(&ZSTD_createDStream), + reinterpret_cast(&ZSTD_dParam_getBounds), + reinterpret_cast(&ZSTD_decompress), + reinterpret_cast(&ZSTD_decompressDCtx), + reinterpret_cast(&ZSTD_decompressStream), + reinterpret_cast(&ZSTD_decompress_usingDDict), + reinterpret_cast(&ZSTD_decompress_usingDict), + reinterpret_cast(&ZSTD_defaultCLevel), + reinterpret_cast(&ZSTD_endStream), + reinterpret_cast(&ZSTD_findFrameCompressedSize), + reinterpret_cast(&ZSTD_flushStream), + reinterpret_cast(&ZSTD_freeCCtx), + reinterpret_cast(&ZSTD_freeCDict), + reinterpret_cast(&ZSTD_freeCStream), + reinterpret_cast(&ZSTD_freeDCtx), + reinterpret_cast(&ZSTD_freeDDict), + reinterpret_cast(&ZSTD_freeDStream), + reinterpret_cast(&ZSTD_getDecompressedSize), + reinterpret_cast(&ZSTD_getDictID_fromCDict), + reinterpret_cast(&ZSTD_getDictID_fromDDict), + reinterpret_cast(&ZSTD_getDictID_fromDict), + reinterpret_cast(&ZSTD_getDictID_fromFrame), + reinterpret_cast(&ZSTD_getErrorCode), + reinterpret_cast(&ZSTD_getErrorName), + reinterpret_cast(&ZSTD_getErrorString), + reinterpret_cast(&ZSTD_getFrameContentSize), + reinterpret_cast(&ZSTD_initCStream), + reinterpret_cast(&ZSTD_initDStream), + reinterpret_cast(&ZSTD_isError), + reinterpret_cast(&ZSTD_maxCLevel), + reinterpret_cast(&ZSTD_minCLevel), + reinterpret_cast(&ZSTD_sizeof_CCtx), + reinterpret_cast(&ZSTD_sizeof_CDict), + reinterpret_cast(&ZSTD_sizeof_CStream), + reinterpret_cast(&ZSTD_sizeof_DCtx), + reinterpret_cast(&ZSTD_sizeof_DDict), + reinterpret_cast(&ZSTD_sizeof_DStream), + reinterpret_cast(&ZSTD_versionNumber), + reinterpret_cast(&ZSTD_versionString), + }; + int missing = 0; + for (const void* f : functions) missing += f == nullptr; + std::println("68 functions linked, {} null", missing); + return missing != 0; +} diff --git a/tools/generate.py b/tools/generate.py index a17ef0d..ae00443 100644 --- a/tools/generate.py +++ b/tools/generate.py @@ -2,8 +2,9 @@ """Write, or with --check audit, the parts of the wrapper that follow from the headers. tools/surface.toml names the headers and classifies what the compiler cannot: -the macros that are how the headers are written (`syntax`), and the names that -only some targets declare (`conditional`). Everything else follows from the +the macros that are how the headers are written (`syntax`), the names that +only some targets declare (`conditional`), and the functions that the compat +package does not build (`unbuilt`, not exported). Everything else follows from the headers as the compiler reads them (tools/api.py), with the include, define and target flags of the last `mcpp build` (target/**/compile_commands.json): @@ -14,8 +15,11 @@ mcpplibs::mwrap::::macros include/mwrap//macros.hpp every other macro, upstream's definition tests/macros.cpp each mapped constant and type against its macro + tests/linked.cpp the address of each exported function, so that + the link proves the compat package defines it -The include guards are recognised and omitted. --check renders the same files +The include guards and reserved identifiers (`__x`, `_X`) are recognised and +omitted. --check renders the same files in memory: on the host named by `generated-on` they must be byte-identical; on every host, each name the headers declare there must be exported, mapped, kept or classified, and each unconditional export must be declared there. @@ -65,7 +69,9 @@ def classify(cfg, compiler, flags): owned = cfg.get('owned', headers) surface, macros = api.extract(compiler, flags, headers, owned) guards = api.guards(macros) - syntax = set(cfg.get('syntax', [])) + # A reserved identifier (`__x`, `_X`) is the implementation's, never the library's API. + reserved = {n for n in macros if re.match(r'_[_A-Z]', n)} + syntax = set(cfg.get('syntax', [])) | reserved open_macros = {n: m for n, m in macros.items() if n not in guards and n not in syntax} constants = api.constant_macros(compiler, flags, headers, open_macros) types = api.type_macros(compiler, flags, headers, {n: m for n, m in open_macros.items() if n not in constants}) @@ -79,7 +85,7 @@ def render(cfg, surface, constants, types, kept): conditional = cfg.get('conditional', {}) includes = '\n'.join(f'#include <{h}>' for h in cfg['headers']) declared = surface['functions'] | surface['variables'] | surface['types'] | surface['enumerators'] - exported = sorted(declared - set(conditional)) + exported = sorted(declared - set(conditional) - set(cfg.get('unbuilt', []))) cap = f'mwrap_{name}_capture' module = [f"// mcpplibs.mwrap.{name} --- {cfg['title']} as a C++ module. Every exported name is", @@ -115,12 +121,14 @@ def render(cfg, surface, constants, types, kept): '// nor a type: they take arguments, or call a function. A named module cannot export a', f'// macro: include this header before `import mcpplibs.mwrap.{name};`. Each definition is', f"// upstream's ({cfg['title']}, {cfg['license']}), made only where the upstream header has", - f'// not defined it, with the constants it names read from {scope}::macros.', + f'// not defined it, with the constants it names read from {scope}::macros', + '// and NULL written nullptr.', '// The upstream header included after this one redefines these macros.', '// Written by tools/generate.py.', f'#ifndef {guard}', f'#define {guard}'] for n in sorted(kept): - body = qualify(kept[n]['body'], set(mapped), f'{scope}::macros') + # A module-only unit has no NULL macro; in C++ nullptr is its meaning. + body = re.sub(r'\bNULL\b', 'nullptr', qualify(kept[n]['body'], set(mapped), f'{scope}::macros')) header += ['', f'#ifndef {n}', f"#define {n}{kept[n]['params'] or ''} {body}".rstrip(), '#endif'] header += ['', f'#endif // {guard}'] @@ -142,9 +150,20 @@ def render(cfg, surface, constants, types, kept): test += [f' if (!same(m::{n}, header::value_{n})) {{ std::println("FAIL {n}"); ++differ; }}' for n in sorted(constants)] test += [f' std::println("{len(constants)} constants and {len(types)} types compared, {{}} differ", differ);', ' return differ != 0;', '}'] + functions = sorted(surface['functions'] - set(conditional) - set(cfg.get('unbuilt', []))) + linked = [f'// Every function that mcpplibs.mwrap.{name} exports, by address: the program links only', + f'// where compat.{cfg["compat"]} defines each one. Written by tools/generate.py.', + 'import std;', f'import mcpplibs.mwrap.{name};', '', 'int main()', '{', + ' // volatile: an optimiser may not drop a reference, so each one reaches the link.', + ' const void* volatile functions[] = {'] + linked += [f' reinterpret_cast(&{n}),' for n in functions] + linked += [' };', ' int missing = 0;', ' for (const void* f : functions) missing += f == nullptr;', + f' std::println("{len(functions)} functions linked, {{}} null", missing);', + ' return missing != 0;', '}'] join = lambda lines: '\n'.join(lines) + '\n' return {f'src/{name}.cppm': join(module), 'src/macros.cppm': join(macros), - f'include/mwrap/{name}/macros.hpp': join(header), 'tests/macros.cpp': join(test)} + f'include/mwrap/{name}/macros.hpp': join(header), 'tests/macros.cpp': join(test), + 'tests/linked.cpp': join(linked)} def main(): @@ -176,10 +195,11 @@ def main(): mapped = set(re.findall(r'^#undef (\w+)$', mapping, flags=re.M)) kept_written = set(re.findall(r'^#ifndef (\w+)$', (ROOT / f"include/mwrap/{cfg['name']}/macros.hpp") .read_text(encoding='utf-8'), flags=re.M)) - problems += [f'declared here, not exported: {n}' for n in sorted(declared - written)] + problems += [f'declared here, not exported: {n}' for n in sorted(declared - written - set(cfg.get('unbuilt', [])))] problems += [f'exported, not declared here: {n}' for n in sorted(unconditional - declared)] problems += [f'macro here, neither mapped, kept nor classified: {n}' for n in sorted( - set(macros) - guards - set(cfg.get('syntax', [])) - mapped - kept_written)] + set(macros) - guards - set(cfg.get('syntax', [])) - mapped - kept_written) + if not re.match(r'_[_A-Z]', n)] problems += [f'mapped, not a constant or type here: {n}' for n in sorted(mapped - constants - types)] problems += [f'internal linkage, cannot be exported: {n}' for n in sorted(surface['internal']) if n not in cfg.get('adapted', [])] From e4855fd68d268ef5e4b3770dab75aeea4406d3d3 Mon Sep 17 00:00:00 2001 From: Sunrisepeak Date: Sat, 7 Nov 2026 06:16:38 +0900 Subject: [PATCH 3/4] generate: named include guards; a macro header with nothing to keep says so Co-authored-by: speak-agent <248744407+speak-agent@users.noreply.github.com> --- tools/api.py | 15 ++++++++++----- tools/generate.py | 2 ++ 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/tools/api.py b/tools/api.py index 1627a90..1ee5ff8 100644 --- a/tools/api.py +++ b/tools/api.py @@ -81,11 +81,13 @@ def walk(nodes): def guards(macros): - """The include guards among `macros`: an empty macro that the first - conditional directive of its file tests (`#ifndef G` or `#if !defined(G)`).""" + """The include guards among `macros`. An empty macro that the first + conditional directive of its file tests (`#ifndef G` or `#if !defined(G)`), + or a macro named like a guard (`..._H`, `..._H_`, `..._INCLUDED`) + that its file defines right after `#ifndef G`.""" found = set() for name, macro in macros.items(): - if macro['params'] is not None or macro['body']: + if macro['params'] is not None: continue try: text = Path(macro['file']).read_text(encoding='utf-8', errors='replace') @@ -93,8 +95,11 @@ def guards(macros): continue text = re.sub(r'/\*.*?\*/', ' ', text, flags=re.S) first = re.search(r'^\s*#\s*if.*$', text, flags=re.M) - if first and re.fullmatch(r'\s*#\s*(?:ifndef\s+' + name + r'|if\s+!\s*defined\s*\(?\s*' + name + r'\s*\)?)\s*', - first.group(0)): + tested_first = first and re.fullmatch( + r'\s*#\s*(?:ifndef\s+' + name + r'|if\s+!\s*defined\s*\(?\s*' + name + r'\s*\)?)\s*', first.group(0)) + named = re.search(r'_H(?:_\w+)?$|_INCLUDED$', name) and re.search( + r'^\s*#\s*ifndef\s+' + name + r'\s*\n\s*#\s*define\s+' + name + r'\b', text, flags=re.M) + if (tested_first and not macro['body']) or named: found.add(name) return found diff --git a/tools/generate.py b/tools/generate.py index ae00443..801ce8f 100644 --- a/tools/generate.py +++ b/tools/generate.py @@ -130,6 +130,8 @@ def render(cfg, surface, constants, types, kept): # A module-only unit has no NULL macro; in C++ nullptr is its meaning. body = re.sub(r'\bNULL\b', 'nullptr', qualify(kept[n]['body'], set(mapped), f'{scope}::macros')) header += ['', f'#ifndef {n}', f"#define {n}{kept[n]['params'] or ''} {body}".rstrip(), '#endif'] + if not kept: + header += ['', '// The headers of this release define no such macro.'] header += ['', f'#endif // {guard}'] test = [f'// Each constant and type of mcpplibs.mwrap.{name}.macros equals its macro. A macro', From f8cc232bd2f3bda577249acdf3c648837fea0936 Mon Sep 17 00:00:00 2001 From: Sunrisepeak Date: Sat, 7 Nov 2026 06:23:06 +0900 Subject: [PATCH 4/4] generate: the configuration consumers share with the compat package (defines) Co-authored-by: speak-agent <248744407+speak-agent@users.noreply.github.com> --- tools/generate.py | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/tools/generate.py b/tools/generate.py index 801ce8f..bc63b7a 100644 --- a/tools/generate.py +++ b/tools/generate.py @@ -2,11 +2,13 @@ """Write, or with --check audit, the parts of the wrapper that follow from the headers. tools/surface.toml names the headers and classifies what the compiler cannot: -the macros that are how the headers are written (`syntax`), the names that -only some targets declare (`conditional`), and the functions that the compat -package does not build (`unbuilt`, not exported). Everything else follows from the -headers as the compiler reads them (tools/api.py), with the include, define -and target flags of the last `mcpp build` (target/**/compile_commands.json): +the configuration that consumers share with the compat package (`defines`, +defined before the headers), the macros that are how the headers are written +(`syntax`), the names that only some targets declare (`conditional`), and the +functions that the compat package does not build (`unbuilt`, not exported). +Everything else follows from the headers as the compiler reads them +(tools/api.py), with the include, define and target flags of the last +`mcpp build` (target/**/compile_commands.json): src/.cppm every function, variable, type and enumerator, `export using ::name` @@ -19,10 +21,10 @@ the link proves the compat package defines it The include guards and reserved identifiers (`__x`, `_X`) are recognised and -omitted. --check renders the same files -in memory: on the host named by `generated-on` they must be byte-identical; -on every host, each name the headers declare there must be exported, mapped, -kept or classified, and each unconditional export must be declared there. +omitted. --check renders the same files in memory: on the host named by +`generated-on` they must be byte-identical; on every host, each name the +headers declare there must be exported, mapped, kept or classified, and each +unconditional export must be declared there. The compiler is the clang of the build; run it after `mcpp build` with LLVM. """ import argparse @@ -65,6 +67,7 @@ def qualify(body, mapped, scope): def classify(cfg, compiler, flags): + flags = flags + [f'-D{d}' for d in cfg.get('defines', [])] headers = cfg['headers'] owned = cfg.get('owned', headers) surface, macros = api.extract(compiler, flags, headers, owned) @@ -83,7 +86,9 @@ def render(cfg, surface, constants, types, kept): name = cfg['name'] scope = f'mcpplibs::mwrap::{name}' conditional = cfg.get('conditional', {}) - includes = '\n'.join(f'#include <{h}>' for h in cfg['headers']) + # The configuration the compat package compiles with and its consumers must share. + includes = '\n'.join([f'#ifndef {d}\n#define {d}\n#endif' for d in cfg.get('defines', [])] + + [f'#include <{h}>' for h in cfg['headers']]) declared = surface['functions'] | surface['variables'] | surface['types'] | surface['enumerators'] exported = sorted(declared - set(conditional) - set(cfg.get('unbuilt', []))) cap = f'mwrap_{name}_capture'